fix: scanner.py - parallel github_stat scan

update DB
6 months ago · ef6b6ff690
7 changed files with 318 additions and 216 deletions
--- a/custom-node-list.json
+++ b/custom-node-list.json
@ -7801,6 +7801,7 @@
        {
            "author": "jtydhr88",
            "title": "ComfyUI-Workflow-Encrypt",
+            "id": "workflow-encrypt",
            "reference": "https://github.com/jtydhr88/ComfyUI-Workflow-Encrypt",
            "files": [
                "https://github.com/jtydhr88/ComfyUI-Workflow-Encrypt"
@ -7808,6 +7809,28 @@
            "install_type": "git-clone",
            "description": "Encrypt your comfyui workflow, and share it with key"
        },
+        {
+            "author": "jtydhr88",
+            "title": "ComfyUI-InstantMesh",
+            "id": "instant-mesh",
+            "reference": "https://github.com/jtydhr88/ComfyUI-InstantMesh",
+            "files": [
+                "https://github.com/jtydhr88/ComfyUI-InstantMesh"
+            ],
+            "install_type": "git-clone",
+            "description": "ComfyUI InstantMesh is custom nodes that running TencentARC/InstantMesh into ComfyUI, this extension depends on ComfyUI-3D-Pack. Please refer to Readme carefully to install."
+        },
+        {
+            "author": "jtydhr88",
+            "title": "ComfyUI LayerDivider",
+            "id": "layer-divider",
+            "reference": "https://github.com/jtydhr88/ComfyUI-LayerDivider",
+            "files": [
+                "https://github.com/jtydhr88/ComfyUI-LayerDivider"
+            ],
+            "install_type": "git-clone",
+            "description": "ComfyUI LayerDivider is custom nodes that generating layered psd files inside ComfyUI[w/This plugin depends on Python 3.10, which means we cannot use the default Python that comes with ComfyUI, as it is Python 3.11. For this reason, it is recommended to use conda to manage and create the ComfyUI runtime environment.]"
+        },
        {
            "author": "SeaArtLab",
            "title": "ComfyUI-Long-CLIP",
@ -8143,6 +8166,7 @@
        {
            "author": "turkyden",
            "title": "ComfyUI-Comic",
+            "id": "comic",
            "reference": "https://github.com/turkyden/ComfyUI-Comic",
            "files": [
                "https://github.com/turkyden/ComfyUI-Comic"
@ -8153,6 +8177,7 @@
        {
            "author": "royceschultz",
            "title": "ComfyUI-TranscriptionTools",
+            "id": "transcription-tools",
            "reference": "https://github.com/royceschultz/ComfyUI-TranscriptionTools",
            "files": [
                "https://github.com/royceschultz/ComfyUI-TranscriptionTools"
@ -8163,6 +8188,7 @@
        {
            "author": "kunieone",
            "title": "ComfyUI_alkaid",
+            "id": "alkadi",
            "reference": "https://github.com/kunieone/ComfyUI_alkaid",
            "files": [
                "https://github.com/kunieone/ComfyUI_alkaid"
@ -8170,19 +8196,10 @@
            "install_type": "git-clone",
            "description": "Nodes:A_Face3DSwapper, A_FaceCrop, A_FacePaste, A_OpenPosePreprocessor, A_EmptyLatentImageLongside, A_GetImageSize, AlkaidLoader, AdapterFaceLoader, AdapterStyleLoader, ..."
        },
-        {
-            "author": "jtydhr88",
-            "title": "ComfyUI-InstantMesh",
-            "reference": "https://github.com/jtydhr88/ComfyUI-InstantMesh",
-            "files": [
-                "https://github.com/jtydhr88/ComfyUI-InstantMesh"
-            ],
-            "install_type": "git-clone",
-            "description": "ComfyUI InstantMesh is custom nodes that running TencentARC/InstantMesh into ComfyUI, this extension depends on ComfyUI-3D-Pack. Please refer to Readme carefully to install."
-        },
        {
            "author": "txt2any",
            "title": "ComfyUI-PromptOrganizer",
+            "id": "prompt-organizer",
            "reference": "https://github.com/txt2any/ComfyUI-PromptOrganizer",
            "files": [
                "https://github.com/txt2any/ComfyUI-PromptOrganizer"
@ -8193,6 +8210,7 @@
        {
            "author": "kealiu",
            "title": "ComfyUI Load and Save file to S3",
+            "id": "savefile-to-s3",
            "reference": "https://github.com/kealiu/ComfyUI-S3-Tools",
            "files": [
                "https://github.com/kealiu/ComfyUI-S3-Tools"
@ -8203,6 +8221,7 @@
        {
            "author": "kealiu",
            "title": "ComfyUI-ZeroShot-MTrans",
+            "id": "zeroshot-mtrans",
            "reference": "https://github.com/kealiu/ComfyUI-ZeroShot-MTrans",
            "files": [
                "https://github.com/kealiu/ComfyUI-ZeroShot-MTrans"
@ -8213,6 +8232,7 @@
        {
            "author": "kealiu",
            "title": "ComfyUI-Zero123-Porting",
+            "id": "zero123-porting",
            "reference": "https://github.com/kealiu/ComfyUI-Zero123-Porting",
            "files": [
                "https://github.com/kealiu/ComfyUI-Zero123-Porting"
@ -8223,6 +8243,7 @@
        {
            "author": "TashaSkyUp",
            "title": "ComfyUI_LiteLLM",
+            "id": "litellm",
            "reference": "https://github.com/Hopping-Mad-Games/ComfyUI_LiteLLM",
            "files": [
                "https://github.com/Hopping-Mad-Games/ComfyUI_LiteLLM"
@ -8233,6 +8254,7 @@
        {
            "author": "AonekoSS",
            "title": "ComfyUI-SimpleCounter",
+            "id": "simplecounter",
            "reference": "https://github.com/AonekoSS/ComfyUI-SimpleCounter",
            "files": [
                "https://github.com/AonekoSS/ComfyUI-SimpleCounter"
@ -8243,6 +8265,7 @@
        {
            "author": "heshengtao",
            "title": "comfyui_LLM_party",
+            "id": "llm-party",
            "reference": "https://github.com/heshengtao/comfyui_LLM_party",
            "files": [
                "https://github.com/heshengtao/comfyui_LLM_party"
@ -8253,6 +8276,7 @@
        {
            "author": "VAST-AI-Research",
            "title": "Tripo for ComfyUI",
+            "id": "tripo",
            "reference": "https://github.com/VAST-AI-Research/ComfyUI-Tripo",
            "files": [
                "https://github.com/VAST-AI-Research/ComfyUI-Tripo"
@ -8263,6 +8287,7 @@
        {
            "author": "JettHu",
            "title": "ComfyUI_TGate",
+            "id": "tgate",
            "reference": "https://github.com/JettHu/ComfyUI_TGate",
            "files": [
                "https://github.com/JettHu/ComfyUI_TGate"
@ -8382,6 +8407,7 @@
        {
            "author": "lquesada",
            "title": "ComfyUI-Prompt-Combinator",
+            "id": "prompt-combinator",
            "reference": "https://github.com/lquesada/ComfyUI-Prompt-Combinator",
            "files": [
                "https://github.com/lquesada/ComfyUI-Prompt-Combinator"
@ -8392,6 +8418,7 @@
        {
            "author": "lquesada",
            "title": "ComfyUI-Inpaint-CropAndStitch",
+            "id": "crop-and-stitch",
            "reference": "https://github.com/lquesada/ComfyUI-Inpaint-CropAndStitch",
            "files": [
                "https://github.com/lquesada/ComfyUI-Inpaint-CropAndStitch"
@ -8412,6 +8439,7 @@
        {
            "author": "FredBill1",
            "title": "comfyui-fb-utils",
+            "id": "fb-utils",
            "reference": "https://github.com/FredBill1/comfyui-fb-utils",
            "files": [
                "https://github.com/FredBill1/comfyui-fb-utils"
@ -8421,7 +8449,8 @@
        },
        {
            "author": "jeffy5",
-            "title": "comfyui-fb-utils",
+            "title": "Faceless Node for ComfyUI",
+            "id": "faceless",
            "reference": "https://github.com/jeffy5/comfyui-faceless-node",
            "files": [
                "https://github.com/jeffy5/comfyui-faceless-node"
--- a/extension-node-map.json
+++ b/extension-node-map.json
@ -3964,6 +3964,7 @@
            "tri3d-atr-parse",
            "tri3d-atr-parse-batch",
            "tri3d-clipdrop-bgremove-api",
+            "tri3d-clipdrop-bgreplace-api",
            "tri3d-composite-image-splitter",
            "tri3d-dwpose",
            "tri3d-extract-hand",
@ -7059,6 +7060,7 @@
    ],
    "https://github.com/cubiq/ComfyUI_essentials": [
        [
+            "ApplyCLIPSeg+",
            "BatchCount+",
            "CLIPTextEncodeSDXL+",
            "ConditioningCombineMultiple+",
@ -7084,6 +7086,7 @@
            "ImageSeamCarving+",
            "KSamplerVariationsStochastic+",
            "KSamplerVariationsWithNoise+",
+            "LoadCLIPSegModels+",
            "MaskBatch+",
            "MaskBlur+",
            "MaskBoundingBox+",
@ -8265,14 +8268,15 @@
            "hello_world",
            "image_histogram_node",
            "image_histograms_node",
+            "image_histograms_self_node",
            "meme_generator_node"
        ],
        {
            "author": "YFG",
-            "description": "This extension just outputs Hello World! as a string.",
-            "nickname": "YFG Hello World",
-            "title": "YFG Hello World",
-            "title_aux": "ComfyUI_YFG_Comical"
+            "description": "This extension calculates the histogram of an image and outputs the results as graph images for individual channels as well as RGB and Luminosity.",
+            "nickname": "YFG Histograms",
+            "title": "YFG Histograms",
+            "title_aux": "\ud83d\ude38 YFG Comical Nodes"
        }
    ],
    "https://github.com/guill/abracadabra-comfyui": [
@ -8849,6 +8853,7 @@
            "FacelessLoadImageUrl",
            "FacelessLoadVideo",
            "FacelessLoadVideoUrl",
+            "FacelessMergeVideos",
            "FacelessRemoveBackground",
            "FacelessSaveVideo",
            "FacelessUploadVideo",
@ -8857,7 +8862,7 @@
            "FacelessVideoRemoveBackground"
        ],
        {
-            "title_aux": "comfyui-fb-utils"
+            "title_aux": "Faceless Node for ComfyUI"
        }
    ],
    "https://github.com/jesenzhang/ComfyUI_StreamDiffusion": [
@ -8945,6 +8950,17 @@
            "title_aux": "ComfyUI-InstantMesh"
        }
    ],
+    "https://github.com/jtydhr88/ComfyUI-LayerDivider": [
+        [
+            "LayerDivider - Color Base",
+            "LayerDivider - Divide Layer",
+            "LayerDivider - Load SAM Mask Generator",
+            "LayerDivider - Segment Mask"
+        ],
+        {
+            "title_aux": "ComfyUI LayerDivider"
+        }
+    ],
    "https://github.com/ka-puna/comfyui-yanc": [
        [
            "YANC.ConcatStrings",
@ -9283,6 +9299,7 @@
            "CreateVoronoiMask",
            "CrossFadeImages",
            "CustomSigmas",
+            "DownloadAndLoadCLIPSeg",
            "DrawInstanceDiffusionTracking",
            "DummyLatentOut",
            "EmptyLatentImagePresets",
@ -9314,6 +9331,7 @@
            "ImagePadForOutpaintMasked",
            "ImagePadForOutpaintTargetSize",
            "ImagePass",
+            "ImageResizeKJ",
            "ImageTransformByNormalizedAmplitude",
            "ImageUpscaleWithModelBatched",
            "InjectNoiseToLatent",
@ -10355,6 +10373,8 @@
            "PettyPaintComponent",
            "PettyPaintConditioningSetMaskAndCombine",
            "PettyPaintConvert",
+            "PettyPaintCountFiles",
+            "PettyPaintEnsureDirectory",
            "PettyPaintExec",
            "PettyPaintImageCompositeMasked",
            "PettyPaintImagePlacement",
@ -10367,6 +10387,7 @@
            "PettyPaintLoadImage",
            "PettyPaintLoadImages",
            "PettyPaintMap",
+            "PettyPaintProcessor",
            "PettyPaintRemoveAddText",
            "PettyPaintSDTurboScheduler",
            "PettyPaintText",
@ -11036,6 +11057,7 @@
            "CharacteristicGuidancePrediction",
            "CombinePredictions",
            "ConditionedPrediction",
+            "EarlyMiddleLatePrediction",
            "InterpolatePredictions",
            "LogSigmas",
            "PerpNegPrediction",
@ -11217,7 +11239,8 @@
    "https://github.com/saftle/suplex_comfy_nodes": [
        [
            "ControlNet Selector",
-            "ControlNetOptionalLoader"
+            "ControlNetOptionalLoader",
+            "DiffusersSelector"
        ],
        {
            "title_aux": "Suplex Misc ComfyUI Nodes"
@ -11867,7 +11890,8 @@
    "https://github.com/sugarkwork/comfyui_tag_fillter": [
        [
            "TagFilter",
-            "TagRemover"
+            "TagRemover",
+            "TagReplace"
        ],
        {
            "title_aux": "comfyui_tag_filter"
@ -12957,7 +12981,8 @@
            "IntAndIntAddOffsetLiteral",
            "IntMultipleAddLiteral",
            "LoadImageWithSwitch",
-            "ModifyTextGender"
+            "ModifyTextGender",
+            "SplitMask"
        ],
        {
            "title_aux": "zhangp365/Some Utils for ComfyUI"
--- a/github-stats.json
+++ b/github-stats.json
--- a/node_db/new/custom-node-list.json
+++ b/node_db/new/custom-node-list.json
@ -11,6 +11,17 @@


        
+        {
+            "author": "jtydhr88",
+            "title": "ComfyUI LayerDivider",
+            "id": "layer-divider",
+            "reference": "https://github.com/jtydhr88/ComfyUI-LayerDivider",
+            "files": [
+                "https://github.com/jtydhr88/ComfyUI-LayerDivider"
+            ],
+            "install_type": "git-clone",
+            "description": "ComfyUI LayerDivider is custom nodes that generating layered psd files inside ComfyUI[w/This plugin depends on Python 3.10, which means we cannot use the default Python that comes with ComfyUI, as it is Python 3.11. For this reason, it is recommended to use conda to manage and create the ComfyUI runtime environment.]"
+        },
        {
            "author": "ShmuelRonen",
            "title": "Wav2Lip Node for ComfyUI",
@ -664,7 +675,8 @@
        },
        {
            "author": "jeffy5",
-            "title": "comfyui-fb-utils",
+            "title": "Faceless Node for ComfyUI",
+            "id": "faceless",
            "reference": "https://github.com/jeffy5/comfyui-faceless-node",
            "files": [
                "https://github.com/jeffy5/comfyui-faceless-node"
@ -681,16 +693,6 @@
            ],
            "install_type": "git-clone",
            "description": "Nodes:SimDATrain, SimDALoader, SimDARun, VHS_FILENAMES_STRING_SimDA"
-        },
-        {
-            "author": "randjtw",
-            "title": "advance-aesthetic-score",
-            "reference": "https://github.com/randjtw/advance-aesthetic-score",
-            "files": [
-                "https://github.com/randjtw/advance-aesthetic-score"
-            ],
-            "install_type": "git-clone",
-            "description": "Nodes:Advance Aesthetic Score"
        }
    ]
 }
--- a/node_db/new/extension-node-map.json
+++ b/node_db/new/extension-node-map.json
@ -3964,6 +3964,7 @@
            "tri3d-atr-parse",
            "tri3d-atr-parse-batch",
            "tri3d-clipdrop-bgremove-api",
+            "tri3d-clipdrop-bgreplace-api",
            "tri3d-composite-image-splitter",
            "tri3d-dwpose",
            "tri3d-extract-hand",
@ -7059,6 +7060,7 @@
    ],
    "https://github.com/cubiq/ComfyUI_essentials": [
        [
+            "ApplyCLIPSeg+",
            "BatchCount+",
            "CLIPTextEncodeSDXL+",
            "ConditioningCombineMultiple+",
@ -7084,6 +7086,7 @@
            "ImageSeamCarving+",
            "KSamplerVariationsStochastic+",
            "KSamplerVariationsWithNoise+",
+            "LoadCLIPSegModels+",
            "MaskBatch+",
            "MaskBlur+",
            "MaskBoundingBox+",
@ -8265,14 +8268,15 @@
            "hello_world",
            "image_histogram_node",
            "image_histograms_node",
+            "image_histograms_self_node",
            "meme_generator_node"
        ],
        {
            "author": "YFG",
-            "description": "This extension just outputs Hello World! as a string.",
-            "nickname": "YFG Hello World",
-            "title": "YFG Hello World",
-            "title_aux": "ComfyUI_YFG_Comical"
+            "description": "This extension calculates the histogram of an image and outputs the results as graph images for individual channels as well as RGB and Luminosity.",
+            "nickname": "YFG Histograms",
+            "title": "YFG Histograms",
+            "title_aux": "\ud83d\ude38 YFG Comical Nodes"
        }
    ],
    "https://github.com/guill/abracadabra-comfyui": [
@ -8849,6 +8853,7 @@
            "FacelessLoadImageUrl",
            "FacelessLoadVideo",
            "FacelessLoadVideoUrl",
+            "FacelessMergeVideos",
            "FacelessRemoveBackground",
            "FacelessSaveVideo",
            "FacelessUploadVideo",
@ -8857,7 +8862,7 @@
            "FacelessVideoRemoveBackground"
        ],
        {
-            "title_aux": "comfyui-fb-utils"
+            "title_aux": "Faceless Node for ComfyUI"
        }
    ],
    "https://github.com/jesenzhang/ComfyUI_StreamDiffusion": [
@ -8945,6 +8950,17 @@
            "title_aux": "ComfyUI-InstantMesh"
        }
    ],
+    "https://github.com/jtydhr88/ComfyUI-LayerDivider": [
+        [
+            "LayerDivider - Color Base",
+            "LayerDivider - Divide Layer",
+            "LayerDivider - Load SAM Mask Generator",
+            "LayerDivider - Segment Mask"
+        ],
+        {
+            "title_aux": "ComfyUI LayerDivider"
+        }
+    ],
    "https://github.com/ka-puna/comfyui-yanc": [
        [
            "YANC.ConcatStrings",
@ -9283,6 +9299,7 @@
            "CreateVoronoiMask",
            "CrossFadeImages",
            "CustomSigmas",
+            "DownloadAndLoadCLIPSeg",
            "DrawInstanceDiffusionTracking",
            "DummyLatentOut",
            "EmptyLatentImagePresets",
@ -9314,6 +9331,7 @@
            "ImagePadForOutpaintMasked",
            "ImagePadForOutpaintTargetSize",
            "ImagePass",
+            "ImageResizeKJ",
            "ImageTransformByNormalizedAmplitude",
            "ImageUpscaleWithModelBatched",
            "InjectNoiseToLatent",
@ -10355,6 +10373,8 @@
            "PettyPaintComponent",
            "PettyPaintConditioningSetMaskAndCombine",
            "PettyPaintConvert",
+            "PettyPaintCountFiles",
+            "PettyPaintEnsureDirectory",
            "PettyPaintExec",
            "PettyPaintImageCompositeMasked",
            "PettyPaintImagePlacement",
@ -10367,6 +10387,7 @@
            "PettyPaintLoadImage",
            "PettyPaintLoadImages",
            "PettyPaintMap",
+            "PettyPaintProcessor",
            "PettyPaintRemoveAddText",
            "PettyPaintSDTurboScheduler",
            "PettyPaintText",
@ -11036,6 +11057,7 @@
            "CharacteristicGuidancePrediction",
            "CombinePredictions",
            "ConditionedPrediction",
+            "EarlyMiddleLatePrediction",
            "InterpolatePredictions",
            "LogSigmas",
            "PerpNegPrediction",
@ -11217,7 +11239,8 @@
    "https://github.com/saftle/suplex_comfy_nodes": [
        [
            "ControlNet Selector",
-            "ControlNetOptionalLoader"
+            "ControlNetOptionalLoader",
+            "DiffusersSelector"
        ],
        {
            "title_aux": "Suplex Misc ComfyUI Nodes"
@ -11867,7 +11890,8 @@
    "https://github.com/sugarkwork/comfyui_tag_fillter": [
        [
            "TagFilter",
-            "TagRemover"
+            "TagRemover",
+            "TagReplace"
        ],
        {
            "title_aux": "comfyui_tag_filter"
@ -12957,7 +12981,8 @@
            "IntAndIntAddOffsetLiteral",
            "IntMultipleAddLiteral",
            "LoadImageWithSwitch",
-            "ModifyTextGender"
+            "ModifyTextGender",
+            "SplitMask"
        ],
        {
            "title_aux": "zhangp365/Some Utils for ComfyUI"
--- a/node_db/tutorial/custom-node-list.json
+++ b/node_db/tutorial/custom-node-list.json
@ -189,6 +189,16 @@
            ],
            "install_type": "git-clone",
            "description": "Custom utility nodes for ComfyUI"
+        },
+        {
+            "author": "nilor-corp",
+            "title": "nilor-nodes",
+            "reference": "https://github.com/nilor-corp/nilor-nodes",
+            "files": [
+                "https://github.com/nilor-corp/nilor-nodes"
+            ],
+            "install_type": "git-clone",
+            "description": "Nodes:Nilor Floats, Nilor Int To List Of Bools, Nilor Bool From List Of Bools, Nilor Int From List Of Ints, Nilor List of Ints, Nilor Count Images In Directory"
        }
    ]
 }
--- a/scanner.py
+++ b/scanner.py
@ -317,9 +317,6 @@ def update_custom_nodes():
                        url, item = url_item
                        github_stats[url] = item

-            with open('github-stats-cache.json', 'w', encoding='utf-8') as file:
-                json.dump(github_stats, file, ensure_ascii=False, indent=4)
-
            # renew outdated cache
            outdated_urls = []
            for k, v in github_stats.items():
@ -327,8 +324,18 @@ def update_custom_nodes():
                if elapsed > 60*60*12:  # 12 hours
                    outdated_urls.append(k)

-            for url in outdated_urls:
-                renew_stat(url)
+            with concurrent.futures.ThreadPoolExecutor(11) as executor:
+                for url in outdated_urls:
+                    futures.append(executor.submit(renew_stat, url))
+
+                for future in concurrent.futures.as_completed(futures):
+                    url_item = future.result()
+                    if url_item is not None:
+                        url, item = url_item
+                        github_stats[url] = item
+                        
+            with open('github-stats-cache.json', 'w', encoding='utf-8') as file:
+                json.dump(github_stats, file, ensure_ascii=False, indent=4)

        with open(GITHUB_STATS_FILENAME, 'w', encoding='utf-8') as file:
            for v in github_stats.values():