From 0b319e453f3fa27b56ddc030e661c8a86593fcad Mon Sep 17 00:00:00 2001 From: Thomas Ward Date: Thu, 4 Jul 2024 14:28:19 -0400 Subject: [PATCH 1/9] cli_args: Add --duplicate-check-hash-function. --- comfy/cli_args.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/comfy/cli_args.py b/comfy/cli_args.py index b72bf3998ae..c153ee4d847 100644 --- a/comfy/cli_args.py +++ b/comfy/cli_args.py @@ -109,6 +109,8 @@ class LatentPreviewMethod(enum.Enum): vram_group.add_argument("--novram", action="store_true", help="When lowvram isn't enough.") vram_group.add_argument("--cpu", action="store_true", help="To use the CPU for everything (slow).") +parser.add_argument("--duplicate-check-hash-function", type=str, choices=['md5', 'sha1', 'sha256', 'sha512'], default='sha256', help="Allows you to choose the hash function to use for duplicate filename / contents comparison. Default is sha256.") + parser.add_argument("--disable-smart-memory", action="store_true", help="Force ComfyUI to agressively offload to regular ram instead of keeping models in vram when it can.") parser.add_argument("--deterministic", action="store_true", help="Make pytorch use slower deterministic algorithms when it can. Note that this might not make images deterministic in all cases.") From a065a9081f3a2be90903ea6f9a5cf27899dd0c38 Mon Sep 17 00:00:00 2001 From: Thomas Ward Date: Thu, 4 Jul 2024 14:38:18 -0400 Subject: [PATCH 2/9] server.py: compare_image_hash configurable hash function Uses an argument added in cli_args to specify the type of hashing to default to for duplicate hash checking. Uses an `eval()` to identify the specific hashlib class to utilize, but ultimately safely operates because we have specific options and only those options/choices in the arg parser. So we don't have any unsafe input there. --- server.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/server.py b/server.py index 38b1bab80f0..dbcfad27a1a 100644 --- a/server.py +++ b/server.py @@ -156,10 +156,16 @@ def get_dir_by_type(dir_type): return type_dir, dir_type def compare_image_hash(filepath, image): + # This eval operation is *safe* because we can only accept certain strings passed in via args. + # Because that can only be one of 'md5', 'sha1', 'sha256', or 'sha512' which we already define + # in comfy.cli_args as an argument passable to the args and only from a set of options, we can + # safely use an eval here, as the eval data is "safe" already. + hasher = eval(f"hashlib.{args.duplicate_check_hash_function}") + # function to compare hashes of two images to see if it already exists, fix to #3465 if os.path.exists(filepath): - a = hashlib.sha256() - b = hashlib.sha256() + a = hasher() + b = hasher() with open(filepath, "rb") as f: a.update(f.read()) b.update(image.file.read()) From ac835a45e1f228cbe788c8f96f884e55b2563422 Mon Sep 17 00:00:00 2001 From: Thomas Ward Date: Thu, 4 Jul 2024 15:05:08 -0400 Subject: [PATCH 3/9] Add hasher() to node_helpers --- node_helpers.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/node_helpers.py b/node_helpers.py index 43b9e829f59..9ee907dcaf5 100644 --- a/node_helpers.py +++ b/node_helpers.py @@ -22,3 +22,8 @@ def pillow(fn, arg): if prev_value is not None: ImageFile.LOAD_TRUNCATED_IMAGES = prev_value return x + +def hasher(): + # This is a safe eval because args.duplicate_check_hash_function can ONLY + # be one of four predefined strings, keeping it as a safe eval. + return eval(f"hashlib.{args.duplicate_check_hash_function}") From a4aeed0593b29a216e790bb97d25eaf92fcd67fd Mon Sep 17 00:00:00 2001 From: Thomas Ward Date: Thu, 4 Jul 2024 15:06:15 -0400 Subject: [PATCH 4/9] hashlib selection moved to node_helpers --- server.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/server.py b/server.py index dbcfad27a1a..c665c106cac 100644 --- a/server.py +++ b/server.py @@ -25,6 +25,7 @@ from comfy.cli_args import args import comfy.utils import comfy.model_management +import node_helpers from app.user_manager import UserManager @@ -156,11 +157,7 @@ def get_dir_by_type(dir_type): return type_dir, dir_type def compare_image_hash(filepath, image): - # This eval operation is *safe* because we can only accept certain strings passed in via args. - # Because that can only be one of 'md5', 'sha1', 'sha256', or 'sha512' which we already define - # in comfy.cli_args as an argument passable to the args and only from a set of options, we can - # safely use an eval here, as the eval data is "safe" already. - hasher = eval(f"hashlib.{args.duplicate_check_hash_function}") + hasher = node_helpers.hasher() # function to compare hashes of two images to see if it already exists, fix to #3465 if os.path.exists(filepath): From f491194440b7b788781d7279b266961a469a6abd Mon Sep 17 00:00:00 2001 From: Thomas Ward Date: Thu, 4 Jul 2024 15:08:52 -0400 Subject: [PATCH 5/9] default-hashing-function instead of dupe checking hasher This makes a default-hashing-function option instead of previous selected option. --- comfy/cli_args.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/comfy/cli_args.py b/comfy/cli_args.py index c153ee4d847..00401662565 100644 --- a/comfy/cli_args.py +++ b/comfy/cli_args.py @@ -109,8 +109,7 @@ class LatentPreviewMethod(enum.Enum): vram_group.add_argument("--novram", action="store_true", help="When lowvram isn't enough.") vram_group.add_argument("--cpu", action="store_true", help="To use the CPU for everything (slow).") -parser.add_argument("--duplicate-check-hash-function", type=str, choices=['md5', 'sha1', 'sha256', 'sha512'], default='sha256', help="Allows you to choose the hash function to use for duplicate filename / contents comparison. Default is sha256.") - +parser.add_argument("--default-hashing-function", type=str, choices=['md5', 'sha1', 'sha256', 'sha512'], default='sha256', help="Allows you to choose the hash function to use for duplicate filename / contents comparison. Default is sha256.") parser.add_argument("--disable-smart-memory", action="store_true", help="Force ComfyUI to agressively offload to regular ram instead of keeping models in vram when it can.") parser.add_argument("--deterministic", action="store_true", help="Make pytorch use slower deterministic algorithms when it can. Note that this might not make images deterministic in all cases.") From a0033b7bc8166d6fd52a44a291f93257a28fc966 Mon Sep 17 00:00:00 2001 From: Thomas Ward Date: Thu, 4 Jul 2024 15:10:40 -0400 Subject: [PATCH 6/9] Use args.default_hashing_function --- node_helpers.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/node_helpers.py b/node_helpers.py index 9ee907dcaf5..8e87ae7256e 100644 --- a/node_helpers.py +++ b/node_helpers.py @@ -1,3 +1,5 @@ +from comfy.cli_args import args + from PIL import ImageFile, UnidentifiedImageError def conditioning_set_values(conditioning, values={}): @@ -24,6 +26,6 @@ def pillow(fn, arg): return x def hasher(): - # This is a safe eval because args.duplicate_check_hash_function can ONLY + # This is a safe eval because args.default_hashing_function can ONLY # be one of four predefined strings, keeping it as a safe eval. - return eval(f"hashlib.{args.duplicate_check_hash_function}") + return eval(f"hashlib.{args.default_hashing_function}") From e1de04604a7209acbbaa0e028f11fa8ef4b4575d Mon Sep 17 00:00:00 2001 From: Thomas Ward Date: Mon, 15 Jul 2024 21:22:06 -0400 Subject: [PATCH 7/9] Use safer handling for node_helpers.hasher() Uses a safer handling method than `eval` to evaluate default hashing function. --- node_helpers.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/node_helpers.py b/node_helpers.py index 8e87ae7256e..e1c80c1b7a7 100644 --- a/node_helpers.py +++ b/node_helpers.py @@ -1,3 +1,5 @@ +import hashlib + from comfy.cli_args import args from PIL import ImageFile, UnidentifiedImageError @@ -26,6 +28,10 @@ def pillow(fn, arg): return x def hasher(): - # This is a safe eval because args.default_hashing_function can ONLY - # be one of four predefined strings, keeping it as a safe eval. - return eval(f"hashlib.{args.default_hashing_function}") + hashfuncs = { + "md5": hashlib.md5, + "sha1": hashlib.sha1, + "sha256": hashlib.sha256, + "sha512": hashlib.sha512 + } + return hashfuncs[args.default_hashing_function]) From 8e045e0b598089c2c885d83d3a22660d00f33274 Mon Sep 17 00:00:00 2001 From: Thomas Ward Date: Tue, 16 Jul 2024 00:40:55 -0400 Subject: [PATCH 8/9] Stray parentheses are evil. --- node_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_helpers.py b/node_helpers.py index e1c80c1b7a7..8c69eaa3e3a 100644 --- a/node_helpers.py +++ b/node_helpers.py @@ -34,4 +34,4 @@ def hasher(): "sha256": hashlib.sha256, "sha512": hashlib.sha512 } - return hashfuncs[args.default_hashing_function]) + return hashfuncs[args.default_hashing_function] From c4467b4e96ea87cba1063f7e1ccd89b6b81a3f07 Mon Sep 17 00:00:00 2001 From: Thomas Ward Date: Tue, 16 Jul 2024 11:23:36 -0400 Subject: [PATCH 9/9] Indentation fix. Somehow when I hit save I didn't notice I missed a space to make indentation work proper. Oops! --- node_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_helpers.py b/node_helpers.py index 8c69eaa3e3a..fee6287901b 100644 --- a/node_helpers.py +++ b/node_helpers.py @@ -34,4 +34,4 @@ def hasher(): "sha256": hashlib.sha256, "sha512": hashlib.sha512 } - return hashfuncs[args.default_hashing_function] + return hashfuncs[args.default_hashing_function]