v1.5.5

README.md

@@ -168,6 +168,9 @@ From v1.5, v1.x goes into maintenance phase.
 Enjoy!
 
 # Change Log
+## v1.5.5
+* update SHA256 function, now it just use the code from pip
+
 ## v1.5.4
 * set sys.stdout to utf-8
 * Add default header for requests to prevent from being blocked by civitai.

scripts/lib/util.py

@@ -1,10 +1,11 @@
 # -*- coding: UTF-8 -*-
 import os
+import io
 import hashlib
 import requests
 import shutil
 
-version = "1.5.4"
+version = "1.5.5"
 
 def_headers = {'User-Agent': 'Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148'}
 
@@ -14,20 +15,44 @@ def printD(msg):
     print(f"Civitai Helper: {msg}")
 
 
-def gen_file_sha256(filname):
+def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE):
-    printD("Calculate SHA256")
+    """Yield pieces of data from a file-like object until EOF."""
-    hash_sha256 = hashlib.sha256()
+    while True:
-    with open(filname, "rb") as f:
+        chunk = file.read(size)
-        # force to use Memory Optimized SHA256
+        if not chunk:
-        # In case people don't understand this and uncheck it then stuck their system
+            break
-        printD("Using Memory Optimized SHA256")
+        yield chunk
-        for chunk in iter(lambda: f.read(4096), b""):
-            hash_sha256.update(chunk)
 
-    hash_value =  hash_sha256.hexdigest()
+# Now, hashing use the same way as pip's source code.
+def gen_file_sha256(filname):
+    printD("Use Memory Optimized SHA256")
+    blocksize=1 << 20
+    h = hashlib.sha256()
+    length = 0
+    with open(filname, 'rb') as f:
+        for block in read_chunks(f, size=blocksize):
+            length += len(block)
+            h.update(block)
+
+    hash_value =  h.hexdigest()
     printD("sha256: " + hash_value)
+    printD("length: " + str(length))
     return hash_value
 
+# def gen_file_sha256(filname):
+#     printD("Calculate SHA256")
+#     # force to use Memory Optimized SHA256
+#     # In case people don't understand this and uncheck it then stuck their system
+#     printD("Using Memory Optimized SHA256")
+#     hash_sha256 = hashlib.sha256()
+#     with open(filname, "rb") as f:
+#         for chunk in iter(lambda: f.read(4096), b""):
+#             hash_sha256.update(chunk)
+
+#     hash_value =  hash_sha256.hexdigest()
+#     printD("sha256: " + hash_value)
+#     return hash_value
+
 
 # get preview image
 def download_file(url, path):