diff --git a/README.md b/README.md index 81d3317..f996124 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,9 @@ From v1.5, v1.x goes into maintenance phase. Enjoy! # Change Log +## v1.5.5 +* update SHA256 function, now it just use the code from pip + ## v1.5.4 * set sys.stdout to utf-8 * Add default header for requests to prevent from being blocked by civitai. diff --git a/scripts/lib/util.py b/scripts/lib/util.py index 46b3967..c23ae40 100644 --- a/scripts/lib/util.py +++ b/scripts/lib/util.py @@ -1,10 +1,11 @@ # -*- coding: UTF-8 -*- import os +import io import hashlib import requests import shutil -version = "1.5.4" +version = "1.5.5" def_headers = {'User-Agent': 'Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148'} @@ -14,20 +15,44 @@ def printD(msg): print(f"Civitai Helper: {msg}") -def gen_file_sha256(filname): - printD("Calculate SHA256") - hash_sha256 = hashlib.sha256() - with open(filname, "rb") as f: - # force to use Memory Optimized SHA256 - # In case people don't understand this and uncheck it then stuck their system - printD("Using Memory Optimized SHA256") - for chunk in iter(lambda: f.read(4096), b""): - hash_sha256.update(chunk) +def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE): + """Yield pieces of data from a file-like object until EOF.""" + while True: + chunk = file.read(size) + if not chunk: + break + yield chunk - hash_value = hash_sha256.hexdigest() +# Now, hashing use the same way as pip's source code. +def gen_file_sha256(filname): + printD("Use Memory Optimized SHA256") + blocksize=1 << 20 + h = hashlib.sha256() + length = 0 + with open(filname, 'rb') as f: + for block in read_chunks(f, size=blocksize): + length += len(block) + h.update(block) + + hash_value = h.hexdigest() printD("sha256: " + hash_value) + printD("length: " + str(length)) return hash_value +# def gen_file_sha256(filname): +# printD("Calculate SHA256") +# # force to use Memory Optimized SHA256 +# # In case people don't understand this and uncheck it then stuck their system +# printD("Using Memory Optimized SHA256") +# hash_sha256 = hashlib.sha256() +# with open(filname, "rb") as f: +# for chunk in iter(lambda: f.read(4096), b""): +# hash_sha256.update(chunk) + +# hash_value = hash_sha256.hexdigest() +# printD("sha256: " + hash_value) +# return hash_value + # get preview image def download_file(url, path):