From 3510b5f7287d4c46754eaf1ceca2f609c34205cb Mon Sep 17 00:00:00 2001 From: Ivory Date: Sun, 13 Aug 2023 21:07:49 -0400 Subject: [PATCH 1/4] add temporary database copying --- preload.py | 8 ++++++++ scripts/wib/wib_db.py | 20 ++++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 preload.py diff --git a/preload.py b/preload.py new file mode 100644 index 0000000..29095d7 --- /dev/null +++ b/preload.py @@ -0,0 +1,8 @@ + + +def preload(parser): + parser.add_argument( + "--image-browser-tmp-db", + action="store_true", + help="Copy database file to and from /tmp when transacting (workaround for filesystems sqlite does not support)" + ) \ No newline at end of file diff --git a/scripts/wib/wib_db.py b/scripts/wib/wib_db.py index 32e1082..4ba4263 100644 --- a/scripts/wib/wib_db.py +++ b/scripts/wib/wib_db.py @@ -2,7 +2,8 @@ import hashlib import json import os import sqlite3 -from modules import scripts +from shutil import copy2 +from modules import scripts, shared from PIL import Image version = 7 @@ -12,7 +13,21 @@ aes_cache_file = os.path.join(scripts.basedir(), "aes_scores.json") exif_cache_file = os.path.join(scripts.basedir(), "exif_data.json") ranking_file = os.path.join(scripts.basedir(), "ranking.json") archive = os.path.join(scripts.basedir(), "archive") -db_file = os.path.join(scripts.basedir(), "wib.sqlite3") +source_db_file = os.path.join(scripts.basedir(), "wib.sqlite3") +tmp_db_file = "/tmp/sd-images-browser.sqlite3" + +db_file = source_db_file +if getattr(shared.cmd_opts, "image_browser_tmp_db", False): + db_file = tmp_db_file + if os.path.exists(source_db_file): + copy2(source_db_file, tmp_db_file) + elif os.path.exists(tmp_db_file): + os.remove(tmp_db_file) + +def backup_tmp_db(): + if(db_file == tmp_db_file): + copy2(tmp_db_file, source_db_file) + np = "Negative prompt: " st = "Steps: " timeout = 30 @@ -659,6 +674,7 @@ def transaction_begin(): def transaction_end(conn, cursor): cursor.execute("COMMIT") conn.close() + backup_tmp_db() return def update_exif_data_by_key(cursor, file, key, value): From b0ffbaf4cad7ab069df9aac09baac8578eb03cea Mon Sep 17 00:00:00 2001 From: Ivory Date: Tue, 15 Aug 2023 07:42:47 -0400 Subject: [PATCH 2/4] normalize transactions & cross platform tempdir --- scripts/wib/wib_db.py | 117 +++++++++++++++++++----------------------- 1 file changed, 53 insertions(+), 64 deletions(-) diff --git a/scripts/wib/wib_db.py b/scripts/wib/wib_db.py index 4ba4263..f3023c8 100644 --- a/scripts/wib/wib_db.py +++ b/scripts/wib/wib_db.py @@ -4,7 +4,9 @@ import os import sqlite3 from shutil import copy2 from modules import scripts, shared +from tempfile import gettempdir from PIL import Image +from contextlib import contextmanager version = 7 @@ -14,7 +16,7 @@ exif_cache_file = os.path.join(scripts.basedir(), "exif_data.json") ranking_file = os.path.join(scripts.basedir(), "ranking.json") archive = os.path.join(scripts.basedir(), "archive") source_db_file = os.path.join(scripts.basedir(), "wib.sqlite3") -tmp_db_file = "/tmp/sd-images-browser.sqlite3" +tmp_db_file = os.path.join(gettempdir(), "sd-images-browser.sqlite3") db_file = source_db_file if getattr(shared.cmd_opts, "image_browser_tmp_db", False): @@ -32,6 +34,19 @@ np = "Negative prompt: " st = "Steps: " timeout = 30 +@contextmanager +def transaction(db = db_file): + conn = sqlite3.connect(db, timeout=timeout) + try: + conn.isolation_level = None + cursor = conn.cursor() + cursor.execute("BEGIN") + yield cursor + cursor.execute("COMMIT") + finally: + conn.close() + backup_tmp_db() + def create_filehash(cursor): cursor.execute(''' CREATE TABLE IF NOT EXISTS filehash ( @@ -307,8 +322,7 @@ def update_db_data(cursor, key, value): return def get_version(): - with sqlite3.connect(db_file, timeout=timeout) as conn: - cursor = conn.cursor() + with transaction() as cursor: cursor.execute(''' SELECT value FROM db_data @@ -319,8 +333,7 @@ def get_version(): return db_version def get_last_default_tab(): - with sqlite3.connect(db_file, timeout=timeout) as conn: - cursor = conn.cursor() + with transaction() as cursor: cursor.execute(''' SELECT value FROM db_data @@ -439,42 +452,40 @@ def migrate_ranking_dirs(cursor, db_version): def check(): if not os.path.exists(db_file): - conn, cursor = transaction_begin() print("Image Browser: Creating database") - create_db(cursor) - update_db_data(cursor, "version", version) - update_db_data(cursor, "last_default_tab", "Maintenance") - migrate_path_recorder(cursor) - migrate_exif_data(cursor) - migrate_ranking(cursor) - migrate_filehash(cursor, str(version)) - transaction_end(conn, cursor) + with transaction() as cursor: + create_db(cursor) + update_db_data(cursor, "version", version) + update_db_data(cursor, "last_default_tab", "Maintenance") + migrate_path_recorder(cursor) + migrate_exif_data(cursor) + migrate_ranking(cursor) + migrate_filehash(cursor, str(version)) print("Image Browser: Database created") db_version = get_version() - conn, cursor = transaction_begin() - if db_version[0] <= "2": - # version 1 database had mixed path notations, changed them all to abspath - # version 2 database still had mixed path notations, because of windows short name, changed them all to realpath - print(f"Image Browser: Upgrading database from version {db_version[0]} to version {version}") - migrate_path_recorder_dirs(cursor) - migrate_exif_data_dirs(cursor) - migrate_ranking_dirs(cursor, db_version[0]) - if db_version[0] <= "4": - migrate_filehash(cursor, db_version[0]) - if db_version[0] <= "5": - migrate_work_files(cursor) - if db_version[0] <= "6": - update_db_data(cursor, "last_default_tab", "Others") + + with transaction() as cursor: + if db_version[0] <= "2": + # version 1 database had mixed path notations, changed them all to abspath + # version 2 database still had mixed path notations, because of windows short name, changed them all to realpath + print(f"Image Browser: Upgrading database from version {db_version[0]} to version {version}") + migrate_path_recorder_dirs(cursor) + migrate_exif_data_dirs(cursor) + migrate_ranking_dirs(cursor, db_version[0]) + if db_version[0] <= "4": + migrate_filehash(cursor, db_version[0]) + if db_version[0] <= "5": + migrate_work_files(cursor) + if db_version[0] <= "6": + update_db_data(cursor, "last_default_tab", "Others") - update_db_data(cursor, "version", version) - print(f"Image Browser: Database upgraded from version {db_version[0]} to version {version}") - transaction_end(conn, cursor) + update_db_data(cursor, "version", version) + print(f"Image Browser: Database upgraded from version {db_version[0]} to version {version}") return version def load_path_recorder(): - with sqlite3.connect(db_file, timeout=timeout) as conn: - cursor = conn.cursor() + with transaction() as cursor: cursor.execute(''' SELECT path, depth, path_display FROM path_recorder @@ -484,8 +495,7 @@ def load_path_recorder(): return path_recorder def select_ranking(file): - with sqlite3.connect(db_file, timeout=timeout) as conn: - cursor = conn.cursor() + with transaction() as cursor: cursor.execute(''' SELECT ranking FROM ranking @@ -502,8 +512,7 @@ def select_ranking(file): def update_ranking(file, ranking): name = os.path.basename(file) - with sqlite3.connect(db_file, timeout=timeout) as conn: - cursor = conn.cursor() + with transaction() as cursor: if ranking == "None": cursor.execute(''' DELETE FROM ranking @@ -526,8 +535,7 @@ def update_ranking(file, ranking): return def update_path_recorder(path, depth, path_display): - with sqlite3.connect(db_file, timeout=timeout) as conn: - cursor = conn.cursor() + with transaction() as cursor: cursor.execute(''' INSERT OR REPLACE INTO path_recorder (path, depth, path_display) @@ -537,8 +545,7 @@ def update_path_recorder(path, depth, path_display): return def update_path_recorder(path, depth, path_display): - with sqlite3.connect(db_file, timeout=timeout) as conn: - cursor = conn.cursor() + with transaction() as cursor: cursor.execute(''' INSERT OR REPLACE INTO path_recorder (path, depth, path_display) @@ -548,8 +555,7 @@ def update_path_recorder(path, depth, path_display): return def delete_path_recorder(path): - with sqlite3.connect(db_file, timeout=timeout) as conn: - cursor = conn.cursor() + with transaction() as cursor: cursor.execute(''' DELETE FROM path_recorder WHERE path = ? @@ -664,19 +670,6 @@ def replace_ranking(cursor, file, alternate_file, hash): return -def transaction_begin(): - conn = sqlite3.connect(db_file, timeout=timeout) - conn.isolation_level = None - cursor = conn.cursor() - cursor.execute("BEGIN") - return conn, cursor - -def transaction_end(conn, cursor): - cursor.execute("COMMIT") - conn.close() - backup_tmp_db() - return - def update_exif_data_by_key(cursor, file, key, value): cursor.execute(''' INSERT OR REPLACE @@ -687,8 +680,7 @@ def update_exif_data_by_key(cursor, file, key, value): return def select_prompts(file): - with sqlite3.connect(db_file, timeout=timeout) as conn: - cursor = conn.cursor() + with transaction() as cursor: cursor.execute(''' SELECT key, value FROM exif_data @@ -709,8 +701,7 @@ def select_prompts(file): return prompt, neg_prompt def load_exif_data(exif_cache): - with sqlite3.connect(db_file, timeout=timeout) as conn: - cursor = conn.cursor() + with transaction() as cursor: cursor.execute(''' SELECT file, group_concat( case when key = 'prompt' or key = 'negative_prompt' then key || ': ' || value || '\n' @@ -735,8 +726,7 @@ def load_exif_data(exif_cache): return exif_cache def load_exif_data_by_key(cache, key1, key2): - with sqlite3.connect(db_file, timeout=timeout) as conn: - cursor = conn.cursor() + with transaction() as cursor: cursor.execute(''' SELECT file, value FROM exif_data @@ -750,8 +740,7 @@ def load_exif_data_by_key(cache, key1, key2): return cache def get_exif_dirs(): - with sqlite3.connect(db_file, timeout=timeout) as conn: - cursor = conn.cursor() + with transaction() as cursor: cursor.execute(''' SELECT file FROM exif_data From 7bb07e0343917e34baf327bd42adcfb473fdd441 Mon Sep 17 00:00:00 2001 From: Ivory Date: Tue, 15 Aug 2023 16:35:23 -0400 Subject: [PATCH 3/4] migrate entry module to contextual transactions --- scripts/image_browser.py | 364 +++++++++++++++++++-------------------- 1 file changed, 178 insertions(+), 186 deletions(-) diff --git a/scripts/image_browser.py b/scripts/image_browser.py index 0d75ae4..0c48f25 100644 --- a/scripts/image_browser.py +++ b/scripts/image_browser.py @@ -82,9 +82,9 @@ def check_image_browser_active_tabs(): last_default_tab = wib_db.get_last_default_tab() if last_default_tab[0] == "Others": # New tabs don't exist yet in image_browser_active_tabs, add them - conn, cursor = wib_db.transaction_begin() - wib_db.update_db_data(cursor, "last_default_tab", "Maintenance") - wib_db.transaction_end(conn, cursor) + with wib_db.transaction() as cursor: + wib_db.update_db_data(cursor, "last_default_tab", "Maintenance") + if hasattr(opts, "image_browser_active_tabs"): active_and_new_tabs = f"{opts.image_browser_active_tabs}, All, Maintenance" shared.opts.__setattr__("image_browser_active_tabs", active_and_new_tabs) @@ -457,67 +457,48 @@ def cache_exif(fileinfos): cache_exif_start = time.time() new_exif = 0 new_aes = 0 - conn, cursor = wib_db.transaction_begin() - for fi_info in fileinfos: - if any(fi_info[0].endswith(ext) for ext in image_ext_list): - found_exif = False - found_aes = False - if fi_info[0] in exif_cache: - found_exif = True - if fi_info[0] in aes_cache: - found_aes = True - if not found_exif or not found_aes: - exif_cache[fi_info[0]] = "0" - aes_cache[fi_info[0]] = "0" - try: - image = Image.open(fi_info[0]) - (_, allExif, allExif_html) = modules.extras.run_pnginfo(image) - image.close() - except SyntaxError: - allExif = False - logger.warning(f"Extension and content don't match: {fi_info[0]}") - except UnidentifiedImageError as e: - allExif = False - logger.warning(f"UnidentifiedImageError: {e}") - except Image.DecompressionBombError as e: - allExif = False - logger.warning(f"DecompressionBombError: {e}: {fi_info[0]}") - except PermissionError as e: - allExif = False - logger.warning(f"PermissionError: {e}: {fi_info[0]}") - except FileNotFoundError as e: - allExif = False - logger.warning(f"FileNotFoundError: {e}: {fi_info[0]}") - except OSError as e: - if e.errno == 22: - logger.warning(f"Caught OSError with error code 22: {fi_info[0]}") - else: - raise - if allExif: - exif_cache[fi_info[0]] = allExif - wib_db.update_exif_data(conn, fi_info[0], allExif) - new_exif = new_exif + 1 - - m = re.search("(?:aesthetic_score:|Score:) (\d+.\d+)", allExif, flags=re.IGNORECASE) - if m: - aes_value = m.group(1) - else: - aes_value = "0" - aes_cache[fi_info[0]] = aes_value - wib_db.update_exif_data_by_key(conn, fi_info[0], "aesthetic_score", aes_value) - new_aes = new_aes + 1 - else: + with wib_db.transaction() as cursor: + for fi_info in fileinfos: + if any(fi_info[0].endswith(ext) for ext in image_ext_list): + found_exif = False + found_aes = False + if fi_info[0] in exif_cache: + found_exif = True + if fi_info[0] in aes_cache: + found_aes = True + if not found_exif or not found_aes: + exif_cache[fi_info[0]] = "0" + aes_cache[fi_info[0]] = "0" try: - filename = os.path.splitext(fi_info[0])[0] + ".txt" - geninfo = "" - with open(filename) as f: - for line in f: - geninfo += line - exif_cache[fi_info[0]] = geninfo - wib_db.update_exif_data_by_key(conn, fi_info[0], geninfo) + image = Image.open(fi_info[0]) + (_, allExif, allExif_html) = modules.extras.run_pnginfo(image) + image.close() + except SyntaxError: + allExif = False + logger.warning(f"Extension and content don't match: {fi_info[0]}") + except UnidentifiedImageError as e: + allExif = False + logger.warning(f"UnidentifiedImageError: {e}") + except Image.DecompressionBombError as e: + allExif = False + logger.warning(f"DecompressionBombError: {e}: {fi_info[0]}") + except PermissionError as e: + allExif = False + logger.warning(f"PermissionError: {e}: {fi_info[0]}") + except FileNotFoundError as e: + allExif = False + logger.warning(f"FileNotFoundError: {e}: {fi_info[0]}") + except OSError as e: + if e.errno == 22: + logger.warning(f"Caught OSError with error code 22: {fi_info[0]}") + else: + raise + if allExif: + exif_cache[fi_info[0]] = allExif + wib_db.update_exif_data(conn, fi_info[0], allExif) new_exif = new_exif + 1 - m = re.search("(?:aesthetic_score:|Score:) (\d+.\d+)", geninfo, flags=re.IGNORECASE) + m = re.search("(?:aesthetic_score:|Score:) (\d+.\d+)", allExif, flags=re.IGNORECASE) if m: aes_value = m.group(1) else: @@ -525,19 +506,37 @@ def cache_exif(fileinfos): aes_cache[fi_info[0]] = aes_value wib_db.update_exif_data_by_key(conn, fi_info[0], "aesthetic_score", aes_value) new_aes = new_aes + 1 - except Exception: - logger.warning(f"cache_exif: No EXIF in image or txt file for {fi_info[0]}") - # Saved with defaults to not scan it again next time - exif_cache[fi_info[0]] = "0" - allExif = "0" - wib_db.update_exif_data(conn, fi_info[0], allExif) - new_exif = new_exif + 1 + else: + try: + filename = os.path.splitext(fi_info[0])[0] + ".txt" + geninfo = "" + with open(filename) as f: + for line in f: + geninfo += line + exif_cache[fi_info[0]] = geninfo + wib_db.update_exif_data_by_key(conn, fi_info[0], geninfo) + new_exif = new_exif + 1 - aes_value = "0" - aes_cache[fi_info[0]] = aes_value - wib_db.update_exif_data_by_key(conn, fi_info[0], "aesthetic_score", aes_value) - new_aes = new_aes + 1 - wib_db.transaction_end(conn, cursor) + m = re.search("(?:aesthetic_score:|Score:) (\d+.\d+)", geninfo, flags=re.IGNORECASE) + if m: + aes_value = m.group(1) + else: + aes_value = "0" + aes_cache[fi_info[0]] = aes_value + wib_db.update_exif_data_by_key(conn, fi_info[0], "aesthetic_score", aes_value) + new_aes = new_aes + 1 + except Exception: + logger.warning(f"cache_exif: No EXIF in image or txt file for {fi_info[0]}") + # Saved with defaults to not scan it again next time + exif_cache[fi_info[0]] = "0" + allExif = "0" + wib_db.update_exif_data(conn, fi_info[0], allExif) + new_exif = new_exif + 1 + + aes_value = "0" + aes_cache[fi_info[0]] = aes_value + wib_db.update_exif_data_by_key(conn, fi_info[0], "aesthetic_score", aes_value) + new_aes = new_aes + 1 if yappi_do: yappi.stop() @@ -573,9 +572,8 @@ def exif_rebuild(maint_wait): def exif_delete_0(maint_wait): global exif_cache, aes_cache if opts.image_browser_scan_exif: - conn, cursor = wib_db.transaction_begin() - wib_db.delete_exif_0(cursor) - wib_db.transaction_end(conn, cursor) + with wib_db.transaction() as cursor: + wib_db.delete_exif_0(cursor) exif_cache = wib_db.load_exif_data(exif_cache) aes_cache = wib_db.load_aes_data(aes_cache) maint_last_msg = "Delete finished" @@ -594,17 +592,16 @@ def exif_update_dirs(maint_update_dirs_path_recorder, maint_update_dirs_exif_dat maint_update_dirs_from = os.path.realpath(maint_update_dirs_from) maint_update_dirs_to = os.path.realpath(maint_update_dirs_to) rows = 0 - conn, cursor = wib_db.transaction_begin() - if maint_update_dirs_path_recorder: - wib_db.update_path_recorder_mult(cursor, maint_update_dirs_from, maint_update_dirs_to) - rows = rows + cursor.rowcount - if maint_update_dirs_exif_data: - wib_db.update_exif_data_mult(cursor, maint_update_dirs_from, maint_update_dirs_to) - rows = rows + cursor.rowcount - if maint_update_dirs_ranking: - wib_db.update_ranking_mult(cursor, maint_update_dirs_from, maint_update_dirs_to) - rows = rows + cursor.rowcount - wib_db.transaction_end(conn, cursor) + with wib_db.transaction() as cursor: + if maint_update_dirs_path_recorder: + wib_db.update_path_recorder_mult(cursor, maint_update_dirs_from, maint_update_dirs_to) + rows = rows + cursor.rowcount + if maint_update_dirs_exif_data: + wib_db.update_exif_data_mult(cursor, maint_update_dirs_from, maint_update_dirs_to) + rows = rows + cursor.rowcount + if maint_update_dirs_ranking: + wib_db.update_ranking_mult(cursor, maint_update_dirs_from, maint_update_dirs_to) + rows = rows + cursor.rowcount if rows == 0: maint_last_msg = "No rows updated" else: @@ -614,9 +611,8 @@ def exif_update_dirs(maint_update_dirs_path_recorder, maint_update_dirs_exif_dat def recreate_hash(maint_wait): version = str(db_version) - conn, cursor = wib_db.transaction_begin() - wib_db.migrate_filehash(cursor, version) - wib_db.transaction_end(conn, cursor) + with wib_db.transaction() as cursor: + wib_db.migrate_filehash(cursor, version) maint_last_msg = "Hashes recreated" return maint_wait, maint_last_msg @@ -632,33 +628,31 @@ def reapply_ranking(path_recorder, maint_wait): if os.path.exists(key): dirs[key] = key - conn, cursor = wib_db.transaction_begin() + with wib_db.transaction() as cursor: + # Traverse all known dirs, check if missing rankings are due to moved files + for key in dirs.keys(): + fileinfos = traverse_all_files(key, [], "", 0) + for (file, _) in fileinfos: + # Is there a ranking for this full filepath + ranking_by_file = wib_db.get_ranking_by_file(cursor, file) + if ranking_by_file is None: + name = os.path.basename(file) + (ranking_by_name, alternate_hash) = wib_db.get_ranking_by_name(cursor, name) + # Is there a ranking only for the filename + if ranking_by_name is not None: + hash = wib_db.get_hash(file) + (alternate_file, alternate_ranking) = ranking_by_name + if alternate_ranking is not None and alternate_hash is not None: + (alternate_hash,) = alternate_hash + # Does the found filename's file have no hash or the same hash? + if alternate_hash is None or hash == alternate_hash: + if os.path.exists(alternate_file): + # Insert ranking as a copy of the found filename's ranking + wib_db.insert_ranking(cursor, file, alternate_ranking, hash) + else: + # Replace ranking of the found filename + wib_db.replace_ranking(cursor, file, alternate_file, hash) - # Traverse all known dirs, check if missing rankings are due to moved files - for key in dirs.keys(): - fileinfos = traverse_all_files(key, [], "", 0) - for (file, _) in fileinfos: - # Is there a ranking for this full filepath - ranking_by_file = wib_db.get_ranking_by_file(cursor, file) - if ranking_by_file is None: - name = os.path.basename(file) - (ranking_by_name, alternate_hash) = wib_db.get_ranking_by_name(cursor, name) - # Is there a ranking only for the filename - if ranking_by_name is not None: - hash = wib_db.get_hash(file) - (alternate_file, alternate_ranking) = ranking_by_name - if alternate_ranking is not None and alternate_hash is not None: - (alternate_hash,) = alternate_hash - # Does the found filename's file have no hash or the same hash? - if alternate_hash is None or hash == alternate_hash: - if os.path.exists(alternate_file): - # Insert ranking as a copy of the found filename's ranking - wib_db.insert_ranking(cursor, file, alternate_ranking, hash) - else: - # Replace ranking of the found filename - wib_db.replace_ranking(cursor, file, alternate_file, hash) - - wib_db.transaction_end(conn, cursor) maint_last_msg = "Rankings reapplied" return maint_wait, maint_last_msg @@ -743,73 +737,71 @@ def get_all_images(dir_name, sort_by, sort_order, keyword, tab_base_tag_box, img filenames = [finfo[0] for finfo in fileinfos] if opts.image_browser_scan_exif: - conn, cursor = wib_db.transaction_begin() - if len(exif_keyword) != 0: - if use_regex: - regex_error = False + with wib_db.transaction() as cursor: + if len(exif_keyword) != 0: + if use_regex: + regex_error = False + try: + test_re = re.compile(exif_keyword, re.DOTALL) + except re.error as e: + regex_error = True + print(f"Regex error: {e}") + if (use_regex and not regex_error) or not use_regex: + if negative_prompt_search == "Yes": + fileinfos = [x for x in fileinfos if exif_search(exif_keyword, exif_cache[x[0]], use_regex, case_sensitive)] + else: + result = [] + for file_info in fileinfos: + file_name = file_info[0] + file_exif = exif_cache[file_name] + file_exif_lc = file_exif.lower() + start_index = file_exif_lc.find(np) + end_index = file_exif.find("\n", start_index) + if negative_prompt_search == "Only": + start_index = start_index + len(np) + sub_string = file_exif[start_index:end_index].strip() + if exif_search(exif_keyword, sub_string, use_regex, case_sensitive): + result.append(file_info) + else: + sub_string = file_exif[start_index:end_index].strip() + file_exif = file_exif.replace(sub_string, "") + + if exif_search(exif_keyword, file_exif, use_regex, case_sensitive): + result.append(file_info) + fileinfos = result + filenames = [finfo[0] for finfo in fileinfos] + wib_db.fill_work_files(cursor, fileinfos) + if len(aes_filter_min) != 0 or len(aes_filter_max) != 0: try: - test_re = re.compile(exif_keyword, re.DOTALL) - except re.error as e: - regex_error = True - print(f"Regex error: {e}") - if (use_regex and not regex_error) or not use_regex: - if negative_prompt_search == "Yes": - fileinfos = [x for x in fileinfos if exif_search(exif_keyword, exif_cache[x[0]], use_regex, case_sensitive)] - else: - result = [] - for file_info in fileinfos: - file_name = file_info[0] - file_exif = exif_cache[file_name] - file_exif_lc = file_exif.lower() - start_index = file_exif_lc.find(np) - end_index = file_exif.find("\n", start_index) - if negative_prompt_search == "Only": - start_index = start_index + len(np) - sub_string = file_exif[start_index:end_index].strip() - if exif_search(exif_keyword, sub_string, use_regex, case_sensitive): - result.append(file_info) - else: - sub_string = file_exif[start_index:end_index].strip() - file_exif = file_exif.replace(sub_string, "") - - if exif_search(exif_keyword, file_exif, use_regex, case_sensitive): - result.append(file_info) - fileinfos = result + aes_filter_min_num = float(aes_filter_min) + except ValueError: + aes_filter_min_num = sys.float_info.min + try: + aes_filter_max_num = float(aes_filter_max) + except ValueError: + aes_filter_max_num = sys.float_info.max + + fileinfos = wib_db.filter_aes(cursor, fileinfos, aes_filter_min_num, aes_filter_max_num) + filenames = [finfo[0] for finfo in fileinfos] + if ranking_filter != "All": + ranking_filter_min_num = 1 + ranking_filter_max_num = 5 + if ranking_filter == "Min-max": + try: + ranking_filter_min_num = int(ranking_filter_min) + except ValueError: + ranking_filter_min_num = 0 + try: + ranking_filter_max_num = int(ranking_filter_max) + except ValueError: + ranking_filter_max_num = 0 + if ranking_filter_min_num < 1: + ranking_filter_min_num = 1 + if ranking_filter_max_num < 1 or ranking_filter_max_num > 5: + ranking_filter_max_num = 5 + + fileinfos = wib_db.filter_ranking(cursor, fileinfos, ranking_filter, ranking_filter_min_num, ranking_filter_max_num) filenames = [finfo[0] for finfo in fileinfos] - wib_db.fill_work_files(cursor, fileinfos) - if len(aes_filter_min) != 0 or len(aes_filter_max) != 0: - try: - aes_filter_min_num = float(aes_filter_min) - except ValueError: - aes_filter_min_num = sys.float_info.min - try: - aes_filter_max_num = float(aes_filter_max) - except ValueError: - aes_filter_max_num = sys.float_info.max - - fileinfos = wib_db.filter_aes(cursor, fileinfos, aes_filter_min_num, aes_filter_max_num) - filenames = [finfo[0] for finfo in fileinfos] - if ranking_filter != "All": - ranking_filter_min_num = 1 - ranking_filter_max_num = 5 - if ranking_filter == "Min-max": - try: - ranking_filter_min_num = int(ranking_filter_min) - except ValueError: - ranking_filter_min_num = 0 - try: - ranking_filter_max_num = int(ranking_filter_max) - except ValueError: - ranking_filter_max_num = 0 - if ranking_filter_min_num < 1: - ranking_filter_min_num = 1 - if ranking_filter_max_num < 1 or ranking_filter_max_num > 5: - ranking_filter_max_num = 5 - - fileinfos = wib_db.filter_ranking(cursor, fileinfos, ranking_filter, ranking_filter_min_num, ranking_filter_max_num) - filenames = [finfo[0] for finfo in fileinfos] - - wib_db.transaction_end(conn, cursor) if sort_by == "date": if sort_order == up_symbol: @@ -905,11 +897,11 @@ def get_image_thumbnail(image_list): def set_tooltip_info(image_list): image_browser_img_info = {} - conn, cursor = wib_db.transaction_begin() - for filename in image_list: - x, y = wib_db.select_x_y(cursor, filename) - image_browser_img_info[filename] = {"x": x, "y": y} - wib_db.transaction_end(conn, cursor) + with wib_db.transaction() as cursor: + for filename in image_list: + x, y = wib_db.select_x_y(cursor, filename) + image_browser_img_info[filename] = {"x": x, "y": y} + image_browser_img_info_json = json.dumps(image_browser_img_info) return image_browser_img_info_json From 51df4c1e5df0cd879bf6369cfbe617071ec42d82 Mon Sep 17 00:00:00 2001 From: AlUlkesh <99896447+AlUlkesh@users.noreply.github.com> Date: Wed, 16 Aug 2023 07:27:23 +0200 Subject: [PATCH 4/4] Remaining "conn" references changed to "cursor" --- scripts/image_browser.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/image_browser.py b/scripts/image_browser.py index 0c48f25..f501b16 100644 --- a/scripts/image_browser.py +++ b/scripts/image_browser.py @@ -495,7 +495,7 @@ def cache_exif(fileinfos): raise if allExif: exif_cache[fi_info[0]] = allExif - wib_db.update_exif_data(conn, fi_info[0], allExif) + wib_db.update_exif_data(cursor, fi_info[0], allExif) new_exif = new_exif + 1 m = re.search("(?:aesthetic_score:|Score:) (\d+.\d+)", allExif, flags=re.IGNORECASE) @@ -504,7 +504,7 @@ def cache_exif(fileinfos): else: aes_value = "0" aes_cache[fi_info[0]] = aes_value - wib_db.update_exif_data_by_key(conn, fi_info[0], "aesthetic_score", aes_value) + wib_db.update_exif_data_by_key(cursor, fi_info[0], "aesthetic_score", aes_value) new_aes = new_aes + 1 else: try: @@ -514,7 +514,7 @@ def cache_exif(fileinfos): for line in f: geninfo += line exif_cache[fi_info[0]] = geninfo - wib_db.update_exif_data_by_key(conn, fi_info[0], geninfo) + wib_db.update_exif_data_by_key(cursor, fi_info[0], geninfo) new_exif = new_exif + 1 m = re.search("(?:aesthetic_score:|Score:) (\d+.\d+)", geninfo, flags=re.IGNORECASE) @@ -523,19 +523,19 @@ def cache_exif(fileinfos): else: aes_value = "0" aes_cache[fi_info[0]] = aes_value - wib_db.update_exif_data_by_key(conn, fi_info[0], "aesthetic_score", aes_value) + wib_db.update_exif_data_by_key(cursor, fi_info[0], "aesthetic_score", aes_value) new_aes = new_aes + 1 except Exception: logger.warning(f"cache_exif: No EXIF in image or txt file for {fi_info[0]}") # Saved with defaults to not scan it again next time exif_cache[fi_info[0]] = "0" allExif = "0" - wib_db.update_exif_data(conn, fi_info[0], allExif) + wib_db.update_exif_data(cursor, fi_info[0], allExif) new_exif = new_exif + 1 aes_value = "0" aes_cache[fi_info[0]] = aes_value - wib_db.update_exif_data_by_key(conn, fi_info[0], "aesthetic_score", aes_value) + wib_db.update_exif_data_by_key(cursor, fi_info[0], "aesthetic_score", aes_value) new_aes = new_aes + 1 if yappi_do: