relocate all jsons to data

Signed-off-by: vladmandic <mandic00@live.com>

.gitconfig

@@ -1,20 +0,0 @@
-[pull]
-  rebase = true
-[https]
-  postBuffer = 100000000
-[filter "lfs"]
-  clean = git-lfs clean -- %f
-  smudge = git-lfs smudge -- %f
-  process = git-lfs filter-process
-  required = true
-[init]
-  defaultBranch = master
-[color]
-  ui = auto
-[alias]
-  lg = log --color --abbrev-commit --graph --pretty=format:'%C(bold blue)%h%C(reset) %C(blue)%an%C(reset) %C(yellow)%ci %cr%C(reset) %C(green)%d%C(reset) %s'
-[core]
-  editor = code --wait
-  whitespace = trailing-space,space-before-tab,indent-with-non-tab,-tab-in-indent,cr-at-eol
-  autocrlf = input
-  eol = lf

.gitignore

@@ -65,6 +65,7 @@ tunableop_results*.csv
 .*/
 
 # force included
+!/data
 !/models/VAE-approx
 !/models/VAE-approx/model.pt
 !/models/Reference

CHANGELOG.md

@@ -20,6 +20,8 @@
     **xpu**: update to `torch==2.10.0`  
     **openvino**: update to `torch==2.10.0` and `openvino==2025.4.1`  
     **rocm/linux**: update to `torch==2.10.0`  
+  - relocate all json data files to `data/` folder  
+    existing data files are auto-migrated on startup  
   - further work on type consistency and type checking, thanks @awsr  
   - add ui placeholders for future agent-scheduler work, thanks @ryanmeador  
   - update package requirements  

data/cache.json

@@ -0,0 +1,21 @@
+{
+  "hashes": {
+    "checkpoint/tempestByVlad_baseV01": {
+      "mtime": 1763424610.0,
+      "sha256": "8bfad1722243955b3f94103c69079c280d348b14729251e86824972c1063b616"
+    },
+    "checkpoint/lyriel_v16": {
+      "mtime": 1763424496.0,
+      "sha256": "ec6f68ea6388951d53d6c8178c22aecfd1b4fedfe31f5a5814ddd7638c4eff37"
+    },
+    "checkpoint/v1-5-pruned-fp16-emaonly": {
+      "mtime": 1768913681.028832,
+      "sha256": "92954befdb6aacf52f86095eba54ac9262459bc21f987c0e51350f3679c4e45a"
+    },
+    "checkpoint/juggernautXL_juggXIByRundiffusion": {
+      "mtime": 1768913991.4270966,
+      "sha256": "33e58e86686f6b386c526682b5da9228ead4f91d994abd4b053442dc5b42719e"
+    }
+  },
+  "hashes-addnet": {}
+}

javascript/ui.js

@@ -574,7 +574,7 @@ function toggleCompact(val, old) {
 
 function previewTheme() {
   let name = gradioApp().getElementById('setting_gradio_theme').querySelectorAll('input')?.[0].value || '';
-  fetch(`${window.subpath}/file=html/themes.json`)
+  fetch(`${window.subpath}/file=data/themes.json`)
     .then((res) => {
       res.json()
         .then((themes) => {

modules/hashes.py

@@ -6,7 +6,7 @@ from modules.json_helpers import readfile, writefile
 from modules.paths import data_path
 
 
-cache_filename = os.path.join(data_path, "cache.json")
+cache_filename = os.path.join(data_path, 'data', 'cache.json')
 cache_data = None
 progress_ok = True
 

modules/json_helpers.py

@@ -47,10 +47,10 @@ def readfile(filename: str, silent: bool = False, lock: bool = False, *, as_type
             log.debug(f'Read: file="{filename}" json={len(data)} bytes={os.path.getsize(filename)} time={t1-t0:.3f} fn={fn}')
     except FileNotFoundError as err:
         if not silent:
-            log.debug(f'Reading failed: {filename} {err}')
+            log.debug(f'Read failed: file="{filename}" {err}')
     except Exception as err:
         if not silent:
-            log.error(f'Reading failed: {filename} {err}')
+            log.error(f'Read failed: file="{filename}" {err}')
     try:
         if locking_available and lock_file is not None:
             lock_file.release_read_lock()

modules/migrate.py

@@ -0,0 +1,36 @@
+import os
+from modules.paths import data_path
+from installer import log
+
+
+files = [
+    'cache.json',
+    'metadata.json',
+    'html/extensions.json',
+    'html/previews.json',
+    'html/upscalers.json',
+    'html/reference.json',
+    'html/themes.json',
+    'html/reference-quant.json',
+    'html/reference-distilled.json',
+    'html/reference-community.json',
+    'html/reference-cloud.json',
+]
+
+
+def migrate_data():
+    for f in files:
+        old_filename = os.path.join(data_path, f)
+        new_filename = os.path.join(data_path, "data", os.path.basename(f))
+        if os.path.exists(old_filename):
+            if not os.path.exists(new_filename):
+                log.info(f'Migrating: file="{old_filename}" target="{new_filename}"')
+                try:
+                    os.rename(old_filename, new_filename)
+                except Exception as e:
+                    log.error(f'Migrating: file="{old_filename}" target="{new_filename}" {e}')
+            else:
+                log.warning(f'Migrating: file="{old_filename}" target="{new_filename}" skip existing')
+
+
+migrate_data()

modules/sd_checkpoint.py

@@ -14,7 +14,7 @@ checkpoint_aliases = {}
 checkpoints_loaded = collections.OrderedDict()
 model_dir = "Stable-diffusion"
 model_path = os.path.abspath(os.path.join(paths.models_path, model_dir))
-sd_metadata_file = os.path.join(paths.data_path, "metadata.json")
+sd_metadata_file = os.path.join(paths.data_path, "data", "metadata.json")
 sd_metadata = None
 sd_metadata_pending = 0
 sd_metadata_timer = 0

modules/sd_models.py

@@ -14,14 +14,13 @@ from installer import log
 from modules import timer, paths, shared, shared_items, modelloader, devices, script_callbacks, sd_vae, sd_unet, errors, sd_models_compile, sd_detect, model_quant, sd_hijack_te, sd_hijack_accelerate, sd_hijack_safetensors, attention
 from modules.memstats import memory_stats
 from modules.modeldata import model_data
-from modules.sd_checkpoint import CheckpointInfo, select_checkpoint, list_models, checkpoints_list, checkpoint_titles, get_closest_checkpoint_match, model_hash, update_model_hashes, setup_model, write_metadata, read_metadata_from_safetensors # pylint: disable=unused-import
+from modules.sd_checkpoint import CheckpointInfo, select_checkpoint, list_models, sd_metadata_file, checkpoints_list, checkpoint_titles, get_closest_checkpoint_match, model_hash, update_model_hashes, setup_model, write_metadata, read_metadata_from_safetensors # pylint: disable=unused-import
 from modules.sd_offload import get_module_names, disable_offload, set_diffuser_offload, apply_balanced_offload, set_accelerate # pylint: disable=unused-import
 from modules.sd_models_utils import NoWatermark, get_signature, get_call, path_to_repo, patch_diffuser_config, convert_to_faketensors, read_state_dict, get_state_dict_from_checkpoint, apply_function_to_model # pylint: disable=unused-import
 
 
 model_dir = "Stable-diffusion"
 model_path = os.path.abspath(os.path.join(paths.models_path, model_dir))
-sd_metadata_file = os.path.join(paths.data_path, "metadata.json")
 sd_metadata = None
 sd_metadata_pending = 0
 sd_metadata_timer = 0

modules/shared.py

@@ -849,7 +849,7 @@ log.info(f'Engine: backend={backend} compute={devices.backend} device={devices.g
 profiler = None
 import modules.styles
 prompt_styles = modules.styles.StyleDatabase(opts)
-reference_models = readfile(os.path.join('html', 'reference.json'), as_type="dict") if opts.extra_network_reference_enable else {}
+reference_models = readfile(os.path.join('data', 'reference.json'), as_type="dict") if opts.extra_network_reference_enable else {}
 cmd_opts.disable_extension_access = (cmd_opts.share or cmd_opts.listen or (cmd_opts.server_name or False)) and not cmd_opts.insecure
 
 log.debug('Initializing: devices')

modules/theme.py

@@ -14,11 +14,11 @@ def list_builtin_themes():
 
 
 def refresh_themes(no_update=False):
-    fn = os.path.join('html', 'themes.json')
+    themes_file = os.path.join('data', 'themes.json')
     res = []
-    if os.path.exists(fn):
+    if os.path.exists(themes_file):
         try:
-            with open(fn, 'r', encoding='utf8') as f:
+            with open(themes_file, 'r', encoding='utf8') as f:
                 res = json.load(f)
         except Exception:
             modules.shared.log.error('Exception loading UI themes')
@@ -28,7 +28,7 @@ def refresh_themes(no_update=False):
             r = modules.shared.req('https://huggingface.co/datasets/freddyaboulton/gradio-theme-subdomains/resolve/main/subdomains.json')
             if r.status_code == 200:
                 res = r.json()
-                modules.shared.writefile(res, fn)
+                modules.shared.writefile(res, themes_file)
             else:
                 modules.shared.log.error('Error refreshing UI themes')
         except Exception:

modules/ui_extensions.py

@@ -28,7 +28,7 @@ sort_ordering = {
     "commits": (True, lambda x: x.get('commits', 0)),
     "issues": (True, lambda x: x.get('issues', 0)),
 }
-
+extensions_data_file = os.path.join("data", "extensions.json")
 
 re_snake_case = re.compile(r'_(?=[a-zA-z0-9])')
 re_camelCase = re.compile(r'(?<=[a-z])([A-Z])')
@@ -41,10 +41,9 @@ def get_installed(ext):
 
 def list_extensions():
     global extensions_list # pylint: disable=global-statement
-    fn = os.path.join(paths.script_path, "html", "extensions.json")
-    extensions_list = shared.readfile(fn, silent=True, as_type="list")
+    extensions_list = shared.readfile(extensions_data_file, silent=True, as_type="list")
     if len(extensions_list) == 0:
-        shared.log.info("Extension List: No information found. Refresh required.")
+        shared.log.info("Extension list: No information found. Refresh required.")
     found = []
     for ext in extensions.extensions:
         ext.read_info()
@@ -260,7 +259,7 @@ def refresh_extensions_list(search_text, sort_column):
         with urllib.request.urlopen(extensions_index, timeout=3.0, context=context) as response:
             text = response.read()
         extensions_list = json.loads(text)
-        with open(os.path.join(paths.script_path, "html", "extensions.json"), "w", encoding="utf-8") as outfile:
+        with open(extensions_data_file, "w", encoding="utf-8") as outfile:
             json_object = json.dumps(extensions_list, indent=2)
             outfile.write(json_object)
             shared.log.info(f'Updated extensions list: items={len(extensions_list)} url={extensions_index}')

modules/ui_extra_networks.py

@@ -453,7 +453,8 @@ class ExtraNetworksPage:
     def update_all_previews(self, items):
         global preview_map # pylint: disable=global-statement
         if preview_map is None:
-            preview_map = shared.readfile('html/previews.json', silent=True, as_type="dict")
+            preview_file = os.path.join('data', 'previews.json')
+            preview_map = shared.readfile(preview_file, silent=True, as_type="dict")
         t0 = time.time()
         reference_path = os.path.abspath(os.path.join('models', 'Reference'))
         possible_paths = list(set([os.path.dirname(item['filename']) for item in items] + [reference_path]))

modules/ui_extra_networks_checkpoints.py

@@ -43,11 +43,11 @@ class ExtraNetworksPageCheckpoints(ui_extra_networks.ExtraNetworksPage):
             return []
         count = { 'total': 0, 'ready': 0, 'hidden': 0, 'experimental': 0, 'base': 0 }
 
-        reference_base = readfile(os.path.join('html', 'reference.json'), as_type="dict")
-        reference_quant = readfile(os.path.join('html', 'reference-quant.json'), as_type="dict")
-        reference_distilled = readfile(os.path.join('html', 'reference-distilled.json'), as_type="dict")
-        reference_community = readfile(os.path.join('html', 'reference-community.json'), as_type="dict")
-        reference_cloud = readfile(os.path.join('html', 'reference-cloud.json'), as_type="dict")
+        reference_base = readfile(os.path.join('data', 'reference.json'), as_type="dict")
+        reference_quant = readfile(os.path.join('data', 'reference-quant.json'), as_type="dict")
+        reference_distilled = readfile(os.path.join('data', 'reference-distilled.json'), as_type="dict")
+        reference_community = readfile(os.path.join('data', 'reference-community.json'), as_type="dict")
+        reference_cloud = readfile(os.path.join('data', 'reference-cloud.json'), as_type="dict")
         shared.reference_models = {}
         shared.reference_models.update(reference_base)
         shared.reference_models.update(reference_quant)

modules/upscaler.py

@@ -23,7 +23,8 @@ class Upscaler:
     def __init__(self, create_dirs=True):
         global models # pylint: disable=global-statement
         if models is None:
-            models = shared.readfile('html/upscalers.json', as_type="dict")
+            models_file = os.path.join('data', 'upscalers.json')
+            models = shared.readfile(models_file, as_type="dict")
         self.mod_pad_h = None
         self.tile_size = shared.opts.upscaler_tile_size
         self.tile_pad = shared.opts.upscaler_tile_overlap

webui.py

@@ -15,6 +15,7 @@ import modules.loader
 import modules.hashes
 import modules.paths
 import modules.devices
+import modules.migrate
 from modules import shared
 from modules.call_queue import queue_lock, wrap_queued_call, wrap_gradio_gpu_call # pylint: disable=unused-import
 import modules.gr_tempdir