change default hfcache folder

Signed-off-by: Vladimir Mandic <mandic00@live.com>

CHANGELOG.md

@@ -10,6 +10,9 @@ Plus continuing with major **UI** work, we have new embedded **Docs/Wiki** searc
 On the compute side, new profiles for high-vram GPUs, offloading improvements, support for new `torch` release and improved quality when using low-bit quantization!      
 And (*as always*) many bugfixes and improvements to existing features!  
 
+*Note*: Change-in-behavior - locations of downloaded HuggingFace models and components are changed to allow for de-duplication of common modules and switched from using system default cache folder to `models/huggingface`  
+SD.Next will warn on startup on unused cache entries that can be removed. Also, to take advantage of de-duplication, you'll need to delete models from your `models/Diffusers` folder and let SD.Next re-download them!  
+
 [ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867)
 
 ### Details for 2025-08-13
@@ -109,6 +112,8 @@ And (*as always*) many bugfixes and improvements to existing features!
   - add `/sdapi/v1/checkpoint` POST endpoint to simply load a model  
   - add `/sdapi/v1/modules` GET endpoint to get info on model components/modules  
 - **Refactor**
+  - change default huggingface cache folder from system default to `models/huggingface`  
+    sd.next will warn on startup on unused cache entries  
   - new unified pipeline component loader in `pipelines/generic`  
   - remove **LDSR**  
   - remove `api-only` cli option  

launch.py

@@ -45,9 +45,9 @@ def init_args():
 def init_paths():
     global script_path, extensions_dir # pylint: disable=global-statement
     import modules.paths
-    modules.paths.register_paths()
     script_path = modules.paths.script_path
     extensions_dir = modules.paths.extensions_dir
+    sys.path.insert(0, script_path)
     rec('paths')
 
 

modules/modelstats.py

@@ -15,15 +15,16 @@ def walk(folder: str):
 def stat(fn: str):
     if fn is None or len(fn) == 0 or not os.path.exists(fn):
         return 0, None
-    fs_stat = os.stat(fn)
+    fs_stat = os.stat(fn, follow_symlinks=False)
     mtime = datetime.fromtimestamp(fs_stat.st_mtime).replace(microsecond=0)
-    if os.path.isfile(fn):
+    if os.path.islink(fn):
+        size = 0
+    elif os.path.isfile(fn):
         size = round(fs_stat.st_size)
     elif os.path.isdir(fn):
         size = round(sum(stat(fn)[0] for fn in walk(fn)))
     else:
         size = 0
-    print('HERE', fn, os.path.isfile(fn), os.path.isdir(fn), size, mtime)
     return size, mtime
 
 

modules/paths.py

@@ -44,23 +44,6 @@ if os.environ.get('SD_PATH_DEBUG', None) is not None:
     log.debug(f'Paths: script-path="{script_path}" data-dir="{data_path}" models-dir="{models_path}" config="{config_path}"')
 
 
-def register_paths():
-    log.debug('Register paths')
-    sys.path.insert(0, script_path)
-    # sd_path = os.path.join(script_path, 'repositories')
-    path_dirs = [
-        # (os.path.join(sd_path, 'codeformer'), 'inference_codeformer.py', 'CodeFormer', []),
-    ]
-    for d, must_exist, what, _options in path_dirs:
-        must_exist_path = os.path.abspath(os.path.join(script_path, d, must_exist))
-        if not os.path.exists(must_exist_path):
-            log.error(f'Required path not found: path={must_exist_path} item={what}')
-        else:
-            d = os.path.abspath(d)
-            sys.path.append(d)
-            paths[what] = d
-
-
 def create_path(folder):
     if folder is None or folder == '':
         return
@@ -103,6 +86,7 @@ def create_paths(opts):
     create_path(fix_path('temp_dir'))
     create_path(fix_path('ckpt_dir'))
     create_path(fix_path('diffusers_dir'))
+    create_path(fix_path('hfcache_dir'))
     create_path(fix_path('vae_dir'))
     create_path(fix_path('unet_dir'))
     create_path(fix_path('te_dir'))
@@ -139,3 +123,14 @@ class Prioritize:
     def __exit__(self, exc_type, exc_val, exc_tb):
         sys.path = self.path
         self.path = None
+
+
+def check_cache(opts):
+    prev_default = os.environ.get("SD_HFCACHEDIR", None) or os.path.join(os.path.expanduser('~'), '.cache', 'huggingface', 'hub')
+    from modules.modelstats import stat
+    if opts.hfcache_dir != prev_default:
+        size, _mtime = stat(prev_default)
+        if (size//1024//1024 > 0):
+            log.warning(f'Cache location changed: previous="{prev_default}" size={size//1024//1024} MB')
+    size, _mtime = stat(opts.hfcache_dir)
+    log.debug(f'Huggingface cache: path="{opts.hfcache_dir}" size={size//1024//1024} MB')

modules/paths_internal.py

@@ -1,30 +1,3 @@
 # no longer used, all paths are defined in paths.py
 
 from modules.paths import modules_path, script_path, sd_configs_path, sd_default_config, sd_model_file, default_sd_model_file, data_path, models_path, extensions_dir, extensions_builtin_dir # pylint: disable=unused-import
-
-"""
-import argparse
-import os
-
-modules_path = os.path.dirname(os.path.realpath(__file__))
-script_path = os.path.dirname(modules_path)
-sd_configs_path = os.path.join(script_path, "configs")
-sd_default_config = os.path.join(sd_configs_path, "v1-inference.yaml")
-
-# Parse the --data-dir flag first so we can use it as a base for our other argument default values
-parser_pre = argparse.ArgumentParser(add_help=False)
-parser_pre.add_argument("--ckpt", type=str, default=os.environ.get("SD_MODEL", None), help="Path to model checkpoint to load immediately, default: %(default)s")
-parser_pre.add_argument("--data-dir", type=str, default=os.environ.get("SD_DATADIR", ''), help="Base path where all user data is stored, default: %(default)s")
-parser_pre.add_argument("--models-dir", type=str, default=os.environ.get("SD_MODELSDIR", 'models'), help="Base path where all models are stored, default: %(default)s",)
-cmd_opts_pre = parser_pre.parse_known_args()[0]
-
-# parser_pre.add_argument("--config", type=str, default=os.environ.get("SD_CONFIG", os.path.join(data_path, 'config.json')), help="Use specific server configuration file, default: %(default)s")
-
-data_path = cmd_opts_pre.data_dir
-models_path = cmd_opts_pre.models_dir if os.path.isabs(cmd_opts_pre.models_dir) else os.path.join(data_path, cmd_opts_pre.models_dir)
-extensions_dir = os.path.join(data_path, "extensions")
-extensions_builtin_dir = "extensions-builtin"
-
-sd_model_file = cmd_opts_pre.ckpt or os.path.join(script_path, 'model.ckpt') # not used
-default_sd_model_file = sd_model_file # not used
-"""

modules/shared.py

@@ -24,6 +24,11 @@ import modules.paths as paths
 from installer import log, print_dict, console, get_version # pylint: disable=unused-import
 
 
+class Backend(Enum):
+    ORIGINAL = 1
+    DIFFUSERS = 2
+
+
 errors.install([gr])
 demo: gr.Blocks = None
 api = None
@@ -58,16 +63,11 @@ restricted_opts = {
 }
 resize_modes = ["None", "Fixed", "Crop", "Fill", "Outpaint", "Context aware"]
 max_workers = 12
-default_hfcache_dir = os.environ.get("SD_HFCACHEDIR", None) or os.path.join(os.path.expanduser('~'), '.cache', 'huggingface', 'hub')
 sdnq_quant_modes = ["int8", "float8_e4m3fn", "int7", "int6", "int5", "uint4", "uint3", "uint2", "float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz", "uint8", "uint7", "uint6", "uint5", "int4", "int3", "int2", "uint1"]
+default_hfcache_dir = os.environ.get("SD_HFCACHEDIR", None) or os.path.join(paths.models_path, 'huggingface')
 state = shared_state.State()
 
 
-class Backend(Enum):
-    ORIGINAL = 1
-    DIFFUSERS = 2
-
-
 # early select backend
 backend = Backend.DIFFUSERS
 if not hasattr(cmd_opts, "use_openvino"):

webui.py

@@ -68,7 +68,7 @@ def initialize():
     modules.sd_checkpoint.init_metadata()
     modules.hashes.init_cache()
 
-    log.debug(f'Huggingface cache: path="{shared.opts.hfcache_dir}"')
+    paths.check_cache(shared.opts)
 
     modules.sd_samplers.list_samplers()
     timer.startup.record("samplers")