update docker build

Signed-off-by: Vladimir Mandic <mandic00@live.com>

.gitignore

@@ -44,7 +44,6 @@ tunableop_results*.csv
 !webui.sh
 !package.json
 !requirements.txt
-!requirements-extra.txt
 
 # pyinstaller
 *.spec

Dockerfile

@@ -6,7 +6,9 @@ WORKDIR /
 COPY . .
 # stop pip and uv from caching
 ENV PIP_NO_CACHE_DIR=true
+ENV PIP_ROOT_USER_ACTION=ignore
 ENV UV_NO_CACHE=true
+ENV SD_INSTALL_DEBUG=true
 # disable model hashing for faster startup
 ENV SD_NOHASHING=true
 # set data directories
@@ -14,11 +16,10 @@ ENV SD_DATADIR="/mnt/data"
 ENV SD_MODELSDIR="/mnt/models"
 # install dependencies
 RUN ["apt-get", "-y", "update"]
-RUN ["apt-get", "-y", "install", "git"]
+RUN ["apt-get", "-y", "install", "git", "build-essential"]
 # sdnext will run all necessary pip install ops and then exit
-RUN ["python", "launch.py", "--debug", "--uv", "--use-cuda", "--log", "sdnext.log", "--test"]
+RUN ["python", "launch.py", "--debug", "--uv", "--use-cuda", "--log", "sdnext.log", "--test", "--optional"]
 # preinstall additional packages to avoid installation during runtime
-RUN ["uv", "pip", "install", "-r", "requirements-extra.txt", "--system"]
 # actually run sdnext
 CMD ["python", "launch.py", "--debug", "--skip-all", "--listen", "--quick", "--api-log", "--log", "sdnext.log"]
 # expose port

installer.py

@@ -27,7 +27,7 @@ log_file = os.path.join(os.path.dirname(__file__), 'sdnext.log')
 log_rolled = False
 first_call = True
 quick_allowed = True
-errors = 0
+errors = []
 opts = {}
 args = Dot({
     'debug': False,
@@ -280,8 +280,7 @@ def pip(arg: str, ignore: bool = False, quiet: bool = False, uv = True):
     txt = txt.strip()
     debug(f'Install {pipCmd}: {txt}')
     if result.returncode != 0 and not ignore:
-        global errors # pylint: disable=global-statement
-        errors += 1
+        errors.append(f'pip: {package}')
         log.error(f'Install: {pipCmd}: {arg}')
         log.debug(f'Install: pip output {txt}')
     return txt
@@ -324,8 +323,7 @@ def git(arg: str, folder: str = None, ignore: bool = False, optional: bool = Fal
     if result.returncode != 0 and not ignore:
         if "couldn't find remote ref" in txt: # not a git repo
             return txt
-        global errors # pylint: disable=global-statement
-        errors += 1
+        errors.append(f'git: {folder}')
         log.error(f'Git: {folder} / {arg}')
         if 'or stash them' in txt:
             log.error(f'Git local changes detected: check details log="{log_file}"')
@@ -668,7 +666,7 @@ def install_torch_addons():
     triton_command = os.environ.get('TRITON_COMMAND', 'triton') if sys.platform == 'linux' else None
     if 'xformers' in xformers_package:
         try:
-            install(f'--no-deps {xformers_package}', ignore=True)
+            install(xformers_package, ignore=True, no_deps=True)
             import torch # pylint: disable=unused-import
             import xformers # pylint: disable=unused-import
         except Exception as e:
@@ -855,8 +853,7 @@ def run_extension_installer(folder):
         txt = result.stdout.decode(encoding="utf8", errors="ignore")
         debug(f'Extension installer: file="{path_installer}" {txt}')
         if result.returncode != 0:
-            global errors # pylint: disable=global-statement
-            errors += 1
+            errors.append(f'ext: {os.path.basename(folder)}')
             if len(result.stderr) > 0:
                 txt = txt + '\n' + result.stderr.decode(encoding="utf8", errors="ignore")
             log.error(f'Extension installer error: {path_installer}')
@@ -997,6 +994,29 @@ def ensure_base_requirements():
     install('requests', 'requests', quiet=True)
 
 
+def install_optional():
+    log.info('Installing optional requirements...')
+    install('basicsr')
+    install('gfpgan')
+    install('clean-fid')
+    install('optimum-quanto', ignore=True)
+    install('bitsandbytes', ignore=True)
+    install('pynvml', ignore=True)
+    install('ultralytics', ignore=True)
+    install('Cython', ignore=True)
+    install('insightface', ignore=True) # problematic build
+    install('nncf==2.7.0', ignore=True, no_deps=True) # requires older pandas
+    # install('flash-attn', ignore=True) # requires cuda and nvcc to be installed
+    install('gguf', ignore=True)
+    try:
+        import gguf
+        scripts_dir = os.path.join(os.path.dirname(gguf.__file__), '..', 'scripts')
+        if os.path.exists(scripts_dir):
+            os.rename(scripts_dir, scripts_dir + '_gguf')
+    except Exception:
+        pass
+
+
 def install_requirements():
     if args.profile:
         pr = cProfile.Profile()
@@ -1006,10 +1026,13 @@ def install_requirements():
     if not installed('diffusers', quiet=True): # diffusers are not installed, so run initial installation
         global quick_allowed # pylint: disable=global-statement
         quick_allowed = False
-        log.info('Installing requirements: this may take a while...')
+        log.info('Install requirements: this may take a while...')
         pip('install -r requirements.txt')
+    if args.optional:
+        quick_allowed = False
+        install_optional()
     installed('torch', reload=True) # reload packages cache
-    log.info('Verifying requirements')
+    log.info('Install: verifying requirements')
     with open('requirements.txt', 'r', encoding='utf8') as f:
         lines = [line.strip() for line in f.readlines() if line.strip() != '' and not line.startswith('#') and line is not None]
         for line in lines:
@@ -1272,6 +1295,7 @@ def add_args(parser):
     group_setup.add_argument('--upgrade', '--update', default = os.environ.get("SD_UPGRADE",False), action='store_true', help = "Upgrade main repository to latest version, default: %(default)s")
     group_setup.add_argument('--requirements', default = os.environ.get("SD_REQUIREMENTS",False), action='store_true', help = "Force re-check of requirements, default: %(default)s")
     group_setup.add_argument('--reinstall', default = os.environ.get("SD_REINSTALL",False), action='store_true', help = "Force reinstallation of all requirements, default: %(default)s")
+    group_setup.add_argument('--optional', default = os.environ.get("SD_OPTIONAL",False), action='store_true', help = "Force installation of optional requirements, default: %(default)s")
     group_setup.add_argument('--uv', default = os.environ.get("SD_UV",False), action='store_true', help = "Use uv instead of pip to install the packages")
 
     group_startup = parser.add_argument_group('Startup')

launch.py

@@ -240,7 +240,7 @@ def main():
             installer.install_extensions()
             installer.install_requirements() # redo requirements since extensions may change them
             installer.update_wiki()
-            if installer.errors == 0:
+            if len(installer.errors) == 0:
                 installer.log.debug(f'Setup complete without errors: {round(time.time())}')
             else:
                 installer.log.warning(f'Setup complete with errors: {installer.errors}')

requirements-extra.txt

@@ -1,14 +0,0 @@
-# additional requirements that are not required for sdnext base operations
-basicsr
-gfpgan
-clean-fid
-optimum-quanto
-bitsandbytes
-pynvml
-ultralytics
-pydantic==1.10.15
-albumentations==1.4.3
-clip_interrogator==0.6.0
-# insightface # problematic build
-# nncf==2.7.0 # requires older pandas
-# gguf # installs scripts

wiki

@@ -1 +1 @@
-Subproject commit 96f28bb7cec5a4e198a3244a88309f1957f75d03
+Subproject commit 4ceff56271ad78faa6ea2678494187d45f4e0fa6