From 6be52643b57b626b9ae3bbc9497314ee6297167a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 14 Nov 2024 18:09:53 -0500 Subject: [PATCH] update docker build Signed-off-by: Vladimir Mandic --- .gitignore | 1 - Dockerfile | 7 ++++--- installer.py | 44 ++++++++++++++++++++++++++++++++---------- launch.py | 2 +- requirements-extra.txt | 14 -------------- wiki | 2 +- 6 files changed, 40 insertions(+), 30 deletions(-) delete mode 100644 requirements-extra.txt diff --git a/.gitignore b/.gitignore index 9fac9b310..6df029445 100644 --- a/.gitignore +++ b/.gitignore @@ -44,7 +44,6 @@ tunableop_results*.csv !webui.sh !package.json !requirements.txt -!requirements-extra.txt # pyinstaller *.spec diff --git a/Dockerfile b/Dockerfile index 28af761f7..8e19d145c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,9 @@ WORKDIR / COPY . . # stop pip and uv from caching ENV PIP_NO_CACHE_DIR=true +ENV PIP_ROOT_USER_ACTION=ignore ENV UV_NO_CACHE=true +ENV SD_INSTALL_DEBUG=true # disable model hashing for faster startup ENV SD_NOHASHING=true # set data directories @@ -14,11 +16,10 @@ ENV SD_DATADIR="/mnt/data" ENV SD_MODELSDIR="/mnt/models" # install dependencies RUN ["apt-get", "-y", "update"] -RUN ["apt-get", "-y", "install", "git"] +RUN ["apt-get", "-y", "install", "git", "build-essential"] # sdnext will run all necessary pip install ops and then exit -RUN ["python", "launch.py", "--debug", "--uv", "--use-cuda", "--log", "sdnext.log", "--test"] +RUN ["python", "launch.py", "--debug", "--uv", "--use-cuda", "--log", "sdnext.log", "--test", "--optional"] # preinstall additional packages to avoid installation during runtime -RUN ["uv", "pip", "install", "-r", "requirements-extra.txt", "--system"] # actually run sdnext CMD ["python", "launch.py", "--debug", "--skip-all", "--listen", "--quick", "--api-log", "--log", "sdnext.log"] # expose port diff --git a/installer.py b/installer.py index 80bca224c..7a5b9d756 100644 --- a/installer.py +++ b/installer.py @@ -27,7 +27,7 @@ log_file = os.path.join(os.path.dirname(__file__), 'sdnext.log') log_rolled = False first_call = True quick_allowed = True -errors = 0 +errors = [] opts = {} args = Dot({ 'debug': False, @@ -280,8 +280,7 @@ def pip(arg: str, ignore: bool = False, quiet: bool = False, uv = True): txt = txt.strip() debug(f'Install {pipCmd}: {txt}') if result.returncode != 0 and not ignore: - global errors # pylint: disable=global-statement - errors += 1 + errors.append(f'pip: {package}') log.error(f'Install: {pipCmd}: {arg}') log.debug(f'Install: pip output {txt}') return txt @@ -324,8 +323,7 @@ def git(arg: str, folder: str = None, ignore: bool = False, optional: bool = Fal if result.returncode != 0 and not ignore: if "couldn't find remote ref" in txt: # not a git repo return txt - global errors # pylint: disable=global-statement - errors += 1 + errors.append(f'git: {folder}') log.error(f'Git: {folder} / {arg}') if 'or stash them' in txt: log.error(f'Git local changes detected: check details log="{log_file}"') @@ -668,7 +666,7 @@ def install_torch_addons(): triton_command = os.environ.get('TRITON_COMMAND', 'triton') if sys.platform == 'linux' else None if 'xformers' in xformers_package: try: - install(f'--no-deps {xformers_package}', ignore=True) + install(xformers_package, ignore=True, no_deps=True) import torch # pylint: disable=unused-import import xformers # pylint: disable=unused-import except Exception as e: @@ -855,8 +853,7 @@ def run_extension_installer(folder): txt = result.stdout.decode(encoding="utf8", errors="ignore") debug(f'Extension installer: file="{path_installer}" {txt}') if result.returncode != 0: - global errors # pylint: disable=global-statement - errors += 1 + errors.append(f'ext: {os.path.basename(folder)}') if len(result.stderr) > 0: txt = txt + '\n' + result.stderr.decode(encoding="utf8", errors="ignore") log.error(f'Extension installer error: {path_installer}') @@ -997,6 +994,29 @@ def ensure_base_requirements(): install('requests', 'requests', quiet=True) +def install_optional(): + log.info('Installing optional requirements...') + install('basicsr') + install('gfpgan') + install('clean-fid') + install('optimum-quanto', ignore=True) + install('bitsandbytes', ignore=True) + install('pynvml', ignore=True) + install('ultralytics', ignore=True) + install('Cython', ignore=True) + install('insightface', ignore=True) # problematic build + install('nncf==2.7.0', ignore=True, no_deps=True) # requires older pandas + # install('flash-attn', ignore=True) # requires cuda and nvcc to be installed + install('gguf', ignore=True) + try: + import gguf + scripts_dir = os.path.join(os.path.dirname(gguf.__file__), '..', 'scripts') + if os.path.exists(scripts_dir): + os.rename(scripts_dir, scripts_dir + '_gguf') + except Exception: + pass + + def install_requirements(): if args.profile: pr = cProfile.Profile() @@ -1006,10 +1026,13 @@ def install_requirements(): if not installed('diffusers', quiet=True): # diffusers are not installed, so run initial installation global quick_allowed # pylint: disable=global-statement quick_allowed = False - log.info('Installing requirements: this may take a while...') + log.info('Install requirements: this may take a while...') pip('install -r requirements.txt') + if args.optional: + quick_allowed = False + install_optional() installed('torch', reload=True) # reload packages cache - log.info('Verifying requirements') + log.info('Install: verifying requirements') with open('requirements.txt', 'r', encoding='utf8') as f: lines = [line.strip() for line in f.readlines() if line.strip() != '' and not line.startswith('#') and line is not None] for line in lines: @@ -1272,6 +1295,7 @@ def add_args(parser): group_setup.add_argument('--upgrade', '--update', default = os.environ.get("SD_UPGRADE",False), action='store_true', help = "Upgrade main repository to latest version, default: %(default)s") group_setup.add_argument('--requirements', default = os.environ.get("SD_REQUIREMENTS",False), action='store_true', help = "Force re-check of requirements, default: %(default)s") group_setup.add_argument('--reinstall', default = os.environ.get("SD_REINSTALL",False), action='store_true', help = "Force reinstallation of all requirements, default: %(default)s") + group_setup.add_argument('--optional', default = os.environ.get("SD_OPTIONAL",False), action='store_true', help = "Force installation of optional requirements, default: %(default)s") group_setup.add_argument('--uv', default = os.environ.get("SD_UV",False), action='store_true', help = "Use uv instead of pip to install the packages") group_startup = parser.add_argument_group('Startup') diff --git a/launch.py b/launch.py index af3db2c0f..9075c358b 100755 --- a/launch.py +++ b/launch.py @@ -240,7 +240,7 @@ def main(): installer.install_extensions() installer.install_requirements() # redo requirements since extensions may change them installer.update_wiki() - if installer.errors == 0: + if len(installer.errors) == 0: installer.log.debug(f'Setup complete without errors: {round(time.time())}') else: installer.log.warning(f'Setup complete with errors: {installer.errors}') diff --git a/requirements-extra.txt b/requirements-extra.txt deleted file mode 100644 index 5be6fd24c..000000000 --- a/requirements-extra.txt +++ /dev/null @@ -1,14 +0,0 @@ -# additional requirements that are not required for sdnext base operations -basicsr -gfpgan -clean-fid -optimum-quanto -bitsandbytes -pynvml -ultralytics -pydantic==1.10.15 -albumentations==1.4.3 -clip_interrogator==0.6.0 -# insightface # problematic build -# nncf==2.7.0 # requires older pandas -# gguf # installs scripts diff --git a/wiki b/wiki index 96f28bb7c..4ceff5627 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 96f28bb7cec5a4e198a3244a88309f1957f75d03 +Subproject commit 4ceff56271ad78faa6ea2678494187d45f4e0fa6