diff --git a/README.md b/README.md index 66e49a917..46031e29c 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ Additional models will be added as they become available and there is public int - Any GPU compatibile with *DirectX* on *Windows* using **DirectML** libraries. This includes support for AMD GPUs that are not supported by native ROCm libraries - *Intel Arc* GPUs using *Intel OneAPI* **Ipex/XPU** libraries -- *Intel* iGPUs using *Intel OneAPI* **OpenVINO** libraries +- *Intel* GPUs using *Intel OneAPI* **OpenVINO** libraries - *Apple M1/M2* on *OSX* using built-in support in Torch with **MPS** optimizations ## Install & Run @@ -96,6 +96,7 @@ Below is partial list of all available parameters, run `webui --help` for the fu Setup options: --use-directml Use DirectML if no compatible GPU is detected, default: False + --use-openvino Use Intel OpenVINO backend, default: False --use-ipex Force use Intel OneAPI XPU backend, default: False --use-cuda Force use nVidia CUDA backend, default: False --use-rocm Force use AMD ROCm backend, default: False diff --git a/installer.py b/installer.py index a6e50ca59..a14908499 100644 --- a/installer.py +++ b/installer.py @@ -319,12 +319,13 @@ def check_torch(): if args.profile: pr = cProfile.Profile() pr.enable() - allow_cuda = not (args.use_rocm or args.use_directml or args.use_ipex) - allow_rocm = not (args.use_cuda or args.use_directml or args.use_ipex) - allow_ipex = not (args.use_cuda or args.use_rocm or args.use_directml) - allow_directml = not (args.use_cuda or args.use_rocm or args.use_ipex) - log.debug(f'Torch overrides: cuda={args.use_cuda} rocm={args.use_rocm} ipex={args.use_ipex} diml={args.use_directml}') - log.debug(f'Torch allowed: cuda={allow_cuda} rocm={allow_rocm} ipex={allow_ipex} diml={allow_directml}') + allow_cuda = not (args.use_rocm or args.use_directml or args.use_ipex or args.use_openvino) + allow_rocm = not (args.use_cuda or args.use_directml or args.use_ipex or args.use_openvino) + allow_ipex = not (args.use_cuda or args.use_rocm or args.use_directml or args.use_openvino) + allow_directml = not (args.use_cuda or args.use_rocm or args.use_ipex or args.use_openvino) + allow_openvino = not (args.use_cuda or args.use_rocm or args.use_ipex or args.use_directml) + log.debug(f'Torch overrides: cuda={args.use_cuda} rocm={args.use_rocm} ipex={args.use_ipex} diml={args.use_directml} openvino={args.use_openvino}') + log.debug(f'Torch allowed: cuda={allow_cuda} rocm={allow_rocm} ipex={allow_ipex} diml={allow_directml} openvino={allow_openvino}') torch_command = os.environ.get('TORCH_COMMAND', '') xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none') if torch_command != '': @@ -392,6 +393,10 @@ def check_torch(): os.environ.setdefault('TENSORFLOW_PACKAGE', 'tensorflow==2.13.0 intel-extension-for-tensorflow[gpu]') else: torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.0.0a0 intel_extension_for_pytorch==2.0.110+gitba7f6c1 -f https://developer.intel.com/ipex-whl-stable-xpu') + elif allow_openvino and args.use_openvino: + #Remove this after 2.1.0 releases + log.info('Using OpenVINO with Torch Nightly CPU') + torch_command = os.environ.get('TORCH_COMMAND', '--pre torch==2.1.0.dev20230713+cpu torchvision==0.16.0.dev20230713+cpu -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html') else: machine = platform.machine() if sys.platform == 'darwin': @@ -456,7 +461,7 @@ def check_torch(): log.debug(f'Cannot install xformers package: {e}') if opts.get('cuda_compile_backend', '') == 'hidet': install('hidet', 'hidet') - if opts.get('cuda_compile_backend', '') == 'openvino_fx': + if args.use_openvino or opts.get('cuda_compile_backend', '') == 'openvino_fx': install('openvino==2023.1.0.dev20230811', 'openvino') os.environ.setdefault('PYTORCH_TRACING_MODE', 'TORCHFX') if args.profile: @@ -495,7 +500,6 @@ def install_packages(): install(invisiblewatermark_package, 'invisible-watermark') install('onnxruntime==1.15.1', 'onnxruntime', ignore=True) install('pi-heif', 'pi_heif', ignore=True) - install('git+https://github.com/damian0815/compel', 'compel', ignore=True) tensorflow_package = os.environ.get('TENSORFLOW_PACKAGE', 'tensorflow==2.13.0') install(tensorflow_package, 'tensorflow', ignore=True) install('git+https://github.com/google-research/torchsde', 'torchsde', ignore=True) @@ -831,6 +835,7 @@ def add_args(parser): group.add_argument('--requirements', default = False, action='store_true', help = "Force re-check of requirements, default: %(default)s") group.add_argument('--quick', default = False, action='store_true', help = "Run with startup sequence only, default: %(default)s") group.add_argument('--use-directml', default = False, action='store_true', help = "Use DirectML if no compatible GPU is detected, default: %(default)s") + group.add_argument("--use-openvino", default = False, action='store_true', help="Use Intel OpenVINO backend, default: %(default)s") group.add_argument("--use-ipex", default = False, action='store_true', help="Force use Intel OneAPI XPU backend, default: %(default)s") group.add_argument("--use-cuda", default=False, action='store_true', help="Force use nVidia CUDA backend, default: %(default)s") group.add_argument("--use-rocm", default=False, action='store_true', help="Force use AMD ROCm backend, default: %(default)s") diff --git a/modules/cmd_args.py b/modules/cmd_args.py index e8d8ef889..41b6340da 100644 --- a/modules/cmd_args.py +++ b/modules/cmd_args.py @@ -41,6 +41,7 @@ group.add_argument("--profile", action='store_true', help="Run profiler, default group.add_argument("--disable-queue", action='store_true', help="Disable queues, default: %(default)s") group.add_argument('--debug', default = False, action='store_true', help = "Run installer with debug logging, default: %(default)s") group.add_argument('--use-directml', default = False, action='store_true', help = "Use DirectML if no compatible GPU is detected, default: %(default)s") +group.add_argument("--use-openvino", default = False, action='store_true', help="Use Intel OpenVINO backend, default: %(default)s") group.add_argument("--use-ipex", default = False, action='store_true', help="Force use Intel OneAPI XPU backend, default: %(default)s") group.add_argument("--use-cuda", default=False, action='store_true', help="Force use nVidia CUDA backend, default: %(default)s") group.add_argument("--use-rocm", default=False, action='store_true', help="Force use AMD ROCm backend, default: %(default)s") diff --git a/modules/intel/openvino/__init__.py b/modules/intel/openvino/__init__.py index d0835726f..d31029e3b 100644 --- a/modules/intel/openvino/__init__.py +++ b/modules/intel/openvino/__init__.py @@ -8,7 +8,6 @@ from torch._dynamo.backends.registry import register_backend from torch.fx.experimental.proxy_tensor import make_fx from torch._inductor.compile_fx import compile_fx from hashlib import sha256 -import modules.shared class ModelState: def __init__(self): @@ -113,3 +112,15 @@ def get_cached_file_name(*args, model_hash_str, device, cache_root): file_name = None model_hash_str = None return file_name + +def openvino_override_opts(): + from modules import shared + if not shared.opts.cuda_compile: + shared.log.warn("OpenVINO: Enabling Torch Compile") + shared.opts.cuda_compile = True + if shared.opts.cuda_compile_backend != "openvino_fx": + shared.log.warn("OpenVINO: Setting Torch Compiler backend to OpenVINO FX") + shared.opts.cuda_compile_backend = "openvino_fx" + if shared.opts.sd_backend != "diffusers": + shared.log.warn("OpenVINO: Setting backend to Diffusers") + shared.opts.sd_backend = "diffusers" diff --git a/modules/shared.py b/modules/shared.py index 6d8275b8f..70da36eea 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -184,8 +184,11 @@ class State: state = State() state.server_start = time.time() - -backend = Backend.DIFFUSERS if (cmd_opts.backend is not None) and (cmd_opts.backend.lower() == 'diffusers') else Backend.ORIGINAL # initial since we don't have opts loaded yet +if cmd_opts.use_openvino: + backend = Backend.DIFFUSERS + cmd_opts.backend = 'diffusers' +else: + backend = Backend.DIFFUSERS if (cmd_opts.backend is not None) and (cmd_opts.backend.lower() == 'diffusers') else Backend.ORIGINAL # initial since we don't have opts loaded yet class OptionInfo: @@ -342,7 +345,7 @@ else: # cuda cross_attention_optimization_default ="Scaled-Dot-Product" options_templates.update(options_section(('sd', "Stable Diffusion"), { - "sd_backend": OptionInfo("original", "Stable Diffusion backend", gr.Radio, lambda: {"choices": ["original", "diffusers"] }), + "sd_backend": OptionInfo("diffusers" if cmd_opts.use_openvino else "original", "Stable Diffusion backend", gr.Radio, lambda: {"choices": ["original", "diffusers"] }), "sd_checkpoint_autoload": OptionInfo(True, "Stable Diffusion checkpoint autoload on server start"), "sd_model_checkpoint": OptionInfo(default_checkpoint, "Stable Diffusion checkpoint", gr.Dropdown, lambda: {"choices": list_checkpoint_tiles()}, refresh=refresh_checkpoints), "sd_model_refiner": OptionInfo('None', "Stable Diffusion refiner", gr.Dropdown, lambda: {"choices": ['None'] + list_checkpoint_tiles()}, refresh=refresh_checkpoints), @@ -373,7 +376,7 @@ options_templates.update(options_section(('optimizations', "Optimizations"), { options_templates.update(options_section(('cuda', "Compute Settings"), { "memmon_poll_rate": OptionInfo(2, "VRAM usage polls per second during generation", gr.Slider, {"minimum": 0, "maximum": 40, "step": 1}), "precision": OptionInfo("Autocast", "Precision type", gr.Radio, lambda: {"choices": ["Autocast", "Full"]}), - "cuda_dtype": OptionInfo("FP32" if sys.platform == "darwin" else "BF16" if devices.backend == "ipex" else "FP16", "Device precision type", gr.Radio, lambda: {"choices": ["FP32", "FP16", "BF16"]}), + "cuda_dtype": OptionInfo("FP32" if sys.platform == "darwin" or cmd_opts.use_openvino else "BF16" if devices.backend == "ipex" else "FP16", "Device precision type", gr.Radio, lambda: {"choices": ["FP32", "FP16", "BF16"]}), "no_half": OptionInfo(False, "Use full precision for model (--no-half)", None, None, None), "no_half_vae": OptionInfo(False, "Use full precision for VAE (--no-half-vae)"), "upcast_sampling": OptionInfo(True if sys.platform == "darwin" else False, "Enable upcast sampling"), @@ -385,8 +388,8 @@ options_templates.update(options_section(('cuda', "Compute Settings"), { "cudnn_benchmark": OptionInfo(False, "Enable full-depth cuDNN benchmark feature"), # "cuda_allow_tf32": OptionInfo(True, "Allow TF32 math ops"), # "cuda_allow_tf16_reduced": OptionInfo(True, "Allow TF16 reduced precision math ops"), - "cuda_compile": OptionInfo(False, "Enable model compile (experimental)"), - "cuda_compile_backend": OptionInfo("none", "Model compile backend (experimental)", gr.Radio, lambda: {"choices": ['none', 'inductor', 'cudagraphs', 'aot_ts_nvfuser', 'hidet', 'ipex', 'openvino_fx']}), + "cuda_compile": OptionInfo(True if cmd_opts.use_openvino else False, "Enable model compile (experimental)"), + "cuda_compile_backend": OptionInfo("openvino_fx" if cmd_opts.use_openvino else "none", "Model compile backend (experimental)", gr.Radio, lambda: {"choices": ['none', 'inductor', 'cudagraphs', 'aot_ts_nvfuser', 'hidet', 'ipex', 'openvino_fx']}), "cuda_compile_mode": OptionInfo("default", "Model compile mode (experimental)", gr.Radio, lambda: {"choices": ['default', 'reduce-overhead', 'max-autotune']}), "cuda_compile_fullgraph": OptionInfo(False, "Model compile fullgraph"), "cuda_compile_verbose": OptionInfo(False, "Model compile verbose mode"), @@ -801,9 +804,9 @@ config_filename = cmd_opts.config opts.load(config_filename) cmd_opts = cmd_args.compatibility_args(opts, cmd_opts) if cmd_opts.backend is None: - backend = Backend.DIFFUSERS if opts.data.get('sd_backend', 'original') == 'diffusers' else Backend.ORIGINAL + backend = Backend.DIFFUSERS if cmd_opts.use_openvino or opts.data.get('sd_backend', 'original') == 'diffusers' else Backend.ORIGINAL else: - backend = Backend.DIFFUSERS if cmd_opts.backend.lower() == 'diffusers' else Backend.ORIGINAL + backend = Backend.DIFFUSERS if cmd_opts.use_openvino or cmd_opts.backend.lower() == 'diffusers' else Backend.ORIGINAL opts.data['sd_backend'] = 'diffusers' if backend == Backend.DIFFUSERS else 'original' opts.data['uni_pc_lower_order_final'] = opts.schedulers_use_loworder opts.data['uni_pc_order'] = opts.schedulers_solver_order @@ -997,7 +1000,7 @@ class Shared(sys.modules[__name__].__class__): # this class is here to provide s @property def backend(self): - return Backend.ORIGINAL if opts.data['sd_backend'] == 'original' else Backend.DIFFUSERS + return Backend.ORIGINAL if not cmd_opts.use_openvino and opts.data['sd_backend'] == 'original' else Backend.DIFFUSERS @property def sd_model_type(self): diff --git a/modules/ui.py b/modules/ui.py index fe62868ae..3b3a608dd 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -15,6 +15,7 @@ from modules.ui_components import FormRow, FormColumn, FormGroup, ToolButton, Fo from modules.paths import script_path, data_path from modules.shared import opts, cmd_opts, readfile from modules.dml import directml_override_opts +from modules.intel.openvino import openvino_override_opts from modules import prompt_parser import modules.codeformer_model import modules.generation_parameters_copypaste as parameters_copypaste @@ -966,6 +967,8 @@ def create_ui(startup_timer = None): changed.append(key) if cmd_opts.use_directml: directml_override_opts() + if cmd_opts.use_openvino: + openvino_override_opts() try: opts.save(modules.shared.config_filename) log.info(f'Settings changed: {len(changed)} {changed}') diff --git a/requirements.txt b/requirements.txt index 098ba25a4..a6c8f9d7c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,6 +39,7 @@ voluptuous yapf scikit-image basicsr +compel==2.0.2 fasteners typing-extensions==4.7.1 antlr4-python3-runtime==4.9.3