From fcd542e5470241228856c2f1e85ea7b16ff087bf Mon Sep 17 00:00:00 2001 From: Chenlei Hu Date: Sat, 30 Mar 2024 03:22:17 +0000 Subject: [PATCH] Unify IP-Adapter preprocessor (#2710) --- scripts/controlnet.py | 16 ++-- scripts/global_state.py | 25 ++----- scripts/ipadapter/presets.py | 138 +++++++++++++++++++++++++++++++++++ scripts/processor.py | 2 +- 4 files changed, 152 insertions(+), 29 deletions(-) create mode 100644 scripts/ipadapter/presets.py diff --git a/scripts/controlnet.py b/scripts/controlnet.py index 34ca87e..edaeed2 100644 --- a/scripts/controlnet.py +++ b/scripts/controlnet.py @@ -17,6 +17,7 @@ from scripts.controlnet_lora import bind_control_lora, unbind_control_lora from scripts.processor import HWC3, preprocessor_sliders_config from scripts.controlnet_lllite import clear_all_lllite from scripts.ipadapter.plugable_ipadapter import ImageEmbed, clear_all_ip_adapter +from scripts.ipadapter.presets import IPAdapterPreset from scripts.utils import load_state_dict, get_unique_axis0, align_dim_latent from scripts.hook import ControlParams, UnetHook, HackedImageRNG from scripts.enums import ControlModelType, StableDiffusionVersion, HiResFixOption @@ -562,6 +563,12 @@ class Script(scripts.Script, metaclass=( enabled_units = [] for idx, unit in enumerate(units): local_unit = Script.parse_remote_call(p, unit, idx) + + # Consolidate meta preprocessors. + if local_unit.module == "ip-adapter-auto": + local_unit.module = IPAdapterPreset.match_model(local_unit.model).module + logger.info(f"ip-adapter-auto => {local_unit.module}") + if not local_unit.enabled: continue if hasattr(local_unit, "unfold_merged"): @@ -907,15 +914,6 @@ class Script(scripts.Script, metaclass=( for idx, unit in enumerate(self.enabled_units): Script.bound_check_params(unit) Script.check_sd_version_compatible(unit) - if ( - "ip-adapter" in unit.module and - not global_state.ip_adapter_pairing_model[unit.module](unit.model) - ): - logger.error(f"Invalid pair of IP-Adapter preprocessor({unit.module}) and model({unit.model}).\n" - "Please follow following pairing logic:\n" - + global_state.ip_adapter_pairing_logic_text) - continue - if ( 'inpaint_only' == unit.module and issubclass(type(p), StableDiffusionProcessingImg2Img) and diff --git a/scripts/global_state.py b/scripts/global_state.py index 454e99a..3463a11 100644 --- a/scripts/global_state.py +++ b/scripts/global_state.py @@ -179,9 +179,15 @@ preprocessor_aliases = { "densepose": "densepose (pruple bg & purple torso)", "densepose_parula": "densepose_parula (black bg & blue torso)", "te_hed": "softedge_teed", + "ip-adapter_clip_sd15": "ip-adapter_clip_h", + "ip-adapter_clip_sdxl": "ip-adapter_clip_g", } +# Preprocessor that automatically maps to other preprocessors. +meta_preprocessors = ["ip-adapter-auto"] + ui_preprocessor_keys = ['none', preprocessor_aliases['invert']] +ui_preprocessor_keys += meta_preprocessors ui_preprocessor_keys += sorted([preprocessor_aliases.get(k, k) for k in cn_preprocessor_modules.keys() if preprocessor_aliases.get(k, k) not in ui_preprocessor_keys]) @@ -353,22 +359,3 @@ def select_control_type( default_option, default_model ) - - -ip_adapter_pairing_model = { - "ip-adapter_clip_sdxl": lambda model: "faceid" not in model and "vit" not in model, - "ip-adapter_clip_sdxl_plus_vith": lambda model: "faceid" not in model and "vit" in model, - "ip-adapter_clip_sd15": lambda model: "faceid" not in model, - "ip-adapter_face_id": lambda model: "faceid" in model and "plus" not in model, - "ip-adapter_face_id_plus": lambda model: "faceid" in model and "plus" in model, -} - -ip_adapter_pairing_logic_text = """ -{ - "ip-adapter_clip_sdxl": lambda model: "faceid" not in model and "vit" not in model, - "ip-adapter_clip_sdxl_plus_vith": lambda model: "faceid" not in model and "vit" in model, - "ip-adapter_clip_sd15": lambda model: "faceid" not in model, - "ip-adapter_face_id": lambda model: "faceid" in model and "plus" not in model, - "ip-adapter_face_id_plus": lambda model: "faceid" in model and "plus" in model, -} -""" diff --git a/scripts/ipadapter/presets.py b/scripts/ipadapter/presets.py new file mode 100644 index 0000000..f463f0f --- /dev/null +++ b/scripts/ipadapter/presets.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +from ..enums import StableDiffusionVersion +from typing import NamedTuple, Optional, List + + +class IPAdapterPreset(NamedTuple): + """Preset for IPAdapter.""" + + name: str + module: str # Preprocessor + model: str # Name of model file + sd_version: StableDiffusionVersion # Supported SD version. + lora: Optional[str] = None + + @staticmethod + def match_model(model_name: str) -> IPAdapterPreset: + model_name = model_name.split("[")[0].strip() + return _preset_by_model[model_name] + + +clip_h = "ip-adapter_clip_h" +clip_g = "ip-adapter_clip_g" +insightface = "ip-adapter_face_id" +insightface_clip_h = "ip-adapter_face_id_plus" + + +ipadapter_presets: List[IPAdapterPreset] = [ + IPAdapterPreset( + name="light", + module=clip_h, + model="ip-adapter_sd15_light", + sd_version=StableDiffusionVersion.SD1x, + ), + IPAdapterPreset( + name="vit-g", + module=clip_g, + model="ip-adapter_sd15_vit-G", + sd_version=StableDiffusionVersion.SD1x, + ), + IPAdapterPreset( + name="standard", + module=clip_h, + model="ip-adapter_sd15", + sd_version=StableDiffusionVersion.SD1x, + ), + IPAdapterPreset( + name="plus", + module=clip_h, + model="ip-adapter-plus_sd15", + sd_version=StableDiffusionVersion.SD1x, + ), + IPAdapterPreset( + name="plus_face", + module=clip_h, + model="ip-adapter-plus-face_sd15", + sd_version=StableDiffusionVersion.SD1x, + ), + IPAdapterPreset( + name="full_face", + module=clip_h, + model="ip-adapter-full-face_sd15", + sd_version=StableDiffusionVersion.SD1x, + ), + IPAdapterPreset( + name="face_id", + module=insightface, + model="ip-adapter-faceid_sd15", + lora="ip-adapter-faceid_sd15_lora", + sd_version=StableDiffusionVersion.SD1x, + ), + IPAdapterPreset( + name="face_id_plus", + module=insightface_clip_h, + model="ip-adapter-faceid-plus_sd15", + lora="ip-adapter-faceid-plus_sd15_lora", + sd_version=StableDiffusionVersion.SD1x, + ), + IPAdapterPreset( + name="face_id_plus_v2", + module=insightface_clip_h, + model="ip-adapter-faceid-plusv2_sd15", + lora="ip-adapter-faceid-plusv2_sd15_lora", + sd_version=StableDiffusionVersion.SD1x, + ), + IPAdapterPreset( + name="face_id_portrait", + module=insightface, + model="ip-adapter-faceid-portrait_sd15", + sd_version=StableDiffusionVersion.SD1x, + ), + IPAdapterPreset( + name="standard-g", + module=clip_g, + model="ip-adapter_sdxl", + sd_version=StableDiffusionVersion.SDXL, + ), + IPAdapterPreset( + name="standard-h", + module=clip_h, + model="ip-adapter_sdxl_vit-h", + sd_version=StableDiffusionVersion.SDXL, + ), + IPAdapterPreset( + name="plus-h", + module=clip_h, + model="ip-adapter-plus_sdxl_vit-h", + sd_version=StableDiffusionVersion.SDXL, + ), + IPAdapterPreset( + name="plus_face-h", + module=clip_h, + model="ip-adapter-plus-face_sdxl_vit-h", + sd_version=StableDiffusionVersion.SDXL, + ), + IPAdapterPreset( + name="face_id", + module=insightface, + model="ip-adapter-faceid_sdxl", + lora="ip-adapter-faceid_sdxl_lora", + sd_version=StableDiffusionVersion.SDXL, + ), + IPAdapterPreset( + name="face_id_plusv2", + module=insightface_clip_h, + model="ip-adapter-faceid-plusv2_sdxl", + lora="ip-adapter-faceid-plusv2_sdxl_lora", + sd_version=StableDiffusionVersion.SDXL, + ), + IPAdapterPreset( + name="face_id_portrait", + module=insightface, + model="ip-adapter-faceid-portrait_sdxl", + sd_version=StableDiffusionVersion.SDXL, + ), +] + +_preset_by_model = {p.model: p for p in ipadapter_presets} diff --git a/scripts/processor.py b/scripts/processor.py index d0f21bf..068504f 100644 --- a/scripts/processor.py +++ b/scripts/processor.py @@ -1355,7 +1355,7 @@ preprocessor_filters = { "Recolor": "recolor_luminance", "Revision": "revision_clipvision", "T2I-Adapter": "none", - "IP-Adapter": "ip-adapter_clip_sd15", + "IP-Adapter": "ip-adapter-auto", "Instant_ID": "instant_id", "SparseCtrl": "none", }