From d28a988f722c045f7da3d3e1f89bc8be20b5d351 Mon Sep 17 00:00:00 2001 From: YSH Date: Thu, 13 Jun 2024 18:13:44 -0700 Subject: [PATCH 1/3] feat: Implement a more flexible method for initializing CLIP instances --- scripts/tokenizer.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/scripts/tokenizer.py b/scripts/tokenizer.py index dc67e64..7989a8e 100644 --- a/scripts/tokenizer.py +++ b/scripts/tokenizer.py @@ -1,8 +1,5 @@ import html - -from ldm.modules.encoders.modules import FrozenCLIPEmbedder, FrozenOpenCLIPEmbedder from modules import script_callbacks, shared -import open_clip.tokenizer import gradio as gr @@ -25,6 +22,9 @@ css = """ class VanillaClip: def __init__(self, clip): self.clip = clip + assert hasattr(self.clip, "tokenizer"), "VanillaClip requires 'tokenizer' attribute" + assert hasattr(self.clip.tokenizer, "get_vocab"), "Tokenizer must have 'get_vocab'" + assert hasattr(self.clip.tokenizer, "byte_decoder"), "Tokenizer must have 'byte_decoder'" def vocab(self): return self.clip.tokenizer.get_vocab() @@ -35,7 +35,9 @@ class VanillaClip: class OpenClip: def __init__(self, clip): self.clip = clip - self.tokenizer = open_clip.tokenizer._tokenizer + assert hasattr(self.clip, "tokenizer"), "OpenClip requires 'tokenizer' attribute" + assert hasattr(self.clip.tokenizer, "_tokenizer"), "Tokenizer must have '_tokenizer'" + self.tokenizer = self.clip.tokenizer._tokenizer def vocab(self): return self.tokenizer.encoder @@ -43,15 +45,23 @@ class OpenClip: def byte_decoder(self): return self.tokenizer.byte_decoder +def initialize_clip_instance(): + # Find all candidate CLIP + # For SDXL, it is FrozenCLIPEmbedderForSDXLWithCustomWords which is initalized in sd_hijack_clip.py as of the embedders + base = shared.sd_model.cond_stage_model + clip_candidates = [base.wrapped] + [embedder.wrapped for embedder in base.embedders if hasattr(embedder, 'wrapped')] + initializers = [VanillaClip, OpenClip] + for clip in clip_candidates: + for initializer in initializers: + try: + return initializer(clip) + except AssertionError: + continue + + raise RuntimeError('Failed to initialize a compatible CLIP instance from any candidate') def tokenize(text, input_is_ids=False): - clip = shared.sd_model.cond_stage_model.wrapped - if isinstance(clip, FrozenCLIPEmbedder): - clip = VanillaClip(shared.sd_model.cond_stage_model.wrapped) - elif isinstance(clip, FrozenOpenCLIPEmbedder): - clip = OpenClip(shared.sd_model.cond_stage_model.wrapped) - else: - raise RuntimeError(f'Unknown CLIP model: {type(clip).__name__}') + clip = initialize_clip_instance() if input_is_ids: tokens = [int(x.strip()) for x in text.split(",")] From 4bf5727123d98ad5a4b8c5ff469ba7cf1b9d4347 Mon Sep 17 00:00:00 2001 From: YSH Date: Thu, 13 Jun 2024 18:14:25 -0700 Subject: [PATCH 2/3] style: Update tokenizer CSS for improved readability and theme --- scripts/tokenizer.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/scripts/tokenizer.py b/scripts/tokenizer.py index 7989a8e..55cd612 100644 --- a/scripts/tokenizer.py +++ b/scripts/tokenizer.py @@ -6,16 +6,15 @@ import gradio as gr css = """ .tokenizer-token{ + opacity: 0.8; cursor: pointer; + --body-text-color: #000; } -.tokenizer-token-0 {background: rgba(255, 0, 0, 0.05);} -.tokenizer-token-0:hover {background: rgba(255, 0, 0, 0.15);} -.tokenizer-token-1 {background: rgba(0, 255, 0, 0.05);} -.tokenizer-token-1:hover {background: rgba(0, 255, 0, 0.15);} -.tokenizer-token-2 {background: rgba(0, 0, 255, 0.05);} -.tokenizer-token-2:hover {background: rgba(0, 0, 255, 0.15);} -.tokenizer-token-3 {background: rgba(255, 156, 0, 0.05);} -.tokenizer-token-3:hover {background: rgba(255, 156, 0, 0.15);} +.tokenizer-token:hover {opacity: 1;} +.tokenizer-token-0 {background: var(--primary-300);} +.tokenizer-token-1 {background: var(--primary-400);} +.tokenizer-token-2 {background: var(--primary-500);} +.tokenizer-token-3 {background: var(--primary-600);} """ From 7e74e5de76c39b314474f9fcc2615fb64381242d Mon Sep 17 00:00:00 2001 From: YSH Date: Thu, 13 Jun 2024 18:26:24 -0700 Subject: [PATCH 3/3] refactor: Improve CLIP instance initialization --- scripts/tokenizer.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/tokenizer.py b/scripts/tokenizer.py index 55cd612..7a45587 100644 --- a/scripts/tokenizer.py +++ b/scripts/tokenizer.py @@ -45,10 +45,13 @@ class OpenClip: return self.tokenizer.byte_decoder def initialize_clip_instance(): - # Find all candidate CLIP - # For SDXL, it is FrozenCLIPEmbedderForSDXLWithCustomWords which is initalized in sd_hijack_clip.py as of the embedders base = shared.sd_model.cond_stage_model - clip_candidates = [base.wrapped] + [embedder.wrapped for embedder in base.embedders if hasattr(embedder, 'wrapped')] + clip_candidates = [base.wrapped] + # For SDXL, it is FrozenCLIPEmbedderForSDXLWithCustomWords and some others which are initalized in sd_hijack_clip.py in the embedders + if hasattr(base, 'embedders'): + for embedder in base.embedders: + if hasattr(embedder, 'wrapped'): + clip_candidates.append(embedder.wrapped) initializers = [VanillaClip, OpenClip] for clip in clip_candidates: for initializer in initializers: @@ -56,7 +59,6 @@ def initialize_clip_instance(): return initializer(clip) except AssertionError: continue - raise RuntimeError('Failed to initialize a compatible CLIP instance from any candidate') def tokenize(text, input_is_ids=False):