From 64a09c59968bb48da3468cd52871edb6f1267b25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E4=B9=A6=E7=9D=BF?= <572754134@qq.com> Date: Wed, 6 Sep 2023 12:19:48 +0800 Subject: [PATCH] get back encoder_hidden_states only use text in controlnet inference --- ip_adapter/attention_processor.py | 6 ++++++ ip_adapter_controlnet_demo_new.ipynb | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ip_adapter/attention_processor.py b/ip_adapter/attention_processor.py index 9efac42..3b3e63f 100644 --- a/ip_adapter/attention_processor.py +++ b/ip_adapter/attention_processor.py @@ -440,6 +440,9 @@ class CNAttnProcessor: end_pos = encoder_hidden_states.shape[1] - self.num_tokens encoder_hidden_states = encoder_hidden_states[:, :end_pos] # only use text encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) + else: # get back text + end_pos = encoder_hidden_states.shape[1] - self.num_tokens + encoder_hidden_states = encoder_hidden_states[:, :end_pos] # only use text key = attn.to_k(encoder_hidden_states) value = attn.to_v(encoder_hidden_states) @@ -519,6 +522,9 @@ class CNAttnProcessor2_0: end_pos = encoder_hidden_states.shape[1] - self.num_tokens encoder_hidden_states = encoder_hidden_states[:, :end_pos] # only use text encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) + else: # get back text + end_pos = encoder_hidden_states.shape[1] - self.num_tokens + encoder_hidden_states = encoder_hidden_states[:, :end_pos] # only use text key = attn.to_k(encoder_hidden_states) value = attn.to_v(encoder_hidden_states) diff --git a/ip_adapter_controlnet_demo_new.ipynb b/ip_adapter_controlnet_demo_new.ipynb index b8c9dcd..48331f7 100644 --- a/ip_adapter_controlnet_demo_new.ipynb +++ b/ip_adapter_controlnet_demo_new.ipynb @@ -356,7 +356,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.9" + "version": "3.9.16" } }, "nbformat": 4,