From 63b69324bccf9bf7808bc417829dfd59502d3c14 Mon Sep 17 00:00:00 2001 From: Chengsong Zhang Date: Thu, 13 Apr 2023 05:36:01 +0800 Subject: [PATCH] runnable version --- GroundingDINO | 1 - README.md | 15 +------ install.py | 7 +-- .../grounding-dino/GroundingDINO_SwinB.cfg.py | 43 +++++++++++++++++++ .../grounding-dino/GroundingDINO_SwinT_OGC.py | 43 +++++++++++++++++++ requirements.txt | 3 +- scripts/dino.py | 14 +++--- 7 files changed, 99 insertions(+), 27 deletions(-) delete mode 160000 GroundingDINO create mode 100644 models/grounding-dino/GroundingDINO_SwinB.cfg.py create mode 100644 models/grounding-dino/GroundingDINO_SwinT_OGC.py diff --git a/GroundingDINO b/GroundingDINO deleted file mode 160000 index 9dac4c6..0000000 --- a/GroundingDINO +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9dac4c605b40d87d82bfff5961467556f4d381a9 diff --git a/README.md b/README.md index 9a71e20..e6ca9cf 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,18 @@ # Segment Anything for Stable Diffusion WebUI -**This branch is still developing. I discourage you to use it before I remove this warning from README.** +This **developing** branch attempts to enable [GroundingDINO](https://github.com/IDEA-Research/GroundingDINO), achieving the goal of text prompt->object detection->segment anything->stable diffusion inpainting. -This branch attemts to enable [GroundingDINO](https://github.com/IDEA-Research/GroundingDINO), achieving the goal of text prompt->object detection->segment anything->stable diffusion inpainting. - -Warning: You may have a lot trouble using this extension from this branch, due to several incompability of SAM, GroundingDINO, etc. Please follow these steps to bypass potential error. - -- If you see `ModuleNotFoundError: No module named 'groundingdino'` on your terminal, restart your webui may help. -- You may need a long time to get GroundingDINO and its dependency build, because they have C++. +Warning: You may need a long time to get GroundingDINO and its dependency build, because they have C++. To use this extension, simply run `git checkout GroundingDINO` on your terminal after `cd` to this extension directory. This branch will be merged to master branch someday, when I think it is stable enough. -## Help Wanted - -If anyone know a good solution of installing `GroundingDINO`, please let me know and submit a pull request. My way of installment (in `install.py`) needs a restart, which is honestly not clean. I have not found another way of installing `GroundingDINO` - ## Next Step: -- Find a smooth way to install GroundingDINO without restart - point + detection, find a best way to satisfy need: - 0 point + N detection->box: BHW, mask: BCHW - ALL point + 1 detection->3 images - Preview detection + select the best - ## How to Use TODO diff --git a/install.py b/install.py index 14feba8..984e858 100644 --- a/install.py +++ b/install.py @@ -8,10 +8,7 @@ with open(req_file) as file: for lib in file: lib = lib.strip() if not launch.is_installed(lib): + if lib == "groundingdino": + lib = "git+https://github.com/IDEA-Research/GroundingDINO" launch.run_pip( f"install {lib}", f"sd-webui-segment-anything requirement: {lib}") - -# dino_dir = os.path.join(current_dir, "GroundingDINO") -# launch.git_clone("https://github.com/IDEA-Research/GroundingDINO.git", dino_dir, "GroundingDINO") -# launch.run(f"{launch.python} -m pip install -e {dino_dir}") -launch.git_clone() diff --git a/models/grounding-dino/GroundingDINO_SwinB.cfg.py b/models/grounding-dino/GroundingDINO_SwinB.cfg.py new file mode 100644 index 0000000..f490c4b --- /dev/null +++ b/models/grounding-dino/GroundingDINO_SwinB.cfg.py @@ -0,0 +1,43 @@ +batch_size = 1 +modelname = "groundingdino" +backbone = "swin_B_384_22k" +position_embedding = "sine" +pe_temperatureH = 20 +pe_temperatureW = 20 +return_interm_indices = [1, 2, 3] +backbone_freeze_keywords = None +enc_layers = 6 +dec_layers = 6 +pre_norm = False +dim_feedforward = 2048 +hidden_dim = 256 +dropout = 0.0 +nheads = 8 +num_queries = 900 +query_dim = 4 +num_patterns = 0 +num_feature_levels = 4 +enc_n_points = 4 +dec_n_points = 4 +two_stage_type = "standard" +two_stage_bbox_embed_share = False +two_stage_class_embed_share = False +transformer_activation = "relu" +dec_pred_bbox_embed_share = True +dn_box_noise_scale = 1.0 +dn_label_noise_ratio = 0.5 +dn_label_coef = 1.0 +dn_bbox_coef = 1.0 +embed_init_tgt = True +dn_labelbook_size = 2000 +max_text_len = 256 +text_encoder_type = "bert-base-uncased" +use_text_enhancer = True +use_fusion_layer = True +use_checkpoint = True +use_transformer_ckpt = True +use_text_cross_attention = True +text_dropout = 0.0 +fusion_dropout = 0.0 +fusion_droppath = 0.1 +sub_sentence_present = True diff --git a/models/grounding-dino/GroundingDINO_SwinT_OGC.py b/models/grounding-dino/GroundingDINO_SwinT_OGC.py new file mode 100644 index 0000000..9158d5f --- /dev/null +++ b/models/grounding-dino/GroundingDINO_SwinT_OGC.py @@ -0,0 +1,43 @@ +batch_size = 1 +modelname = "groundingdino" +backbone = "swin_T_224_1k" +position_embedding = "sine" +pe_temperatureH = 20 +pe_temperatureW = 20 +return_interm_indices = [1, 2, 3] +backbone_freeze_keywords = None +enc_layers = 6 +dec_layers = 6 +pre_norm = False +dim_feedforward = 2048 +hidden_dim = 256 +dropout = 0.0 +nheads = 8 +num_queries = 900 +query_dim = 4 +num_patterns = 0 +num_feature_levels = 4 +enc_n_points = 4 +dec_n_points = 4 +two_stage_type = "standard" +two_stage_bbox_embed_share = False +two_stage_class_embed_share = False +transformer_activation = "relu" +dec_pred_bbox_embed_share = True +dn_box_noise_scale = 1.0 +dn_label_noise_ratio = 0.5 +dn_label_coef = 1.0 +dn_bbox_coef = 1.0 +embed_init_tgt = True +dn_labelbook_size = 2000 +max_text_len = 256 +text_encoder_type = "bert-base-uncased" +use_text_enhancer = True +use_fusion_layer = True +use_checkpoint = True +use_transformer_ckpt = True +use_text_cross_attention = True +text_dropout = 0.0 +fusion_dropout = 0.0 +fusion_droppath = 0.1 +sub_sentence_present = True diff --git a/requirements.txt b/requirements.txt index 4032d22..9417c1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -segment_anything \ No newline at end of file +segment_anything +groundingdino \ No newline at end of file diff --git a/scripts/dino.py b/scripts/dino.py index 8b5688d..a31ab44 100644 --- a/scripts/dino.py +++ b/scripts/dino.py @@ -7,10 +7,10 @@ from modules import scripts, shared from modules.devices import device, torch_gc, cpu # Grounding DINO -import GroundingDINO.groundingdino.datasets.transforms as T -from GroundingDINO.groundingdino.models import build_model -from GroundingDINO.groundingdino.util.slconfig import SLConfig -from GroundingDINO.groundingdino.util.utils import clean_state_dict +import groundingdino.datasets.transforms as T +from groundingdino.models import build_model +from groundingdino.util.slconfig import SLConfig +from groundingdino.util.utils import clean_state_dict dino_model_cache = OrderedDict() @@ -21,12 +21,12 @@ dino_model_info = { "repo_id": "ShilongLiu/GroundingDINO", "GroundingDINO_SwinT_OGC (694MB)": { "checkpoint": "groundingdino_swint_ogc.pth", - "config": os.path.join(scripts.basedir(), "GroundingDINO/groundingdino/config/GroundingDINO_SwinT_OGC.py"), - "url": "https://huggingface.co/ShilongLiu/GroundingDINO/resolve/main/groundingdino_swint_ogc.pth" + "config": os.path.join(dino_model_dir, "GroundingDINO_SwinT_OGC.py"), + "url": "https://huggingface.co/ShilongLiu/GroundingDINO/resolve/main/groundingdino_swint_ogc.pth", }, "GroundingDINO_SwinB (938MB)": { "checkpoint": "groundingdino_swinb_cogcoor.pth", - "config": os.path.join(scripts.basedir(), "GroundingDINO/groundingdino/config/GroundingDINO_SwinB.cfg.py"), + "config": os.path.join(dino_model_dir, "GroundingDINO_SwinB.cfg.py"), "url": "https://huggingface.co/ShilongLiu/GroundingDINO/resolve/main/groundingdino_swinb_cogcoor.pth" }, }