1580158
/
sdweb-merge-board
mirror of https://github.com/bbc-mc/sdweb-merge-board.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add support of "Merge Log" 

feature: add log support
fix: small fixes
master
bbc_mc 2022-12-28 01:30:00 +09:00
parent 9aec9ef27b
commit d145aeddb4
7 changed files with 179 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,2 +1,3 @@
/csv/history.tsv
#
_*

28
README.md
View File

@ -1,11 +1,30 @@
# Marge Board
| Multi Merge | Merge Recipe (Import/Export) | Checkpoint List |
| ------------------ | ---------------------------- | ------------------ |
| ![](misc/ss01.png) | ![](misc/ss02.png) | ![](misc/ss03.png) |
| [Multi-Merge](## Multi-Marge) | [Merge Recipe](## Recipe) (Import/Export) | Checkpoint List |
| ----------------------------- | ----------------------------------------- | ------------------ |
| ![](misc/ss01.png) | ![](misc/ss02.png) | ![](misc/ss03.png) |
- This is Extension for [AUTOMATIC1111's Stable Diffusion Web UI](https://github.com/AUTOMATIC1111/stable-diffusion-webui)
##
## Features
- **Multiple step marge** support ( up to 10 step)
- Save and Load your merging combination as `Recipe`, which is simple text.
## Recent Update
2022/12/28
- Add support of **Log file** for keep record of your merges.
- you can find your logfile on `(extension's dir)/csv/history.tsv`
## How to Install
- Go to `Extensions` tab on your web UI
@ -14,10 +33,7 @@
- Install
## Features
- **Multiple step marge** support ( up to 10 step)
- Save and Load your merging combination as `Recipe`, which is simple text.
## Multi-Marge

2
scripts/merge_board.py
View File

@ -172,7 +172,7 @@ def on_ui_tabs():
def reload_checkpoints():
sd_models.list_models()
return [gr.update(choices=ui_merge.get_choise_of_models_with_vars(), value="") for i in range(len(_checkpoint_listener))]
return [gr.update(choices=ui_merge.get_choise_of_models_with_vars(i//3+1), value="") for i in range(len(_checkpoint_listener))]
btn_reload_checkpoints.click(
fn=reload_checkpoints,
inputs=[],

2
scripts/multimerge/operation.py
View File

@ -7,7 +7,7 @@ class MergeOperation:
def can_process(self):
_ret = True
for _recipe in self.recipes.values():
for _index, _recipe in self.recipes.items():
_recipe:MergeRecipe = _recipe
_ret = _ret and _recipe.can_process()
return _ret

68
scripts/multimerge/recipe.py
View File

@ -1,14 +1,19 @@
import math
import os
import sys
import re
from modules import sd_models, extras, shared
from scripts.multimerge.util.merge_history import MergeHistory
S_WS = "Weighted sum"
S_AD = "Add difference"
S_SG = "Sigmoid"
choise_of_method = [S_WS, S_AD, S_SG]
mergeHistory = MergeHistory()
class MergeRecipe():
def __init__(self, A, B, C, O, M, S, F:bool, CF):
@ -28,7 +33,7 @@ class MergeRecipe():
self.A = A
self.B = B
self.C = C
self.O = O
self.O = re.sub(r'[\\|/|:|?|.|"|<|>|\|\*]', '-', O)
self.S = self._adjust_method(method=S, model_C=C)
self.M = self._adjust_multi_by_method(method=S, multi=M)
self.F = self.row_F
@ -36,11 +41,16 @@ class MergeRecipe():
self.vars = {} # runtime variables
def can_process(self):
def can_process(self, index=0):
if self.A == "" or self.B == "" or self.A == None or self.B == None:
return False
if (self.C == "" or self.C == None) and self.S == S_AD:
return False
if index > 0:
# invalid var check
# __O3__, line=4 => ok
# __O3__, line=2 => error
pass
return True
def apply_variables(self, _vars:dict):
@ -58,13 +68,15 @@ class MergeRecipe():
def run_merge(self, index, skip_merge_if_exists):
sd_models.list_models()
if skip_merge_if_exists and self._check_ckpt_exists():
if skip_merge_if_exists:
_filename = self.O + "." + self.CF if self.O != "" else self._estimate_ckpt_name()
_result = f"Checkpoint already exist: {_filename}"
print(f"Merge skipped. Same name checkpoint already exists.")
print(f" O: {_filename}")
self._update_o_filename(index, _result)
return [f"[skipped] {_filename}", f"[skipped] {_filename}"]
if self._check_ckpt_exists(_filename):
# exists
_result = f"Checkpoint already exist: {_filename}"
print(f"Merge skipped. Same name checkpoint already exists.")
print(f" O: {_filename}")
self._update_o_filename(index, _result)
return [f"[skipped] {_filename}", f"[skipped] {_filename}"]
print( "Starting merge under settings below,")
print( " A: {}".format(f"{self.A}" if self.A == self.row_A else f"{self.row_A} -> {self.A}"))
@ -107,15 +119,41 @@ class MergeRecipe():
print(e)
return ["Error", "Error"]
except Exception as e:
print("Error loading/saving model file:", file=sys.stderr)
print(type(e))
print(e)
print("Error: at recipe.run_merge: ", file=sys.stderr)
print(type(e), file=sys.stderr)
print(e, file=sys.stderr)
sd_models.list_models() # to remove the potentially missing models from the list
return ["Error: loading/saving model file. It doesn't exist or the name contains illegal characters"] *2
# try to figure out whats going on
def _dprint_model_exists(header, model):
if model != "" and sd_models.get_closet_checkpoint_match(model) is not None:
if os.path.exists(sd_models.get_closet_checkpoint_match(model).filename):
print(" {}: is exists:True [{}]".format(header, model), file=sys.stderr)
else:
print(" {}: is exists:False [{}]".format(header, model), file=sys.stderr)
else:
print(" {}: not found: [{}]".format(header, model), file=sys.stderr)
_dprint_model_exists("A", self.A)
_dprint_model_exists("B", self.B)
_dprint_model_exists("C", self.C)
return ["Error: at recipe.run_merge. "] *2
# update vars
self._update_o_filename(index, results[0])
# save log
mergeHistory.add_history(
self.A,
self.B,
self.C,
self.S,
self.M,
self.F,
self.O,
self.CF,
index)
#
return [f"Merge complete. Checkpoint saved as: [{self.O}]", self.O]
@ -168,11 +206,7 @@ class MergeRecipe():
alpha = float(alpha)
return 0.5 - math.sin(math.asin(1.0 - 2.0 * alpha) / 3.0)
def _check_ckpt_exists(self):
if self.O == "":
_O = self._estimate_ckpt_name()
else:
_O = self.O + "." + self.CF
def _check_ckpt_exists(self, _O):
ckpt_dir = shared.cmd_opts.ckpt_dir or sd_models.model_path
output_modelname = os.path.join(ckpt_dir, _O)
if os.path.exists(ckpt_dir) and os.path.exists(output_modelname) and os.path.isfile(output_modelname):

65
scripts/multimerge/ui_merge.py
View File

@ -7,13 +7,12 @@ from scripts.multimerge.recipe import S_WS, S_AD, S_SG, choise_of_method
Variables_Output = ["__O1__", "__O2__", "__O3__", "__O4__", "__O5__", "__O6__", "__O7__", "__O8__", "__O9__", "__O10__"]
def get_choise_of_models_with_vars():
return Variables_Output + sd_models.checkpoint_tiles()
def get_choise_of_models_with_vars(current_line=10):
return Variables_Output[:current_line-1] + sd_models.checkpoint_tiles()
def on_ui_tabs():
choise_of_models = get_choise_of_models_with_vars()
_checkpoint_listener = []
with gr.Tab("Multi Merge"):
@ -37,9 +36,9 @@ def on_ui_tabs():
with gr.Column():
with gr.Row(variant="panel"):
_line_number = 1
A1 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(A{_line_number}) Primary")
B1 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(B{_line_number}) Secondary")
C1 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(C{_line_number}) Thertiary")
A1 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(A{_line_number}) Primary")
B1 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(B{_line_number}) Secondary")
C1 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(C{_line_number}) Thertiary")
O1 = gr.Textbox(label=f"(O{_line_number}) Output ckpt Name", interactive=True)
M1 = gr.Slider(minimum=0.0, maximum=1.0, step=0.001, label=f'(M{_line_number}) Multiplier', value=0.5, interactive=True)
S1 = gr.Dropdown(choices=choise_of_method, interactive=True, value=choise_of_method[0], label=f"(S{_line_number}) Inter-Method")
@ -50,9 +49,9 @@ def on_ui_tabs():
with gr.Column():
with gr.Row(variant="panel"):
_line_number = 2
A2 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(A{_line_number}) Primary")
B2 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(B{_line_number}) Secondary")
C2 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(C{_line_number}) Thertiary")
A2 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(A{_line_number}) Primary")
B2 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(B{_line_number}) Secondary")
C2 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(C{_line_number}) Thertiary")
O2 = gr.Textbox(label=f"(O{_line_number}) Output ckpt Name", interactive=True)
M2 = gr.Slider(minimum=0.0, maximum=1.0, step=0.001, label=f'(M{_line_number}) Multiplier', value=0.5, interactive=True)
S2 = gr.Dropdown(choices=choise_of_method, interactive=True, value=choise_of_method[0], label=f"(S{_line_number}) Inter-Method")
@ -63,9 +62,9 @@ def on_ui_tabs():
with gr.Column():
with gr.Row(variant="panel"):
_line_number = 3
A3 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(A{_line_number}) Primary")
B3 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(B{_line_number}) Secondary")
C3 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(C{_line_number}) Thertiary")
A3 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(A{_line_number}) Primary")
B3 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(B{_line_number}) Secondary")
C3 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(C{_line_number}) Thertiary")
O3 = gr.Textbox(label=f"(O{_line_number}) Output ckpt Name", interactive=True)
M3 = gr.Slider(minimum=0.0, maximum=1.0, step=0.001, label=f'(M{_line_number}) Multiplier', value=0.5, interactive=True)
S3 = gr.Dropdown(choices=choise_of_method, interactive=True, value=choise_of_method[0], label=f"(S{_line_number}) Inter-Method")
@ -76,9 +75,9 @@ def on_ui_tabs():
with gr.Column():
with gr.Row(variant="panel"):
_line_number = 4
A4 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(A{_line_number}) Primary")
B4 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(B{_line_number}) Secondary")
C4 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(C{_line_number}) Thertiary")
A4 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(A{_line_number}) Primary")
B4 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(B{_line_number}) Secondary")
C4 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(C{_line_number}) Thertiary")
O4 = gr.Textbox(label=f"(O{_line_number}) Output ckpt Name", interactive=True)
M4 = gr.Slider(minimum=0.0, maximum=1.0, step=0.001, label=f'(M{_line_number}) Multiplier', value=0.5, interactive=True)
S4 = gr.Dropdown(choices=choise_of_method, interactive=True, value=choise_of_method[0], label=f"(S{_line_number}) Inter-Method")
@ -89,9 +88,9 @@ def on_ui_tabs():
with gr.Column():
with gr.Row(variant="panel"):
_line_number = 5
A5 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(A{_line_number}) Primary")
B5 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(B{_line_number}) Secondary")
C5 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(C{_line_number}) Thertiary")
A5 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(A{_line_number}) Primary")
B5 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(B{_line_number}) Secondary")
C5 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(C{_line_number}) Thertiary")
O5 = gr.Textbox(label=f"(O{_line_number}) Output ckpt Name", interactive=True)
M5 = gr.Slider(minimum=0.0, maximum=1.0, step=0.001, label=f'(M{_line_number}) Multiplier', value=0.5, interactive=True)
S5 = gr.Dropdown(choices=choise_of_method, interactive=True, value=choise_of_method[0], label=f"(S{_line_number}) Inter-Method")
@ -102,9 +101,9 @@ def on_ui_tabs():
with gr.Column():
with gr.Row(variant="panel"):
_line_number = 6
A6 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(A{_line_number}) Primary")
B6 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(B{_line_number}) Secondary")
C6 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(C{_line_number}) Thertiary")
A6 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(A{_line_number}) Primary")
B6 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(B{_line_number}) Secondary")
C6 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(C{_line_number}) Thertiary")
O6 = gr.Textbox(label=f"(O{_line_number}) Output ckpt Name", interactive=True)
M6 = gr.Slider(minimum=0.0, maximum=1.0, step=0.001, label=f'(M{_line_number}) Multiplier', value=0.5, interactive=True)
S6 = gr.Dropdown(choices=choise_of_method, interactive=True, value=choise_of_method[0], label=f"(S{_line_number}) Inter-Method")
@ -115,9 +114,9 @@ def on_ui_tabs():
with gr.Column():
with gr.Row(variant="panel"):
_line_number = 7
A7 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(A{_line_number}) Primary")
B7 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(B{_line_number}) Secondary")
C7 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(C{_line_number}) Thertiary")
A7 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(A{_line_number}) Primary")
B7 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(B{_line_number}) Secondary")
C7 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(C{_line_number}) Thertiary")
O7 = gr.Textbox(label=f"(O{_line_number}) Output ckpt Name", interactive=True)
M7 = gr.Slider(minimum=0.0, maximum=1.0, step=0.001, label=f'(M{_line_number}) Multiplier', value=0.5, interactive=True)
S7 = gr.Dropdown(choices=choise_of_method, interactive=True, value=choise_of_method[0], label=f"(S{_line_number}) Inter-Method")
@ -128,9 +127,9 @@ def on_ui_tabs():
with gr.Column():
with gr.Row(variant="panel"):
_line_number = 8
A8 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(A{_line_number}) Primary")
B8 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(B{_line_number}) Secondary")
C8 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(C{_line_number}) Thertiary")
A8 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(A{_line_number}) Primary")
B8 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(B{_line_number}) Secondary")
C8 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(C{_line_number}) Thertiary")
O8 = gr.Textbox(label=f"(O{_line_number}) Output ckpt Name", interactive=True)
M8 = gr.Slider(minimum=0.0, maximum=1.0, step=0.001, label=f'(M{_line_number}) Multiplier', value=0.5, interactive=True)
S8 = gr.Dropdown(choices=choise_of_method, interactive=True, value=choise_of_method[0], label=f"(S{_line_number}) Inter-Method")
@ -141,9 +140,9 @@ def on_ui_tabs():
with gr.Column():
with gr.Row(variant="panel"):
_line_number = 9
A9 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(A{_line_number}) Primary")
B9 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(B{_line_number}) Secondary")
C9 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(C{_line_number}) Thertiary")
A9 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(A{_line_number}) Primary")
B9 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(B{_line_number}) Secondary")
C9 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(C{_line_number}) Thertiary")
O9 = gr.Textbox(label=f"(O{_line_number}) Output ckpt Name", interactive=True)
M9 = gr.Slider(minimum=0.0, maximum=1.0, step=0.001, label=f'(M{_line_number}) Multiplier', value=0.5, interactive=True)
S9 = gr.Dropdown(choices=choise_of_method, interactive=True, value=choise_of_method[0], label=f"(S{_line_number}) Inter-Method")
@ -154,9 +153,9 @@ def on_ui_tabs():
with gr.Column():
with gr.Row(variant="panel"):
_line_number = 10
A10 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(A{_line_number}) Primary")
B10 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(B{_line_number}) Secondary")
C10 = gr.Dropdown(choices=choise_of_models, interactive=True, label=f"(C{_line_number}) Thertiary")
A10 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(A{_line_number}) Primary")
B10 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(B{_line_number}) Secondary")
C10 = gr.Dropdown(choices=get_choise_of_models_with_vars(_line_number), interactive=True, label=f"(C{_line_number}) Thertiary")
O10 = gr.Textbox(label=f"(O{_line_number}) Output ckpt Name", interactive=True)
M10 = gr.Slider(minimum=0.0, maximum=1.0, step=0.001, label=f'(M{_line_number}) Multiplier', value=0.5, interactive=True)
S10 = gr.Dropdown(choices=choise_of_method, interactive=True, value=choise_of_method[0], label=f"(S{_line_number}) Inter-Method")

71
scripts/multimerge/util/merge_history.py Normal file
View File

@ -0,0 +1,71 @@
#
#
#
import os
import datetime
from csv import DictWriter, DictReader
from modules import scripts, sd_models
CSV_FILE_PATH = "csv/history.tsv"
HEADERS = [
"model_O", "model_O_hash", "model_A", "model_A_hash", "model_B", "model_B_hash", "model_C", "model_C_hash",
"multiplier", "interpolation_method", "save_as_float16", "checkpoint_format", "lane_index", "datetime",
]
path_root = scripts.basedir()
class MergeHistory():
def __init__(self):
self.filepath = os.path.join(path_root, CSV_FILE_PATH)
if os.path.exists(self.filepath):
self.update_header()
def add_history(self, A, B, C, S, M, F, O, CF, index):
def _update_model_data(_dict, M, header=""):
if M is not None and M != "" and header != "":
_m = sd_models.get_closet_checkpoint_match(M)
_dict.update({
f"model_{header}": os.path.basename(_m.filename),
f"model_{header}_hash": _m.hash
})
_history_dict = {}
_update_model_data(_history_dict, A, "A")
_update_model_data(_history_dict, B, "B")
_update_model_data(_history_dict, C, "C")
_update_model_data(_history_dict, O, "O")
_history_dict.update({"interpolation_method": f"{S}"})
_history_dict.update({"multiplier": f"{M}"})
_history_dict.update({"save_as_float16": f"{F}"})
_history_dict.update({"checkpoint_format": f"{CF}"})
_history_dict.update({"lane_index": index})
_history_dict.update({"datetime": f"{datetime.datetime.now()}"})
if not os.path.exists(self.filepath):
with open(self.filepath, "w", newline="", encoding="utf-8") as f:
dw = DictWriter(f, fieldnames=HEADERS, delimiter='\t')
dw.writeheader()
# save to file
with open(self.filepath, "a", newline="", encoding='utf-8') as f:
dw = DictWriter(f, fieldnames=HEADERS, delimiter='\t')
dw.writerow(_history_dict)
def update_header(self):
hist_data = []
if os.path.exists(self.filepath):
# check header in case HEADERS updated
with open(self.filepath, "r", newline="", encoding="utf-8") as f:
dr = DictReader(f, delimiter='\t')
if dr.fieldnames:
new_header = [ x for x in HEADERS if x not in dr.fieldnames ]
if len(new_header) > 0:
# need update.
hist_data = [ x for x in dr]
if len(hist_data) > 0:
with open(self.filepath, "w", newline="", encoding="utf-8") as f:
dw = DictWriter(f, fieldnames=HEADERS, delimiter='\t')
dw.writeheader()
dw.writerows(hist_data)