1580158
/
kohya_ss
mirror of https://github.com/bmaltais/kohya_ss
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #948 from bmaltais/dev2 

v21.7.6
pull/974/head v21.7.6
bmaltais 2023-06-07 16:24:51 -04:00 committed by GitHub
parent 9125fb0f65 797e2b4433
commit fa8fbe1ac1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 253 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Dockerfile
View File

@ -9,9 +9,10 @@ RUN add-apt-repository ppa:deadsnakes/ppa && \
python3.10-dev python3.10-tk python3-html5lib python3-apt python3-pip python3.10-distutils && \
rm -rf /var/lib/apt/lists/*
# Set python 3.10 as default
# Set python 3.10 and cuda 11.8 as default
RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 3 && \
update-alternatives --config python3
update-alternatives --set python3 /usr/bin/python3.10 && \
update-alternatives --set cuda /usr/local/cuda-11.8
RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3
@ -25,8 +26,8 @@ RUN python3 -m pip install wheel
## RUN python3 -m pip install -v -U git+https://github.com/facebookresearch/xformers.git@main#egg=xformers
# Install requirements
COPY requirements.txt setup.py ./
RUN python3 -m pip install --use-pep517 -r requirements.txt xformers
COPY requirements_unix.txt setup.py ./
RUN python3 -m pip install --use-pep517 -r requirements_unix.txt xformers
# Replace pillow with pillow-simd
RUN python3 -m pip uninstall -y pillow && \

8
README.md
View File

@ -345,7 +345,13 @@ This will store a backup file with your current locally installed pip packages a
## Change History
* 2023/07/05 (v21 7.5)
* 2023/06/06 (v21.7.6)
- Small UI improvements
- Fix `train_network.py` to probably work with older versions of LyCORIS.
- `gen_img_diffusers.py` now supports `BREAK` syntax.
- Add Lycoris iA3, LoKr and DyLoRA support to the UI
- Upgrade LuCORIS python module to 0.1.6
* 2023/06/05 (v21 7.5)
- Fix reported issue with LoHA: https://github.com/bmaltais/kohya_ss/issues/922
* 2023/06/05 (v21.7.4)
- Add manual accelerate config option

21
gen_img_diffusers.py
View File

@ -457,7 +457,7 @@ def replace_vae_modules(vae: diffusers.models.AutoencoderKL, mem_eff_attn, xform
upsampler.forward = make_replacer(upsampler)
"""
def replace_vae_attn_to_memory_efficient():
print("AttentionBlock.forward has been replaced to FlashAttention (not xformers)")
@ -1795,6 +1795,9 @@ def parse_prompt_attention(text):
for p in range(start_position, len(res)):
res[p][1] *= multiplier
# keep break as separate token
text = text.replace("BREAK", "\\BREAK\\")
for m in re_attention.finditer(text):
text = m.group(0)
weight = m.group(1)
@ -1826,7 +1829,7 @@ def parse_prompt_attention(text):
# merge runs of identical weights
i = 0
while i + 1 < len(res):
if res[i][1] == res[i + 1][1]:
if res[i][1] == res[i + 1][1] and res[i][0].strip() != "BREAK" and res[i + 1][0].strip() != "BREAK":
res[i][0] += res[i + 1][0]
res.pop(i + 1)
else:
@ -1843,11 +1846,25 @@ def get_prompts_with_weights(pipe: PipelineLike, prompt: List[str], max_length:
tokens = []
weights = []
truncated = False
for text in prompt:
texts_and_weights = parse_prompt_attention(text)
text_token = []
text_weight = []
for word, weight in texts_and_weights:
if word.strip() == "BREAK":
# pad until next multiple of tokenizer's max token length
pad_len = pipe.tokenizer.model_max_length - (len(text_token) % pipe.tokenizer.model_max_length)
print(f"BREAK pad_len: {pad_len}")
for i in range(pad_len):
# v2のときEOSをつけるべきかどうかわからないぜ
# if i == 0:
# text_token.append(pipe.tokenizer.eos_token_id)
# else:
text_token.append(pipe.tokenizer.pad_token_id)
text_weight.append(1.0)
continue
# tokenize and discard the starting and the ending token
token = pipe.tokenizer(word).input_ids[1:-1]

2
gui.sh
View File

@ -17,6 +17,6 @@ cd "$SCRIPT_DIR"
source "$SCRIPT_DIR/venv/bin/activate"
# If the requirements are validated, run the kohya_gui.py script with the command-line arguments
if python "$SCRIPT_DIR"/tools/validate_requirements.py -r "$SCRIPT_DIR"/requirements.txt; then
if python "$SCRIPT_DIR"/tools/validate_requirements.py -r "$SCRIPT_DIR"/requirements_unix.txt; then
python "$SCRIPT_DIR/kohya_gui.py" "$@"
fi

2
library/common_gui.py
View File

@ -952,7 +952,7 @@ def gradio_advanced_training(headless=False):
info='(Optional) Save only the specified number of models (old models will be deleted)',
)
save_last_n_steps_state = gr.Number(
label='Save last N steps',
label='Save last N states',
value=0,
precision=0,
info='(Optional) Save only the specified number of states (old models will be deleted)',

2
library/merge_lora_gui.py
View File

@ -104,7 +104,7 @@ def merge_lora(
def gradio_merge_lora_tab(headless=False):
with gr.Tab('Merge LoRA'):
gr.Markdown(
'This utility can merge up to 4 LoRA together or alternativelly merge up to 4 LoRA into a SD checkpoint.'
'This utility can merge up to 4 LoRA together or alternatively merge up to 4 LoRA into a SD checkpoint.'
)
lora_ext = gr.Textbox(value='*.safetensors *.pt', visible=False)

2
library/wd14_caption_gui.py
View File

@ -156,7 +156,7 @@ def gradio_wd14_caption_gui_tab(headless=False):
character_threshold = gr.Slider(
value=0.35,
label='Character threshold',
info='useful if you want to train with characte',
info='useful if you want to train with character',
minimum=0,
maximum=1,
step=0.05,

245
lora_gui.py
View File

@ -133,6 +133,10 @@ def save_configuration(
multires_noise_iterations,
multires_noise_discount,
LoRA_type,
factor,
use_cp,
decompose_both,
train_on_input,
conv_dim,
conv_alpha,
sample_every_n_steps,
@ -278,6 +282,10 @@ def open_configuration(
multires_noise_iterations,
multires_noise_discount,
LoRA_type,
factor,
use_cp,
decompose_both,
train_on_input,
conv_dim,
conv_alpha,
sample_every_n_steps,
@ -415,6 +423,10 @@ def train_model(
multires_noise_iterations,
multires_noise_discount,
LoRA_type,
factor,
use_cp,
decompose_both,
train_on_input,
conv_dim,
conv_alpha,
sample_every_n_steps,
@ -676,7 +688,49 @@ def train_model(
)
return
run_cmd += f' --network_module=lycoris.kohya'
run_cmd += f' --network_args "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "algo=loha"'
run_cmd += f' --network_args "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "use_cp={use_cp}" "algo=loha"'
# This is a hack to fix a train_network LoHA logic issue
if not network_dropout > 0.0:
run_cmd += f' --network_dropout="{network_dropout}"'
if LoRA_type == 'LyCORIS/iA3':
try:
import lycoris
except ModuleNotFoundError:
log.info(
"\033[1;31mError:\033[0m The required module 'lycoris_lora' is not installed. Please install by running \033[33mupgrade.ps1\033[0m before running this program."
)
return
run_cmd += f' --network_module=lycoris.kohya'
run_cmd += f' --network_args "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "train_on_input={train_on_input}" "algo=ia3"'
# This is a hack to fix a train_network LoHA logic issue
if not network_dropout > 0.0:
run_cmd += f' --network_dropout="{network_dropout}"'
if LoRA_type == 'LyCORIS/DyLoRA':
try:
import lycoris
except ModuleNotFoundError:
log.info(
"\033[1;31mError:\033[0m The required module 'lycoris_lora' is not installed. Please install by running \033[33mupgrade.ps1\033[0m before running this program."
)
return
run_cmd += f' --network_module=lycoris.kohya'
run_cmd += f' --network_args "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "use_cp={use_cp}" "block_size={unit}" "algo=dylora"'
# This is a hack to fix a train_network LoHA logic issue
if not network_dropout > 0.0:
run_cmd += f' --network_dropout="{network_dropout}"'
if LoRA_type == 'LyCORIS/LoKr':
try:
import lycoris
except ModuleNotFoundError:
log.info(
"\033[1;31mError:\033[0m The required module 'lycoris_lora' is not installed. Please install by running \033[33mupgrade.ps1\033[0m before running this program."
)
return
run_cmd += f' --network_module=lycoris.kohya'
run_cmd += f' --network_args "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "factor={factor}" "use_cp={use_cp}" "algo=lokr"'
# This is a hack to fix a train_network LoHA logic issue
if not network_dropout > 0.0:
run_cmd += f' --network_dropout="{network_dropout}"'
@ -1018,9 +1072,11 @@ def lora_tab(
choices=[
'Kohya DyLoRA',
'Kohya LoCon',
# 'LoCon',
'LyCORIS/DyLoRA',
'LyCORIS/iA3',
'LyCORIS/LoCon',
'LyCORIS/LoHa',
'LyCORIS/LoKr',
'Standard',
],
value='Standard',
@ -1081,6 +1137,30 @@ def lora_tab(
value='0.0001',
info='Optional',
)
with gr.Row():
factor = gr.Slider(
label='LoKr factor',
value=-1,
minimum=-1,
maximum=64,
step=1,
visible=False,
)
use_cp = gr.Checkbox(
value=False,
label='Use CP decomposition',
info='A two-step approach utilizing tensor decomposition and fine-tuning to accelerate convolution layers in large neural networks, resulting in significant CPU speedups with minor accuracy drops.',
)
decompose_both = gr.Checkbox(
value=False,
label='LoKr decompose both',
)
train_on_input = gr.Checkbox(
value=False,
label='iA3 train on input',
)
with gr.Row():
network_dim = gr.Slider(
minimum=1,
maximum=1024,
@ -1115,11 +1195,45 @@ def lora_tab(
step=0.1,
label='Convolution Alpha',
)
with gr.Row():
scale_weight_norms = gr.Slider(
label='Scale weight norms',
value=0,
minimum=0,
maximum=1,
step=0.01,
info='Max Norm Regularization is a technique to stabilize network training by limiting the norm of network weights. It may be effective in suppressing overfitting of LoRA and improving stability when used with other LoRAs. See PR for details.',
interactive=True,
)
network_dropout = gr.Slider(
label='Network dropout',
value=0,
minimum=0,
maximum=1,
step=0.01,
info='Is a normal probability dropout at the neuron level. In the case of LoRA, it is applied to the output of down. Recommended range 0.1 to 0.5',
)
rank_dropout = gr.Slider(
label='Rank dropout',
value=0,
minimum=0,
maximum=1,
step=0.01,
info='can specify `rank_dropout` to dropout each rank with specified probability. Recommended range 0.1 to 0.3',
)
module_dropout = gr.Slider(
label='Module dropout',
value=0.0,
minimum=0.0,
maximum=1.0,
step=0.01,
info='can specify `module_dropout` to dropout each rank with specified probability. Recommended range 0.1 to 0.3',
)
with gr.Row(visible=False) as kohya_dylora:
unit = gr.Slider(
minimum=1,
maximum=64,
label='DyLoRA Unit',
label='DyLoRA Unit / Block size',
value=1,
step=1,
interactive=True,
@ -1135,7 +1249,10 @@ def lora_tab(
'LoCon',
'Kohya DyLoRA',
'Kohya LoCon',
'LyCORIS/DyLoRA',
'LyCORIS/iA3',
'LyCORIS/LoHa',
'LyCORIS/LoKr',
'LyCORIS/LoCon',
}
@ -1154,8 +1271,65 @@ def lora_tab(
'Kohya LoCon',
}
# Determine if LyCORIS factor should be visible based on LoRA_type
LoKr_factor_visible = LoRA_type in {
'LyCORIS/LoKr',
}
# Determine if LyCORIS use_cp should be visible based on LoRA_type
use_cp_visible = LoRA_type in {
'LyCORIS/DyLoRA',
'LyCORIS/LoHa',
'LyCORIS/LoCon',
'LyCORIS/LoKr',
}
# Determine if LyCORIS decompose_both should be visible based on LoRA_type
LoKr_decompose_both_visible = LoRA_type in {
'LyCORIS/LoKr',
}
# Determine if scale_weight_norms should be visible based on LoRA_type
scale_weight_norms_visible = LoRA_type in {
'Kohya DyLoRA',
'Kohya LoCon',
}
# Determine if network_dropout should be visible based on LoRA_type
network_dropout_visible = LoRA_type in {
'LoCon',
'Kohya DyLoRA',
'Kohya LoCon',
'LyCORIS/DyLoRA',
'LyCORIS/LoHa',
'LyCORIS/LoCon',
'LyCORIS/LoKr',
}
# Determine if scale_weight_norms should be visible based on LoRA_type
rank_dropout_visible = LoRA_type in {
'LoCon',
'Kohya DyLoRA',
'Kohya LoCon',
}
# Determine if module_dropout should be visible based on LoRA_type
module_dropout_visible = LoRA_type in {
'LoCon',
'Kohya DyLoRA',
'Kohya LoCon',
}
# Determine if train_on_input should be visible based on LoRA_type
train_on_input_visible = LoRA_type in {
'LyCORIS/iA3',
}
# Determine if kohya_dylora_visible should be visible based on LoRA_type
kohya_dylora_visible = LoRA_type == 'Kohya DyLoRA'
kohya_dylora_visible = LoRA_type in {
'Kohya DyLoRA',
'LyCORIS/DyLoRA',
}
# Return the updated visibility settings for the groups
return (
@ -1165,6 +1339,14 @@ def lora_tab(
gr.Textbox.update(visible=LoRA_network_weights_visible),
gr.Button.update(visible=LoRA_network_weights_visible),
gr.Checkbox.update(visible=LoRA_network_weights_visible),
gr.Slider.update(visible=LoKr_factor_visible),
gr.Slider.update(visible=use_cp_visible),
gr.Slider.update(visible=LoKr_decompose_both_visible),
gr.Slider.update(visible=train_on_input_visible),
gr.Slider.update(visible=scale_weight_norms_visible),
gr.Slider.update(visible=network_dropout_visible),
gr.Slider.update(visible=rank_dropout_visible),
gr.Slider.update(visible=module_dropout_visible),
)
with gr.Row():
@ -1240,8 +1422,12 @@ def lora_tab(
no_token_padding = gr.Checkbox(
label='No token padding', value=False
)
gradient_accumulation_steps = gr.Number(
label='Gradient accumulate steps', value='1'
gradient_accumulation_steps = gr.Slider(
label='Gradient accumulate steps',
value='1',
minimum=1,
maximum=128,
step=1,
)
weighted_captions = gr.Checkbox(
label='Weighted captions',
@ -1261,40 +1447,7 @@ def lora_tab(
label='LR power',
placeholder='(Optional) For Cosine with restart and polynomial only',
)
with gr.Row():
scale_weight_norms = gr.Slider(
label='Scale weight norms',
value=0,
minimum=0,
maximum=1,
step=0.01,
info='Max Norm Regularization is a technique to stabilize network training by limiting the norm of network weights. It may be effective in suppressing overfitting of LoRA and improving stability when used with other LoRAs. See PR for details.',
interactive=True,
)
network_dropout = gr.Slider(
label='Network dropout',
value=0,
minimum=0,
maximum=1,
step=0.01,
info='Is a normal probability dropout at the neuron level. In the case of LoRA, it is applied to the output of down. Recommended range 0.1 to 0.5',
)
rank_dropout = gr.Slider(
label='Rank dropout',
value=0,
minimum=0,
maximum=1,
step=0.01,
info='can specify `rank_dropout` to dropout each rank with specified probability. Recommended range 0.1 to 0.3',
)
module_dropout = gr.Slider(
label='Module dropout',
value=0.0,
minimum=0.0,
maximum=1.0,
step=0.01,
info='can specify `module_dropout` to dropout each rank with specified probability. Recommended range 0.1 to 0.3',
)
(
# use_8bit_adam,
xformers,
@ -1355,6 +1508,14 @@ def lora_tab(
lora_network_weights,
lora_network_weights_file,
dim_from_weights,
factor,
use_cp,
decompose_both,
train_on_input,
scale_weight_norms,
network_dropout,
rank_dropout,
module_dropout,
],
)
@ -1461,6 +1622,10 @@ def lora_tab(
multires_noise_iterations,
multires_noise_discount,
LoRA_type,
factor,
use_cp,
decompose_both,
train_on_input,
conv_dim,
conv_alpha,
sample_every_n_steps,

4
networks/lora.py
View File

@ -402,7 +402,7 @@ def parse_block_lr_kwargs(nw_kwargs):
return down_lr_weight, mid_lr_weight, up_lr_weight
def create_network(multiplier, network_dim, network_alpha, vae, text_encoder, unet, dropout=None, **kwargs):
def create_network(multiplier, network_dim, network_alpha, vae, text_encoder, unet, neuron_dropout=None, **kwargs):
if network_dim is None:
network_dim = 4 # default
if network_alpha is None:
@ -457,7 +457,7 @@ def create_network(multiplier, network_dim, network_alpha, vae, text_encoder, un
multiplier=multiplier,
lora_dim=network_dim,
alpha=network_alpha,
dropout=dropout,
dropout=neuron_dropout,
rank_dropout=rank_dropout,
module_dropout=module_dropout,
conv_lora_dim=conv_dim,

2
requirements.txt → requirements_unix.txt
View File

@ -13,7 +13,7 @@ gradio==3.32.0; sys_platform != 'darwin'
huggingface-hub==0.13.0; sys_platform == 'darwin'
huggingface-hub==0.13.3; sys_platform != 'darwin'
lion-pytorch==0.0.6
lycoris_lora==0.1.4
lycoris_lora==0.1.6
opencv-python==4.7.0.68
pytorch-lightning==1.9.0
rich==13.4.1

2
requirements_windows_torch1.txt
View File

@ -11,7 +11,7 @@ ftfy==6.1.1
gradio==3.32.0
huggingface-hub==0.13.3
lion-pytorch==0.0.6
lycoris_lora==0.1.4
lycoris_lora==0.1.6
opencv-python==4.7.0.68
pytorch-lightning==1.9.0
rich==13.4.1

2
requirements_windows_torch2.txt
View File

@ -11,7 +11,7 @@ ftfy==6.1.1
gradio==3.33.1
huggingface-hub==0.15.1
lion-pytorch==0.0.6
lycoris_lora==0.1.4
lycoris_lora==0.1.6
opencv-python==4.7.0.68
pytorch-lightning==1.9.0
rich==13.4.1

10
setup.sh
View File

@ -3,7 +3,7 @@
# This file will be the host environment setup file for all operating systems other than base Windows.
# Set the required package versions here.
# They will be appended to the requirements.txt file in the installation directory.
# They will be appended to the requirements_unix.txt file in the installation directory.
TENSORFLOW_VERSION="2.12.0"
TENSORFLOW_MACOS_VERSION="2.12.0"
TENSORFLOW_METAL_VERSION="0.8.0"
@ -262,14 +262,14 @@ install_python_dependencies() {
# DEBUG ONLY (Update this version number to whatever PyCharm recommends)
# pip install pydevd-pycharm~=223.8836.43
#This will copy our requirements.txt file out and make the khoya_ss lib a dynamic location then cleanup.
#This will copy our requirements_unix.txt file out and make the khoya_ss lib a dynamic location then cleanup.
local TEMP_REQUIREMENTS_FILE="$DIR/requirements_tmp_for_setup.txt"
echo "Copying $DIR/requirements.txt to $TEMP_REQUIREMENTS_FILE" >&3
echo "Copying $DIR/requirements_unix.txt to $TEMP_REQUIREMENTS_FILE" >&3
echo "Replacing the . for lib to our DIR variable in $TEMP_REQUIREMENTS_FILE." >&3
awk -v dir="$DIR" '/#.*kohya_ss.*library/{print; getline; sub(/^\.$/, dir)}1' "$DIR/requirements.txt" >"$TEMP_REQUIREMENTS_FILE"
awk -v dir="$DIR" '/#.*kohya_ss.*library/{print; getline; sub(/^\.$/, dir)}1' "$DIR/requirements_unix.txt" >"$TEMP_REQUIREMENTS_FILE"
# This will check if macOS is running then determine if M1+ or Intel CPU.
# It will append the appropriate packages to the requirements.txt file.
# It will append the appropriate packages to the requirements_unix.txt file.
# Other OSs won't be affected and the version variables are at the top of this file.
if [[ "$(uname)" == "Darwin" ]]; then
# Check if the processor is Apple Silicon (arm64)

1
tools/validate_requirements.py
View File

@ -86,7 +86,6 @@ def main():
'-r',
'--requirements',
type=str,
default='requirements.txt',
help='Path to the requirements file.',
)
parser.add_argument('--debug', action='store_true', help='Debug on')

4
train_network.py
View File

@ -212,7 +212,7 @@ def train(args):
else:
# LyCORIS will work with this...
network = network_module.create_network(
1.0, args.network_dim, args.network_alpha, vae, text_encoder, unet, dropout=args.network_dropout, **net_kwargs
1.0, args.network_dim, args.network_alpha, vae, text_encoder, unet, neuron_dropout=args.network_dropout, **net_kwargs
)
if network is None:
return
@ -724,7 +724,7 @@ def train(args):
progress_bar.set_postfix(**logs)
if args.scale_weight_norms:
progress_bar.set_postfix(**max_mean_logs)
progress_bar.set_postfix(**{**max_mean_logs, **logs})
if args.logging_dir is not None:
logs = generate_step_logs(args, current_loss, avr_loss, lr_scheduler, keys_scaled, mean_norm, maximum_norm)

2
upgrade.sh
View File

@ -13,4 +13,4 @@ git pull
source venv/bin/activate
# Upgrade the required packages
pip install --use-pep517 --upgrade -r requirements.txt
pip install --use-pep517 --upgrade -r requirements_unix.txt