1580158
/
kohya_ss
mirror of https://github.com/bmaltais/kohya_ss
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add max_train_steps and lr_scheduler_args

pull/1368/head
bmaltais 2023-08-09 20:04:04 -04:00
parent 254069ce5f
commit 78c14bd46f
20 changed files with 185 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
dreambooth_gui.py
View File

@ -98,7 +98,7 @@ def save_configuration(
vae,
output_name,
max_token_length,
max_train_epochs,
max_train_epochs,max_train_steps,
max_data_loader_n_workers,
mem_eff_attn,
gradient_accumulation_steps,
@ -113,7 +113,7 @@ def save_configuration(
caption_dropout_every_n_epochs,
caption_dropout_rate,
optimizer,
optimizer_args,
optimizer_args, lr_scheduler_args,
noise_offset_type,
noise_offset,
adaptive_noise_scale,
@ -212,7 +212,7 @@ def open_configuration(
vae,
output_name,
max_token_length,
max_train_epochs,
max_train_epochs,max_train_steps,
max_data_loader_n_workers,
mem_eff_attn,
gradient_accumulation_steps,
@ -227,7 +227,7 @@ def open_configuration(
caption_dropout_every_n_epochs,
caption_dropout_rate,
optimizer,
optimizer_args,
optimizer_args, lr_scheduler_args,
noise_offset_type,
noise_offset,
adaptive_noise_scale,
@ -325,7 +325,7 @@ def train_model(
vae,
output_name,
max_token_length,
max_train_epochs,
max_train_epochs,max_train_steps,
max_data_loader_n_workers,
mem_eff_attn,
gradient_accumulation_steps,
@ -340,7 +340,7 @@ def train_model(
caption_dropout_every_n_epochs,
caption_dropout_rate,
optimizer,
optimizer_args,
optimizer_args, lr_scheduler_args,
noise_offset_type,
noise_offset,
adaptive_noise_scale,
@ -498,17 +498,20 @@ def train_model(
)
reg_factor = 2
# calculate max_train_steps
max_train_steps = int(
math.ceil(
float(total_steps)
/ int(train_batch_size)
/ int(gradient_accumulation_steps)
* int(epoch)
* int(reg_factor)
if max_train_steps == '' or max_train_steps == '0':
# calculate max_train_steps
max_train_steps = int(
math.ceil(
float(total_steps)
/ int(train_batch_size)
/ int(gradient_accumulation_steps)
* int(epoch)
* int(reg_factor)
)
)
log.info(
f'max_train_steps ({total_steps} / {train_batch_size} / {gradient_accumulation_steps} * {epoch} * {reg_factor}) = {max_train_steps}'
)
)
log.info(f'max_train_steps = {max_train_steps}')
# calculate stop encoder training
if int(stop_text_encoder_training_pct) == -1:
@ -599,6 +602,7 @@ def train_model(
cache_latents_to_disk=cache_latents_to_disk,
optimizer=optimizer,
optimizer_args=optimizer_args,
lr_scheduler_args=lr_scheduler_args,
)
run_cmd += run_cmd_advanced_training(
@ -802,6 +806,7 @@ def dreambooth_tab(
folders.output_name,
advanced_training.max_token_length,
basic_training.max_train_epochs,
basic_training.max_train_steps,
advanced_training.max_data_loader_n_workers,
advanced_training.mem_eff_attn,
advanced_training.gradient_accumulation_steps,
@ -818,6 +823,7 @@ def dreambooth_tab(
advanced_training.caption_dropout_rate,
basic_training.optimizer,
basic_training.optimizer_args,
basic_training.lr_scheduler_args,
advanced_training.noise_offset_type,
advanced_training.noise_offset,
advanced_training.adaptive_noise_scale,

8
finetune_gui.py
View File

@ -114,7 +114,7 @@ def save_configuration(
caption_dropout_every_n_epochs,
caption_dropout_rate,
optimizer,
optimizer_args,
optimizer_args, lr_scheduler_args,
noise_offset_type,
noise_offset,
adaptive_noise_scale,
@ -234,7 +234,7 @@ def open_configuration(
caption_dropout_every_n_epochs,
caption_dropout_rate,
optimizer,
optimizer_args,
optimizer_args, lr_scheduler_args,
noise_offset_type,
noise_offset,
adaptive_noise_scale,
@ -351,7 +351,7 @@ def train_model(
caption_dropout_every_n_epochs,
caption_dropout_rate,
optimizer,
optimizer_args,
optimizer_args, lr_scheduler_args,
noise_offset_type,
noise_offset,
adaptive_noise_scale,
@ -562,6 +562,7 @@ def train_model(
cache_latents_to_disk=cache_latents_to_disk,
optimizer=optimizer,
optimizer_args=optimizer_args,
lr_scheduler_args=lr_scheduler_args,
)
run_cmd += run_cmd_advanced_training(
@ -897,6 +898,7 @@ def finetune_tab(headless=False):
advanced_training.caption_dropout_rate,
basic_training.optimizer,
basic_training.optimizer_args,
basic_training.lr_scheduler_args,
advanced_training.noise_offset_type,
advanced_training.noise_offset,
advanced_training.adaptive_noise_scale,

1
library/class_advanced_training.py
View File

@ -36,6 +36,7 @@ class AdvancedTraining:
self.vae = gr.Textbox(
label='VAE',
placeholder='(Optional) path to checkpoint of vae to replace for training',
interactive=True
)
self.vae_button = gr.Button(
'📂', elem_id='open_folder_small', visible=(not headless)

39
library/class_basic_training.py
View File

@ -27,6 +27,10 @@ class BasicTraining:
label='Max train epoch',
placeholder='(Optional) Enforce number of epoch',
)
self.max_train_steps = gr.Textbox(
label='Max train steps',
placeholder='(Optional) Enforce number of steps',
)
self.save_every_n_epochs = gr.Number(
label='Save every N epochs', value=1, precision=0
)
@ -68,9 +72,6 @@ class BasicTraining:
label='Cache latents to disk', value=False
)
with gr.Row():
self.learning_rate = gr.Number(
label='Learning rate', value=learning_rate_value
)
self.lr_scheduler = gr.Dropdown(
label='LR Scheduler',
choices=[
@ -84,13 +85,6 @@ class BasicTraining:
],
value=lr_scheduler_value,
)
self.lr_warmup = gr.Slider(
label='LR warmup (% of steps)',
value=lr_warmup_value,
minimum=0,
maximum=100,
step=1,
)
self.optimizer = gr.Dropdown(
label='Optimizer',
choices=[
@ -116,6 +110,26 @@ class BasicTraining:
value='AdamW8bit',
interactive=True,
)
with gr.Row():
self.lr_scheduler_args = gr.Textbox(
label='LR scheduler extra arguments',
placeholder='(Optional) eg: "lr_end=5e-5"',
)
self.optimizer_args = gr.Textbox(
label='Optimizer extra arguments',
placeholder='(Optional) eg: relative_step=True scale_parameter=True warmup_init=True',
)
with gr.Row():
self.learning_rate = gr.Number(
label='Learning rate', value=learning_rate_value
)
self.lr_warmup = gr.Slider(
label='LR warmup (% of steps)',
value=lr_warmup_value,
minimum=0,
maximum=100,
step=1,
)
with gr.Row(visible=not finetuning):
self.lr_scheduler_num_cycles = gr.Textbox(
label='LR number of cycles',
@ -126,11 +140,6 @@ class BasicTraining:
label='LR power',
placeholder='(Optional) For Cosine with restart and polynomial only',
)
with gr.Row():
self.optimizer_args = gr.Textbox(
label='Optimizer extra arguments',
placeholder='(Optional) eg: relative_step=True scale_parameter=True warmup_init=True',
)
with gr.Row(visible=not finetuning):
self.max_resolution = gr.Textbox(
label='Max resolution',

3
library/common_gui.py
View File

@ -643,6 +643,9 @@ def run_cmd_training(**kwargs):
if optimizer_args != '':
run_cmd += f' --optimizer_args {optimizer_args}'
lr_scheduler_args = kwargs.get("lr_scheduler_args", "")
if lr_scheduler_args != '':
run_cmd += f' --lr_scheduler_args {optimizer_args}'
return run_cmd

42
lora_gui.py
View File

@ -111,6 +111,7 @@ def save_configuration(
model_list,
max_token_length,
max_train_epochs,
max_train_steps,
max_data_loader_n_workers,
network_alpha,
training_comment,
@ -124,7 +125,7 @@ def save_configuration(
caption_dropout_every_n_epochs,
caption_dropout_rate,
optimizer,
optimizer_args,
optimizer_args, lr_scheduler_args,
noise_offset_type,
noise_offset,
adaptive_noise_scale,
@ -256,6 +257,7 @@ def open_configuration(
model_list,
max_token_length,
max_train_epochs,
max_train_steps,
max_data_loader_n_workers,
network_alpha,
training_comment,
@ -265,11 +267,11 @@ def open_configuration(
persistent_data_loader_workers,
bucket_no_upscale,
random_crop,
bucket_reso_steps,v_pred_like_loss,
bucket_reso_steps, v_pred_like_loss,
caption_dropout_every_n_epochs,
caption_dropout_rate,
optimizer,
optimizer_args,
optimizer_args, lr_scheduler_args,
noise_offset_type,
noise_offset,
adaptive_noise_scale,
@ -426,7 +428,7 @@ def train_model(
output_name,
model_list, # Keep this. Yes, it is unused here but required given the common list used
max_token_length,
max_train_epochs,
max_train_epochs, max_train_steps,
max_data_loader_n_workers,
network_alpha,
training_comment,
@ -436,11 +438,11 @@ def train_model(
persistent_data_loader_workers,
bucket_no_upscale,
random_crop,
bucket_reso_steps,v_pred_like_loss,
bucket_reso_steps, v_pred_like_loss,
caption_dropout_every_n_epochs,
caption_dropout_rate,
optimizer,
optimizer_args,
optimizer_args, lr_scheduler_args,
noise_offset_type,
noise_offset,
adaptive_noise_scale,
@ -650,19 +652,20 @@ def train_model(
log.info(f'Epoch: {epoch}')
log.info(f'Regulatization factor: {reg_factor}')
# calculate max_train_steps
max_train_steps = int(
math.ceil(
float(total_steps)
/ int(train_batch_size)
/ int(gradient_accumulation_steps)
* int(epoch)
* int(reg_factor)
if max_train_steps == '' or max_train_steps == '0':
# calculate max_train_steps
max_train_steps = int(
math.ceil(
float(total_steps)
/ int(train_batch_size)
/ int(gradient_accumulation_steps)
* int(epoch)
* int(reg_factor)
)
)
log.info(
f'max_train_steps ({total_steps} / {train_batch_size} / {gradient_accumulation_steps} * {epoch} * {reg_factor}) = {max_train_steps}'
)
)
log.info(
f'max_train_steps ({total_steps} / {train_batch_size} / {gradient_accumulation_steps} * {epoch} * {reg_factor}) = {max_train_steps}'
)
# calculate stop encoder training
if stop_text_encoder_training_pct == None:
@ -913,6 +916,7 @@ def train_model(
cache_latents_to_disk=cache_latents_to_disk,
optimizer=optimizer,
optimizer_args=optimizer_args,
lr_scheduler_args=lr_scheduler_args,
)
run_cmd += run_cmd_advanced_training(
@ -1484,6 +1488,7 @@ def lora_tab(
source_model.model_list,
advanced_training.max_token_length,
basic_training.max_train_epochs,
basic_training.max_train_steps,
advanced_training.max_data_loader_n_workers,
network_alpha,
folders.training_comment,
@ -1499,6 +1504,7 @@ def lora_tab(
advanced_training.caption_dropout_rate,
basic_training.optimizer,
basic_training.optimizer_args,
basic_training.lr_scheduler_args,
advanced_training.noise_offset_type,
advanced_training.noise_offset,
advanced_training.adaptive_noise_scale,

1
presets/lora/SDXL - LoHA by AI_Characters.json
View File

@ -89,7 +89,6 @@
"v2": false,
"v_parameterization": false,
"vae_batch_size": 0,
"wandb_api_key": "",
"weighted_captions": false,
"xformers": true
}

1
presets/lora/SDXL - LoRA adafactor v1.0.json
View File

@ -89,7 +89,6 @@
"v2": false,
"v_parameterization": false,
"vae_batch_size": 0,
"wandb_api_key": "",
"weighted_captions": false,
"xformers": true
}

1
presets/lora/SDXL - LoRA by malcolmrey training v1.0.json
View File

@ -89,7 +89,6 @@
"v2": false,
"v_parameterization": false,
"vae_batch_size": 0,
"wandb_api_key": "",
"weighted_captions": false,
"xformers": "xformers"
}

1
presets/lora/SDXL - LoRA finetuning phase 1_v1.1.json
View File

@ -89,7 +89,6 @@
"v2": false,
"v_parameterization": false,
"vae_batch_size": 0,
"wandb_api_key": "",
"weighted_captions": false,
"xformers": true
}

1
presets/lora/SDXL - LoRA finetuning phase 2_v1.1.json
View File

@ -89,7 +89,6 @@
"v2": false,
"v_parameterization": false,
"vae_batch_size": 0,
"wandb_api_key": "",
"weighted_captions": false,
"xformers": true
}

1
presets/lora/SDXL - LoRA prodigy AI_Now v1.0.json
View File

@ -89,7 +89,6 @@
"v2": false,
"v_parameterization": false,
"vae_batch_size": 0,
"wandb_api_key": "",
"weighted_captions": false,
"xformers": "xformers"
}

94
presets/lora/SDXL - edgLoRAXL.json Normal file
View File

@ -0,0 +1,94 @@
{
"LoRA_type": "Standard",
"adaptive_noise_scale": 0,
"additional_parameters": "--max_grad_norm=0",
"block_alphas": "",
"block_dims": "",
"block_lr_zero_threshold": "",
"bucket_no_upscale": true,
"bucket_reso_steps": 32,
"cache_latents": true,
"cache_latents_to_disk": true,
"caption_dropout_every_n_epochs": 0.0,
"caption_dropout_rate": 0,
"caption_extension": ".txt",
"clip_skip": "1",
"color_aug": false,
"conv_alpha": 4,
"conv_block_alphas": "",
"conv_block_dims": "",
"conv_dim": 8,
"decompose_both": false,
"dim_from_weights": false,
"down_lr_weight": "",
"enable_bucket": true,
"epoch": 1,
"factor": -1,
"flip_aug": false,
"full_bf16": false,
"full_fp16": false,
"gradient_accumulation_steps": 1.0,
"gradient_checkpointing": true,
"keep_tokens": "0",
"learning_rate": 1.0,
"lora_network_weights": "",
"lr_scheduler": "cosine",
"lr_scheduler_num_cycles": "",
"lr_scheduler_power": "",
"lr_warmup": 0,
"max_bucket_reso": 2048,
"max_data_loader_n_workers": "0",
"max_resolution": "1024,1024",
"max_timestep": 1000,
"max_token_length": "75",
"max_train_epochs": "",
"mem_eff_attn": false,
"mid_lr_weight": "",
"min_bucket_reso": 256,
"min_snr_gamma": 0,
"min_timestep": 0,
"mixed_precision": "fp16",
"module_dropout": 0,
"multires_noise_discount": 0.2,
"multires_noise_iterations": 8,
"network_alpha": 32,
"network_dim": 32,
"network_dropout": 0,
"no_token_padding": false,
"noise_offset": 0.0357,
"noise_offset_type": "Original",
"num_cpu_threads_per_process": 2,
"optimizer": "Prodigy",
"optimizer_args": "decouple=True weight_decay=0.5 betas=0.9,0.99 use_bias_correction=False",
"persistent_data_loader_workers": false,
"prior_loss_weight": 1.0,
"random_crop": false,
"rank_dropout": 0,
"save_every_n_epochs": 1,
"save_every_n_steps": 0,
"save_last_n_steps": 0,
"save_last_n_steps_state": 0,
"save_precision": "fp16",
"scale_v_pred_loss_like_noise_pred": false,
"scale_weight_norms": 1,
"sdxl": true,
"sdxl_cache_text_encoder_outputs": false,
"sdxl_no_half_vae": true,
"seed": "12345",
"shuffle_caption": false,
"stop_text_encoder_training": 0,
"text_encoder_lr": 1.0,
"train_batch_size": 4,
"train_on_input": false,
"training_comment": "",
"unet_lr": 1.0,
"unit": 1,
"up_lr_weight": "",
"use_cp": true,
"use_wandb": false,
"v2": false,
"v_parameterization": false,
"vae_batch_size": 0,
"weighted_captions": false,
"xformers": true
}

1
presets/lora/SDXL - lokr_v1.0.json
View File

@ -86,7 +86,6 @@
"v2": false,
"v_parameterization": false,
"vae_batch_size": 0,
"wandb_api_key": "",
"weighted_captions": false,
"xformers": true
}

1
presets/lora/ia3-sd15.json
View File

@ -81,7 +81,6 @@
"v2": false,
"v_parameterization": false,
"vae_batch_size": 0,
"wandb_api_key": "",
"weighted_captions": false,
"xformers": true
}

1
presets/lora/locon-dadaptation-sdxl.json
View File

@ -82,7 +82,6 @@
"v2": false,
"v_parameterization": false,
"vae_batch_size": 0,
"wandb_api_key": "",
"weighted_captions": false,
"xformers": true
}

1
presets/lora/loha-sd15.json
View File

@ -81,7 +81,6 @@
"v2": false,
"v_parameterization": false,
"vae_batch_size": 0,
"wandb_api_key": "",
"weighted_captions": false,
"xformers": true
}

1
presets/lora/lokr-sd15.json
View File

@ -79,7 +79,6 @@
"v2": false,
"v_parameterization": false,
"vae_batch_size": 0,
"wandb_api_key": "",
"weighted_captions": false,
"xformers": true
}

20
textual_inversion_gui.py
View File

@ -117,7 +117,7 @@ def save_configuration(
caption_dropout_every_n_epochs,
caption_dropout_rate,
optimizer,
optimizer_args,
optimizer_args,lr_scheduler_args,
noise_offset_type,
noise_offset,
adaptive_noise_scale,
@ -238,7 +238,7 @@ def open_configuration(
caption_dropout_every_n_epochs,
caption_dropout_rate,
optimizer,
optimizer_args,
optimizer_args,lr_scheduler_args,
noise_offset_type,
noise_offset,
adaptive_noise_scale,
@ -356,7 +356,7 @@ def train_model(
caption_dropout_every_n_epochs,
caption_dropout_rate,
optimizer,
optimizer_args,
optimizer_args,lr_scheduler_args,
noise_offset_type,
noise_offset,
adaptive_noise_scale,
@ -506,7 +506,7 @@ def train_model(
reg_factor = 2
# calculate max_train_steps
if max_train_steps == '':
if max_train_steps == '' or max_train_steps == '0':
max_train_steps = int(
math.ceil(
float(total_steps)
@ -606,6 +606,7 @@ def train_model(
cache_latents_to_disk=cache_latents_to_disk,
optimizer=optimizer,
optimizer_args=optimizer_args,
lr_scheduler_args=lr_scheduler_args,
)
run_cmd += run_cmd_advanced_training(
@ -753,10 +754,10 @@ def ti_tab(
step=1,
label='Vectors',
)
max_train_steps = gr.Textbox(
label='Max train steps',
placeholder='(Optional) Maximum number of steps',
)
# max_train_steps = gr.Textbox(
# label='Max train steps',
# placeholder='(Optional) Maximum number of steps',
# )
template = gr.Dropdown(
label='Template',
choices=[
@ -869,7 +870,7 @@ def ti_tab(
token_string,
init_word,
num_vectors_per_token,
max_train_steps,
basic_training.max_train_steps,
weights,
template,
advanced_training.keep_tokens,
@ -884,6 +885,7 @@ def ti_tab(
advanced_training.caption_dropout_rate,
basic_training.optimizer,
basic_training.optimizer_args,
basic_training.lr_scheduler_args,
advanced_training.noise_offset_type,
advanced_training.noise_offset,
advanced_training.adaptive_noise_scale,

2
tools/prepare_presets.py
View File

@ -3,7 +3,7 @@ import argparse
import glob
def remove_items_with_keywords(json_file_path):
keywords = ["pretrained_model_name_or_path", "dir", "save_model_as", "save_state", "resume", "output_name", "model_list", "sample_"]
keywords = ["pretrained_model_name_or_path", "dir", "save_model_as", "save_state", "resume", "output_name", "model_list", "sample_", "wandb_api_key"]
with open(json_file_path) as file:
data = json.load(file)