1580158
/
kohya_ss
mirror of https://github.com/bmaltais/kohya_ss
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add support for finetuning presets

pull/1559/head
bmaltais 2023-10-01 15:19:58 -04:00
parent b4ce7e04da
commit 383f4af137
7 changed files with 197 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@ -647,6 +647,7 @@ ControlNet-LLLite, a novel method for ControlNet with SDXL, is added. See [docum
- New SDXL presets
- Update wandb module version
- Add support for Chinese zh-CN localisation. You can use it with `.\gui.bat --language=zh-CN`
- Add presets support to `Finetuning`. You can add your own finetuning user presets under the `/presets/finetune/user_presets` folder.
* 2023/09/23 (v21.8.10)
- Minor point upgrade. Mostly adding a new preset.

108
finetune_gui.py
View File

@ -180,6 +180,7 @@ def save_configuration(
def open_configuration(
ask_for_file,
apply_preset,
file_path,
pretrained_model_name_or_path,
v2,
@ -267,11 +268,27 @@ def open_configuration(
sdxl_no_half_vae,
min_timestep,
max_timestep,
training_preset,
):
# Get list of function parameters and values
parameters = list(locals().items())
ask_for_file = True if ask_for_file.get('label') == 'True' else False
apply_preset = True if apply_preset.get('label') == 'True' else False
# Check if we are "applying" a preset or a config
if apply_preset:
log.info(f'Applying preset {training_preset}...')
file_path = f'./presets/finetune/{training_preset}.json'
else:
# If not applying a preset, set the `training_preset` field to an empty string
# Find the index of the `training_preset` parameter using the `index()` method
training_preset_index = parameters.index(
('training_preset', training_preset)
)
# Update the value of `training_preset` by directly assigning an empty string value
parameters[training_preset_index] = ('training_preset', '')
original_file_path = file_path
@ -291,9 +308,10 @@ def open_configuration(
values = [file_path]
for key, value in parameters:
json_value = my_data.get(key)
# Set the value in the dictionary to the corresponding value in `my_data`, or the default value if not found
if not key in ['ask_for_file', 'file_path']:
values.append(my_data.get(key, value))
if not key in ['ask_for_file', 'apply_preset', 'file_path']:
values.append(json_value if json_value is not None else value)
return tuple(values)
@ -808,6 +826,31 @@ def finetune_tab(headless=False):
label='Weighted captions', value=False
)
with gr.Tab('Parameters'):
def list_presets(path):
json_files = []
for file in os.listdir(path):
if file.endswith('.json'):
json_files.append(os.path.splitext(file)[0])
user_presets_path = os.path.join(path, 'user_presets')
if os.path.isdir(user_presets_path):
for file in os.listdir(user_presets_path):
if file.endswith('.json'):
preset_name = os.path.splitext(file)[0]
json_files.append(
os.path.join('user_presets', preset_name)
)
return json_files
training_preset = gr.Dropdown(
label='Presets',
choices=list_presets('./presets/finetune'),
elem_id='myDropdown',
)
with gr.Tab('Basic', elem_id='basic_tab'):
basic_training = BasicTraining(
learning_rate_value='1e-5', finetuning=True
@ -960,6 +1003,53 @@ def finetune_tab(headless=False):
advanced_training.max_timestep,
]
config.button_open_config.click(
open_configuration,
inputs=[dummy_db_true, dummy_db_false, config.config_file_name]
+ settings_list
+ [training_preset],
outputs=[config.config_file_name]
+ settings_list
+ [training_preset],
show_progress=False,
)
# config.button_open_config.click(
# open_configuration,
# inputs=[dummy_db_true, dummy_db_false, config.config_file_name] + settings_list,
# outputs=[config.config_file_name] + settings_list,
# show_progress=False,
# )
config.button_load_config.click(
open_configuration,
inputs=[dummy_db_false, dummy_db_false, config.config_file_name]
+ settings_list
+ [training_preset],
outputs=[config.config_file_name]
+ settings_list
+ [training_preset],
show_progress=False,
)
# config.button_load_config.click(
# open_configuration,
# inputs=[dummy_db_false, config.config_file_name] + settings_list,
# outputs=[config.config_file_name] + settings_list,
# show_progress=False,
# )
training_preset.input(
open_configuration,
inputs=[dummy_db_false, dummy_db_true, config.config_file_name]
+ settings_list
+ [training_preset],
outputs=[gr.Textbox()]
+ settings_list
+ [training_preset],
show_progress=False,
)
button_run.click(
train_model,
inputs=[dummy_headless] + [dummy_db_false] + settings_list,
@ -974,20 +1064,6 @@ def finetune_tab(headless=False):
show_progress=False,
)
config.button_open_config.click(
open_configuration,
inputs=[dummy_db_true, config.config_file_name] + settings_list,
outputs=[config.config_file_name] + settings_list,
show_progress=False,
)
config.button_load_config.click(
open_configuration,
inputs=[dummy_db_false, config.config_file_name] + settings_list,
outputs=[config.config_file_name] + settings_list,
show_progress=False,
)
config.button_save_config.click(
save_configuration,
inputs=[dummy_db_false, config.config_file_name] + settings_list,

106
presets/finetune/adafactor.json
View File

@ -1,61 +1,49 @@
{
"pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
"v2": false,
"v_parameterization": false,
"train_dir": "D:/dataset/paige_spiranac/ft",
"image_folder": "D:\\dataset\\paige_spiranac\\lora\\img4_g8\\16_paige_spiranac",
"output_dir": "D:/models/test",
"logging_dir": "D:/dataset/paige_spiranac/ft/logs",
"max_resolution": "512,512",
"min_bucket_reso": "256",
"max_bucket_reso": "1024",
"batch_size": "1",
"flip_aug": false,
"caption_metadata_filename": "meta_cap.json",
"latent_metadata_filename": "meta_lat.json",
"full_path": true,
"learning_rate": "1e-6",
"lr_scheduler": "adafactor",
"lr_warmup": "10",
"dataset_repeats": "10",
"train_batch_size": 4,
"epoch": "2",
"save_every_n_epochs": "1",
"mixed_precision": "bf16",
"save_precision": "fp16",
"seed": "1234",
"num_cpu_threads_per_process": 2,
"train_text_encoder": true,
"create_caption": true,
"create_buckets": false,
"save_model_as": "safetensors",
"caption_extension": ".txt",
"use_8bit_adam": false,
"xformers": true,
"clip_skip": 1,
"save_state": false,
"resume": "",
"gradient_checkpointing": false,
"gradient_accumulation_steps": 1.0,
"mem_eff_attn": false,
"shuffle_caption": true,
"output_name": "paige_spiranac_v1.5e",
"max_token_length": "150",
"max_train_epochs": "",
"max_data_loader_n_workers": "0",
"full_fp16": false,
"color_aug": false,
"model_list": "runwayml/stable-diffusion-v1-5",
"cache_latents": true,
"use_latent_files": "No",
"keep_tokens": 1,
"persistent_data_loader_workers": false,
"bucket_no_upscale": true,
"random_crop": false,
"bucket_reso_steps": 1.0,
"caption_dropout_every_n_epochs": 0.0,
"caption_dropout_rate": 0.1,
"optimizer": "Adafactor",
"optimizer_args": "scale_parameter=True relative_step=True warmup_init=True weight_decay=2",
"noise_offset": ""
"batch_size": "1",
"bucket_no_upscale": true,
"bucket_reso_steps": 1.0,
"cache_latents": true,
"caption_dropout_every_n_epochs": 0.0,
"caption_dropout_rate": 0.1,
"caption_extension": ".txt",
"clip_skip": 1,
"color_aug": false,
"create_buckets": false,
"create_caption": true,
"dataset_repeats": "10",
"epoch": "2",
"flip_aug": false,
"full_fp16": false,
"full_path": true,
"gradient_accumulation_steps": 1.0,
"gradient_checkpointing": false,
"keep_tokens": 1,
"learning_rate": "1e-6",
"lr_scheduler": "adafactor",
"lr_warmup": "10",
"max_bucket_reso": "1024",
"max_data_loader_n_workers": "0",
"max_resolution": "512,512",
"max_token_length": "150",
"max_train_epochs": "",
"mem_eff_attn": false,
"min_bucket_reso": "256",
"mixed_precision": "bf16",
"noise_offset": "",
"num_cpu_threads_per_process": 2,
"optimizer": "Adafactor",
"optimizer_args": "scale_parameter=True relative_step=True warmup_init=True weight_decay=2",
"persistent_data_loader_workers": false,
"random_crop": false,
"save_every_n_epochs": "1",
"save_precision": "fp16",
"seed": "1234",
"shuffle_caption": true,
"train_batch_size": 4,
"train_text_encoder": true,
"use_8bit_adam": false,
"use_latent_files": "No",
"v2": false,
"v_parameterization": false,
"xformers": true
}

106
presets/finetune/lion.json
View File

@ -1,61 +1,49 @@
{
"pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
"v2": false,
"v_parameterization": false,
"train_dir": "D:/dataset/paige_spiranac/ft",
"image_folder": "D:\\dataset\\paige_spiranac\\lora\\img4_g8\\16_paige_spiranac",
"output_dir": "D:/models/test",
"logging_dir": "D:/dataset/paige_spiranac/ft/logs",
"max_resolution": "512,512",
"min_bucket_reso": "256",
"max_bucket_reso": "1024",
"batch_size": "1",
"flip_aug": false,
"caption_metadata_filename": "meta_cap.json",
"latent_metadata_filename": "meta_lat.json",
"full_path": true,
"learning_rate": "0.0000166666666",
"lr_scheduler": "cosine",
"lr_warmup": "10",
"dataset_repeats": "10",
"train_batch_size": 4,
"epoch": "2",
"save_every_n_epochs": "1",
"mixed_precision": "bf16",
"save_precision": "fp16",
"seed": "1234",
"num_cpu_threads_per_process": 2,
"train_text_encoder": true,
"create_caption": true,
"create_buckets": false,
"save_model_as": "safetensors",
"caption_extension": ".txt",
"use_8bit_adam": false,
"xformers": true,
"clip_skip": 1,
"save_state": false,
"resume": "",
"gradient_checkpointing": false,
"gradient_accumulation_steps": 1.0,
"mem_eff_attn": false,
"shuffle_caption": true,
"output_name": "paige_spiranac_v1.5e",
"max_token_length": "150",
"max_train_epochs": "",
"max_data_loader_n_workers": "0",
"full_fp16": false,
"color_aug": false,
"model_list": "runwayml/stable-diffusion-v1-5",
"cache_latents": true,
"use_latent_files": "No",
"keep_tokens": 1,
"persistent_data_loader_workers": false,
"bucket_no_upscale": true,
"random_crop": false,
"bucket_reso_steps": 1.0,
"caption_dropout_every_n_epochs": 0.0,
"caption_dropout_rate": 0.1,
"optimizer": "Lion",
"optimizer_args": "",
"noise_offset": ""
"batch_size": "1",
"bucket_no_upscale": true,
"bucket_reso_steps": 1.0,
"cache_latents": true,
"caption_dropout_every_n_epochs": 0.0,
"caption_dropout_rate": 0.1,
"caption_extension": ".txt",
"clip_skip": 1,
"color_aug": false,
"create_buckets": false,
"create_caption": true,
"dataset_repeats": "10",
"epoch": "2",
"flip_aug": false,
"full_fp16": false,
"full_path": true,
"gradient_accumulation_steps": 1.0,
"gradient_checkpointing": false,
"keep_tokens": 1,
"learning_rate": "0.0000166666666",
"lr_scheduler": "cosine",
"lr_warmup": "10",
"max_bucket_reso": "1024",
"max_data_loader_n_workers": "0",
"max_resolution": "512,512",
"max_token_length": "150",
"max_train_epochs": "",
"mem_eff_attn": false,
"min_bucket_reso": "256",
"mixed_precision": "bf16",
"noise_offset": "",
"num_cpu_threads_per_process": 2,
"optimizer": "Lion",
"optimizer_args": "",
"persistent_data_loader_workers": false,
"random_crop": false,
"save_every_n_epochs": "1",
"save_precision": "fp16",
"seed": "1234",
"shuffle_caption": true,
"train_batch_size": 4,
"train_text_encoder": true,
"use_8bit_adam": false,
"use_latent_files": "No",
"v2": false,
"v_parameterization": false,
"xformers": true
}

7
presets/finetune/prepare_presets.md Normal file
View File

@ -0,0 +1,7 @@
# Preparing presets for users
Run the followinf command to prepare new presets for release to users:
```
python.exe .\tools\prepare_presets.py .\presets\finetune\*.json
```

4
presets/lora/SDXL - LoRA AI_Now ADamW v1.0.json
View File

@ -40,7 +40,7 @@
"max_bucket_reso": 2048,
"max_data_loader_n_workers": "0",
"max_resolution": "1024,1024",
"max_timestep": 800,
"max_timestep": 900,
"max_token_length": "75",
"max_train_epochs": "",
"max_train_steps": "320",
@ -48,7 +48,7 @@
"mid_lr_weight": "",
"min_bucket_reso": 256,
"min_snr_gamma": 5,
"min_timestep": 200,
"min_timestep": 100,
"mixed_precision": "bf16",
"module_dropout": 0,
"multires_noise_discount": 0,

2
tools/prepare_presets.py
View File

@ -3,7 +3,7 @@ import argparse
import glob
def remove_items_with_keywords(json_file_path):
keywords = ["pretrained_model_name_or_path", "dir", "save_model_as", "save_state", "resume", "output_name", "model_list", "sample_", "wandb_api_key"]
keywords = ["pretrained_model_name_or_path", "train_dir", "output_dir", "logging_dir", "image_folder", "dir", "caption_metadata_filename", "latent_metadata_filename", "save_model_as", "save_state", "resume", "output_name", "model_list", "sample_", "wandb_api_key"]
with open(json_file_path) as file:
data = json.load(file)