diff --git a/README.md b/README.md index 628ee56..93f2ced 100644 --- a/README.md +++ b/README.md @@ -647,6 +647,7 @@ ControlNet-LLLite, a novel method for ControlNet with SDXL, is added. See [docum - New SDXL presets - Update wandb module version - Add support for Chinese zh-CN localisation. You can use it with `.\gui.bat --language=zh-CN` + - Add presets support to `Finetuning`. You can add your own finetuning user presets under the `/presets/finetune/user_presets` folder. * 2023/09/23 (v21.8.10) - Minor point upgrade. Mostly adding a new preset. diff --git a/finetune_gui.py b/finetune_gui.py index 872c51d..892c8be 100644 --- a/finetune_gui.py +++ b/finetune_gui.py @@ -180,6 +180,7 @@ def save_configuration( def open_configuration( ask_for_file, + apply_preset, file_path, pretrained_model_name_or_path, v2, @@ -267,11 +268,27 @@ def open_configuration( sdxl_no_half_vae, min_timestep, max_timestep, + training_preset, ): # Get list of function parameters and values parameters = list(locals().items()) ask_for_file = True if ask_for_file.get('label') == 'True' else False + apply_preset = True if apply_preset.get('label') == 'True' else False + + # Check if we are "applying" a preset or a config + if apply_preset: + log.info(f'Applying preset {training_preset}...') + file_path = f'./presets/finetune/{training_preset}.json' + else: + # If not applying a preset, set the `training_preset` field to an empty string + # Find the index of the `training_preset` parameter using the `index()` method + training_preset_index = parameters.index( + ('training_preset', training_preset) + ) + + # Update the value of `training_preset` by directly assigning an empty string value + parameters[training_preset_index] = ('training_preset', '') original_file_path = file_path @@ -291,9 +308,10 @@ def open_configuration( values = [file_path] for key, value in parameters: + json_value = my_data.get(key) # Set the value in the dictionary to the corresponding value in `my_data`, or the default value if not found - if not key in ['ask_for_file', 'file_path']: - values.append(my_data.get(key, value)) + if not key in ['ask_for_file', 'apply_preset', 'file_path']: + values.append(json_value if json_value is not None else value) return tuple(values) @@ -808,6 +826,31 @@ def finetune_tab(headless=False): label='Weighted captions', value=False ) with gr.Tab('Parameters'): + + def list_presets(path): + json_files = [] + + for file in os.listdir(path): + if file.endswith('.json'): + json_files.append(os.path.splitext(file)[0]) + + user_presets_path = os.path.join(path, 'user_presets') + if os.path.isdir(user_presets_path): + for file in os.listdir(user_presets_path): + if file.endswith('.json'): + preset_name = os.path.splitext(file)[0] + json_files.append( + os.path.join('user_presets', preset_name) + ) + + return json_files + + training_preset = gr.Dropdown( + label='Presets', + choices=list_presets('./presets/finetune'), + elem_id='myDropdown', + ) + with gr.Tab('Basic', elem_id='basic_tab'): basic_training = BasicTraining( learning_rate_value='1e-5', finetuning=True @@ -960,6 +1003,53 @@ def finetune_tab(headless=False): advanced_training.max_timestep, ] + config.button_open_config.click( + open_configuration, + inputs=[dummy_db_true, dummy_db_false, config.config_file_name] + + settings_list + + [training_preset], + outputs=[config.config_file_name] + + settings_list + + [training_preset], + show_progress=False, + ) + + # config.button_open_config.click( + # open_configuration, + # inputs=[dummy_db_true, dummy_db_false, config.config_file_name] + settings_list, + # outputs=[config.config_file_name] + settings_list, + # show_progress=False, + # ) + + config.button_load_config.click( + open_configuration, + inputs=[dummy_db_false, dummy_db_false, config.config_file_name] + + settings_list + + [training_preset], + outputs=[config.config_file_name] + + settings_list + + [training_preset], + show_progress=False, + ) + + # config.button_load_config.click( + # open_configuration, + # inputs=[dummy_db_false, config.config_file_name] + settings_list, + # outputs=[config.config_file_name] + settings_list, + # show_progress=False, + # ) + + training_preset.input( + open_configuration, + inputs=[dummy_db_false, dummy_db_true, config.config_file_name] + + settings_list + + [training_preset], + outputs=[gr.Textbox()] + + settings_list + + [training_preset], + show_progress=False, + ) + button_run.click( train_model, inputs=[dummy_headless] + [dummy_db_false] + settings_list, @@ -974,20 +1064,6 @@ def finetune_tab(headless=False): show_progress=False, ) - config.button_open_config.click( - open_configuration, - inputs=[dummy_db_true, config.config_file_name] + settings_list, - outputs=[config.config_file_name] + settings_list, - show_progress=False, - ) - - config.button_load_config.click( - open_configuration, - inputs=[dummy_db_false, config.config_file_name] + settings_list, - outputs=[config.config_file_name] + settings_list, - show_progress=False, - ) - config.button_save_config.click( save_configuration, inputs=[dummy_db_false, config.config_file_name] + settings_list, diff --git a/presets/finetune/adafactor.json b/presets/finetune/adafactor.json index 0e0149d..6f7a109 100644 --- a/presets/finetune/adafactor.json +++ b/presets/finetune/adafactor.json @@ -1,61 +1,49 @@ { - "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5", - "v2": false, - "v_parameterization": false, - "train_dir": "D:/dataset/paige_spiranac/ft", - "image_folder": "D:\\dataset\\paige_spiranac\\lora\\img4_g8\\16_paige_spiranac", - "output_dir": "D:/models/test", - "logging_dir": "D:/dataset/paige_spiranac/ft/logs", - "max_resolution": "512,512", - "min_bucket_reso": "256", - "max_bucket_reso": "1024", - "batch_size": "1", - "flip_aug": false, - "caption_metadata_filename": "meta_cap.json", - "latent_metadata_filename": "meta_lat.json", - "full_path": true, - "learning_rate": "1e-6", - "lr_scheduler": "adafactor", - "lr_warmup": "10", - "dataset_repeats": "10", - "train_batch_size": 4, - "epoch": "2", - "save_every_n_epochs": "1", - "mixed_precision": "bf16", - "save_precision": "fp16", - "seed": "1234", - "num_cpu_threads_per_process": 2, - "train_text_encoder": true, - "create_caption": true, - "create_buckets": false, - "save_model_as": "safetensors", - "caption_extension": ".txt", - "use_8bit_adam": false, - "xformers": true, - "clip_skip": 1, - "save_state": false, - "resume": "", - "gradient_checkpointing": false, - "gradient_accumulation_steps": 1.0, - "mem_eff_attn": false, - "shuffle_caption": true, - "output_name": "paige_spiranac_v1.5e", - "max_token_length": "150", - "max_train_epochs": "", - "max_data_loader_n_workers": "0", - "full_fp16": false, - "color_aug": false, - "model_list": "runwayml/stable-diffusion-v1-5", - "cache_latents": true, - "use_latent_files": "No", - "keep_tokens": 1, - "persistent_data_loader_workers": false, - "bucket_no_upscale": true, - "random_crop": false, - "bucket_reso_steps": 1.0, - "caption_dropout_every_n_epochs": 0.0, - "caption_dropout_rate": 0.1, - "optimizer": "Adafactor", - "optimizer_args": "scale_parameter=True relative_step=True warmup_init=True weight_decay=2", - "noise_offset": "" + "batch_size": "1", + "bucket_no_upscale": true, + "bucket_reso_steps": 1.0, + "cache_latents": true, + "caption_dropout_every_n_epochs": 0.0, + "caption_dropout_rate": 0.1, + "caption_extension": ".txt", + "clip_skip": 1, + "color_aug": false, + "create_buckets": false, + "create_caption": true, + "dataset_repeats": "10", + "epoch": "2", + "flip_aug": false, + "full_fp16": false, + "full_path": true, + "gradient_accumulation_steps": 1.0, + "gradient_checkpointing": false, + "keep_tokens": 1, + "learning_rate": "1e-6", + "lr_scheduler": "adafactor", + "lr_warmup": "10", + "max_bucket_reso": "1024", + "max_data_loader_n_workers": "0", + "max_resolution": "512,512", + "max_token_length": "150", + "max_train_epochs": "", + "mem_eff_attn": false, + "min_bucket_reso": "256", + "mixed_precision": "bf16", + "noise_offset": "", + "num_cpu_threads_per_process": 2, + "optimizer": "Adafactor", + "optimizer_args": "scale_parameter=True relative_step=True warmup_init=True weight_decay=2", + "persistent_data_loader_workers": false, + "random_crop": false, + "save_every_n_epochs": "1", + "save_precision": "fp16", + "seed": "1234", + "shuffle_caption": true, + "train_batch_size": 4, + "train_text_encoder": true, + "use_8bit_adam": false, + "use_latent_files": "No", + "v2": false, + "v_parameterization": false, + "xformers": true } \ No newline at end of file diff --git a/presets/finetune/lion.json b/presets/finetune/lion.json index 982c8a8..0f74105 100644 --- a/presets/finetune/lion.json +++ b/presets/finetune/lion.json @@ -1,61 +1,49 @@ { - "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5", - "v2": false, - "v_parameterization": false, - "train_dir": "D:/dataset/paige_spiranac/ft", - "image_folder": "D:\\dataset\\paige_spiranac\\lora\\img4_g8\\16_paige_spiranac", - "output_dir": "D:/models/test", - "logging_dir": "D:/dataset/paige_spiranac/ft/logs", - "max_resolution": "512,512", - "min_bucket_reso": "256", - "max_bucket_reso": "1024", - "batch_size": "1", - "flip_aug": false, - "caption_metadata_filename": "meta_cap.json", - "latent_metadata_filename": "meta_lat.json", - "full_path": true, - "learning_rate": "0.0000166666666", - "lr_scheduler": "cosine", - "lr_warmup": "10", - "dataset_repeats": "10", - "train_batch_size": 4, - "epoch": "2", - "save_every_n_epochs": "1", - "mixed_precision": "bf16", - "save_precision": "fp16", - "seed": "1234", - "num_cpu_threads_per_process": 2, - "train_text_encoder": true, - "create_caption": true, - "create_buckets": false, - "save_model_as": "safetensors", - "caption_extension": ".txt", - "use_8bit_adam": false, - "xformers": true, - "clip_skip": 1, - "save_state": false, - "resume": "", - "gradient_checkpointing": false, - "gradient_accumulation_steps": 1.0, - "mem_eff_attn": false, - "shuffle_caption": true, - "output_name": "paige_spiranac_v1.5e", - "max_token_length": "150", - "max_train_epochs": "", - "max_data_loader_n_workers": "0", - "full_fp16": false, - "color_aug": false, - "model_list": "runwayml/stable-diffusion-v1-5", - "cache_latents": true, - "use_latent_files": "No", - "keep_tokens": 1, - "persistent_data_loader_workers": false, - "bucket_no_upscale": true, - "random_crop": false, - "bucket_reso_steps": 1.0, - "caption_dropout_every_n_epochs": 0.0, - "caption_dropout_rate": 0.1, - "optimizer": "Lion", - "optimizer_args": "", - "noise_offset": "" + "batch_size": "1", + "bucket_no_upscale": true, + "bucket_reso_steps": 1.0, + "cache_latents": true, + "caption_dropout_every_n_epochs": 0.0, + "caption_dropout_rate": 0.1, + "caption_extension": ".txt", + "clip_skip": 1, + "color_aug": false, + "create_buckets": false, + "create_caption": true, + "dataset_repeats": "10", + "epoch": "2", + "flip_aug": false, + "full_fp16": false, + "full_path": true, + "gradient_accumulation_steps": 1.0, + "gradient_checkpointing": false, + "keep_tokens": 1, + "learning_rate": "0.0000166666666", + "lr_scheduler": "cosine", + "lr_warmup": "10", + "max_bucket_reso": "1024", + "max_data_loader_n_workers": "0", + "max_resolution": "512,512", + "max_token_length": "150", + "max_train_epochs": "", + "mem_eff_attn": false, + "min_bucket_reso": "256", + "mixed_precision": "bf16", + "noise_offset": "", + "num_cpu_threads_per_process": 2, + "optimizer": "Lion", + "optimizer_args": "", + "persistent_data_loader_workers": false, + "random_crop": false, + "save_every_n_epochs": "1", + "save_precision": "fp16", + "seed": "1234", + "shuffle_caption": true, + "train_batch_size": 4, + "train_text_encoder": true, + "use_8bit_adam": false, + "use_latent_files": "No", + "v2": false, + "v_parameterization": false, + "xformers": true } \ No newline at end of file diff --git a/presets/finetune/prepare_presets.md b/presets/finetune/prepare_presets.md new file mode 100644 index 0000000..48e2101 --- /dev/null +++ b/presets/finetune/prepare_presets.md @@ -0,0 +1,7 @@ +# Preparing presets for users + +Run the followinf command to prepare new presets for release to users: + +``` +python.exe .\tools\prepare_presets.py .\presets\finetune\*.json +``` \ No newline at end of file diff --git a/presets/lora/SDXL - LoRA AI_Now ADamW v1.0.json b/presets/lora/SDXL - LoRA AI_Now ADamW v1.0.json index 061e415..da25fba 100644 --- a/presets/lora/SDXL - LoRA AI_Now ADamW v1.0.json +++ b/presets/lora/SDXL - LoRA AI_Now ADamW v1.0.json @@ -40,7 +40,7 @@ "max_bucket_reso": 2048, "max_data_loader_n_workers": "0", "max_resolution": "1024,1024", - "max_timestep": 800, + "max_timestep": 900, "max_token_length": "75", "max_train_epochs": "", "max_train_steps": "320", @@ -48,7 +48,7 @@ "mid_lr_weight": "", "min_bucket_reso": 256, "min_snr_gamma": 5, - "min_timestep": 200, + "min_timestep": 100, "mixed_precision": "bf16", "module_dropout": 0, "multires_noise_discount": 0, diff --git a/tools/prepare_presets.py b/tools/prepare_presets.py index 8c00af4..a94b876 100644 --- a/tools/prepare_presets.py +++ b/tools/prepare_presets.py @@ -3,7 +3,7 @@ import argparse import glob def remove_items_with_keywords(json_file_path): - keywords = ["pretrained_model_name_or_path", "dir", "save_model_as", "save_state", "resume", "output_name", "model_list", "sample_", "wandb_api_key"] + keywords = ["pretrained_model_name_or_path", "train_dir", "output_dir", "logging_dir", "image_folder", "dir", "caption_metadata_filename", "latent_metadata_filename", "save_model_as", "save_state", "resume", "output_name", "model_list", "sample_", "wandb_api_key"] with open(json_file_path) as file: data = json.load(file)