Update Finetuning tab structure

docs/Finetuning/top_level.md

@@ -0,0 +1,28 @@
+# Finetuning Resource Guide
+
+This guide is a resource compilation to facilitate the development of robust LoRA models.
+
+-Need to add resources here
+
+## Guidelines for SDXL Finetuning 
+
+- Set the `Max resolution` to at least 1024x1024, as this is the standard resolution for SDXL.
+- The fine-tuning can be done with 24GB GPU memory with the batch size of 1.
+  - Train U-Net only.
+  - Use gradient checkpointing.
+  - Use `--cache_text_encoder_outputs` option and caching latents.
+  - Use Adafactor optimizer. RMSprop 8bit or Adagrad 8bit may work. AdamW 8bit doesn't seem to work.
+- PyTorch 2 seems to use slightly less GPU memory than PyTorch 1.
+
+Example of the optimizer settings for Adafactor with the fixed learning rate:
+```
+optimizer_type = "adafactor"
+optimizer_args = [ "scale_parameter=False", "relative_step=False", "warmup_init=False" ]
+lr_scheduler = "constant_with_warmup"
+lr_warmup_steps = 100
+learning_rate = 4e-7 # SDXL original learning rate
+```
+
+## Resource Contributions
+
+If you have valuable resources to add, kindly create a PR on Github.

docs/LoRA/top_level.md

@@ -1,4 +1,4 @@
-# LoRA Development Resource Guide
+# LoRA Resource Guide
 
 This guide is a resource compilation to facilitate the development of robust LoRA models.
 

finetune_gui.py

@@ -632,6 +632,7 @@ def finetune_tab(headless=False):
     dummy_db_true = gr.Label(value=True, visible=False)
     dummy_db_false = gr.Label(value=False, visible=False)
     dummy_headless = gr.Label(value=headless, visible=False)
+    with gr.Tab('Training'):
         gr.Markdown('Train a custom model using kohya finetune python code...')
 
         (
@@ -774,7 +775,7 @@ def finetune_tab(headless=False):
                     weighted_captions = gr.Checkbox(
                         label='Weighted captions', value=False
                     )
-    with gr.Tab('Training parameters'):
+        with gr.Tab('Parameters'):
             (
                 learning_rate,
                 lr_scheduler,
@@ -1009,6 +1010,16 @@ def finetune_tab(headless=False):
             show_progress=False,
         )
         
+    with gr.Tab('Guides'):
+        gr.Markdown(
+            'This section provide Various Finetuning guides and information...'
+        )
+        top_level_path = './docs/Finetuning/top_level.md'
+        if os.path.exists(top_level_path):
+            with open(os.path.join(top_level_path), 'r', encoding='utf8') as file:
+                guides_top_level = file.read() + '\n'
+        gr.Markdown(guides_top_level)
+
 
 def UI(**kwargs):
     css = ''

presets/lora/lokr-sd15.json

@@ -22,7 +22,6 @@
   "dim_from_weights": false,
   "down_lr_weight": "",
   "enable_bucket": true,
-  "epoch": 1,
   "factor": -1,
   "flip_aug": false,
   "full_fp16": false,
@@ -30,7 +29,6 @@
   "gradient_checkpointing": false,
   "keep_tokens": "0",
   "learning_rate": 1.0,
-  "logging_dir": "",
   "lora_network_weights": "",
   "lr_scheduler": "cosine",
   "lr_scheduler_num_cycles": "",
@@ -44,7 +42,6 @@
   "mid_lr_weight": "",
   "min_snr_gamma": 10,
   "mixed_precision": "bf16",
-  "model_list": "runwayml/stable-diffusion-v1-5",
   "module_dropout": 0,
   "multires_noise_discount": 0.2,
   "multires_noise_iterations": 8,
@@ -57,20 +54,16 @@
   "num_cpu_threads_per_process": 2,
   "optimizer": "Prodigy",
   "optimizer_args": "",
-  "output_dir": "",
-  "output_name": "",
   "persistent_data_loader_workers": false,
   "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
   "prior_loss_weight": 1.0,
   "random_crop": false,
   "rank_dropout": 0,
-  "reg_data_dir": "",
   "resume": "",
   "sample_every_n_epochs": 0,
   "sample_every_n_steps": 0,
   "sample_prompts": "",
   "sample_sampler": "euler_a",
-  "save_every_n_epochs": 1,
   "save_every_n_steps": 0,
   "save_last_n_steps": 0,
   "save_last_n_steps_state": 0,
@@ -84,7 +77,6 @@
   "stop_text_encoder_training": 0,
   "text_encoder_lr": 1.0,
   "train_batch_size": 1,
-  "train_data_dir": "",
   "train_on_input": false,
   "training_comment": "",
   "unet_lr": 1.0,

test/config/iA3-Prodigy.json

@@ -67,8 +67,8 @@
   "reg_data_dir": "",
   "resume": "",
   "sample_every_n_epochs": 1,
-  "sample_every_n_steps": null,
-  "sample_prompts": "a painting of man wearing a gas mask , by darius kawasaki",
+  "sample_every_n_steps": 0,
+  "sample_prompts": "a man wearing a gas mask, by darius kawasaki",
   "sample_sampler": "euler_a",
   "save_every_n_epochs": 1,
   "save_every_n_steps": 0,