Merge pull request #2137 from bmaltais/dev

v23.0.15

.gitignore

@@ -51,3 +51,4 @@ dataset/**
 models
 data
 config.toml
+sd-scripts

.release

@@ -1 +1 @@
-v23.0.14
+v23.0.15

README.md

@@ -38,8 +38,9 @@ The GUI allows you to set the training parameters and generate and run the requi
     - [No module called tkinter](#no-module-called-tkinter)
   - [SDXL training](#sdxl-training)
   - [Change History](#change-history)
-    - [2024/03/13 (v23.0.14)](#20240313-v23014)
+    - [2024/03/20 (v23.0.15)](#20240320-v23015)
-    - [2024/03/13 (v23.0.13)](#20240313-v23013)
+    - [2024/03/19 (v23.0.14)](#20240319-v23014)
+    - [2024/03/19 (v23.0.13)](#20240319-v23013)
     - [2024/03/16 (v23.0.12)](#20240316-v23012)
       - [New Features \& Improvements](#new-features--improvements)
       - [Software Updates](#software-updates)
@@ -381,11 +382,19 @@ The documentation in this section will be moved to a separate document later.
 
 ## Change History
 
-### 2024/03/13 (v23.0.14)
+### 2024/03/21 (v23.0.15)
+
+- Add support for toml dataset configuration fole to all trainers
+- Add new setup menu option to install Triton 2.1.0 for Windows
+- Add support for LyCORIS BOFT and DoRA and QLyCORIS options for LoHA, LoKr and LoCon
+- Fix issue with vae path validation
+- Other fixes
+
+### 2024/03/19 (v23.0.14)
 
 - Fix blip caption issue
-- 
+
-### 2024/03/13 (v23.0.13)
+### 2024/03/19 (v23.0.13)
 
 - Fix issue with image samples.
 

kohya_gui/class_advanced_training.py

@@ -7,6 +7,7 @@ from .common_gui import (
     list_files,
     list_dirs,
     create_refresh_button,
+    document_symbol
 )
 
 
@@ -38,8 +39,7 @@ class AdvancedTraining:
             headless (bool): Run in headless mode without GUI.
             finetuning (bool): Enable model fine-tuning.
             training_type (str): The type of training to be performed.
-            default_vae_dir (str): Default directory for VAE models.
+            config (dict): Configuration options for the training process.
-            default_output_dir (str): Default directory for output files.
         """
         self.headless = headless
         self.finetuning = finetuning
@@ -368,10 +368,6 @@ class AdvancedTraining:
                 outputs=self.resume,
                 show_progress=False,
             )
-            # self.max_train_epochs = gr.Textbox(
-            #     label='Max train epoch',
-            #     placeholder='(Optional) Override number of epoch',
-            # )
             self.max_data_loader_n_workers = gr.Textbox(
                 label="Max num workers for DataLoader",
                 placeholder="(Optional) Override number of epoch. Default: 8",
@@ -437,7 +433,7 @@ class AdvancedTraining:
                 "open_folder_small",
             )
             self.log_tracker_config_button = gr.Button(
-                "📂", elem_id="open_folder_small", visible=(not headless)
+                document_symbol, elem_id="open_folder_small", visible=(not headless)
             )
             self.log_tracker_config_button.click(
                 get_any_file_path,

kohya_gui/class_folders.py

@@ -1,6 +1,6 @@
 import gradio as gr
 import os
-from .common_gui import get_folder_path, scriptdir, list_dirs, create_refresh_button
+from .common_gui import get_folder_path, scriptdir, list_dirs, list_files, create_refresh_button
 
 class Folders:
     """

kohya_gui/class_source_model.py

@@ -2,7 +2,7 @@ import gradio as gr
 import os
 
 from .common_gui import (
-    get_any_file_path,
+    get_file_path,
     get_folder_path,
     set_pretrained_model_name_or_path_input,
     scriptdir,
@@ -61,6 +61,8 @@ class SourceModel:
         self.current_train_data_dir = self.config.get(
             "train_data_dir", os.path.join(scriptdir, "data")
         )
+        self.current_dataset_config_dir = self.config.get('dataset_config_dir', os.path.join(scriptdir, "dataset_config"))
+
 
         model_checkpoints = list(
             list_files(
@@ -80,6 +82,21 @@ class SourceModel:
             self.current_train_data_dir = path if not path == "" else "."
             return list(list_dirs(path))
         
+        def list_dataset_config_dirs(path: str) -> list:
+            """
+            List directories and toml files in the dataset_config directory.
+
+            Parameters:
+            - path (str): The path to list directories and files from.
+
+            Returns:
+            - list: A list of directories and files.
+            """
+            current_dataset_config_dir = path if not path == "" else "."
+            # Lists all .json files in the current configuration directory, used for populating dropdown choices.
+            return list(list_files(current_dataset_config_dir, exts=[".toml"], all=True))
+
+
         with gr.Column(), gr.Group():
             # Define the input elements
             with gr.Row():
@@ -107,7 +124,7 @@ class SourceModel:
                         visible=(not headless),
                     )
                     self.pretrained_model_name_or_path_file.click(
-                        get_any_file_path,
+                        get_file_path,
                         inputs=self.pretrained_model_name_or_path,
                         outputs=self.pretrained_model_name_or_path,
                         show_progress=False,
@@ -125,6 +142,14 @@ class SourceModel:
                         show_progress=False,
                     )
                     
+                with gr.Column(), gr.Row():
+                    self.output_name = gr.Textbox(
+                        label="Trained Model output name",
+                        placeholder="(Name of the model to output)",
+                        value="last",
+                        interactive=True,
+                    )
+            with gr.Row():
                 with gr.Column(), gr.Row():
                     self.train_data_dir = gr.Dropdown(
                         label=(
@@ -158,6 +183,36 @@ class SourceModel:
                         outputs=self.train_data_dir,
                         show_progress=False,
                     )
+                with gr.Column(), gr.Row():
+                    # Toml directory dropdown
+                    self.dataset_config = gr.Dropdown(
+                        label='Dataset config file (Optional. Select the toml configuration file to use for the dataset)',
+                        choices=[""] + list_dataset_config_dirs(self.current_dataset_config_dir),
+                        value="",
+                        interactive=True,
+                        allow_custom_value=True,
+                    )
+                    # Refresh button for dataset_config directory
+                    create_refresh_button(self.dataset_config, lambda: None, lambda: {"choices": [""] + list_dataset_config_dirs(self.current_dataset_config_dir)}, "open_folder_small")
+                    # Toml directory button
+                    self.dataset_config_folder = gr.Button(
+                        document_symbol, elem_id='open_folder_small', elem_classes=["tool"], visible=(not self.headless)
+                    )
+                    
+                    # Toml directory button click event
+                    self.dataset_config_folder.click(
+                        get_file_path,
+                        inputs=[self.dataset_config, gr.Textbox(value='*.toml', visible=False), gr.Textbox(value='Dataset config types', visible=False)],
+                        outputs=self.dataset_config,
+                        show_progress=False,
+                    )
+                    # Change event for dataset_config directory dropdown
+                    self.dataset_config.change(
+                        fn=lambda path: gr.Dropdown(choices=[""] + list_dataset_config_dirs(path)),
+                        inputs=self.dataset_config,
+                        outputs=self.dataset_config,
+                        show_progress=False,
+                    )
 
             with gr.Row():
                 with gr.Column():
@@ -181,12 +236,6 @@ class SourceModel:
                     gr.Box(visible=False)
 
             with gr.Row():
-                self.output_name = gr.Textbox(
-                    label="Trained Model output name",
-                    placeholder="(Name of the model to output)",
-                    value="last",
-                    interactive=True,
-                )
                 self.training_comment = gr.Textbox(
                     label="Training comment",
                     placeholder="(Optional) Add training comment to be included in metadata",

kohya_gui/common_gui.py

@@ -54,6 +54,23 @@ ALL_PRESET_MODELS = V2_BASE_MODELS + V_PARAMETERIZATION_MODELS + V1_MODELS + SDX
 
 ENV_EXCLUSION = ["COLAB_GPU", "RUNPOD_POD_ID"]
 
+def calculate_max_train_steps(
+    total_steps: int,
+    train_batch_size: int,
+    gradient_accumulation_steps: int,
+    epoch: int,
+    reg_factor: int,
+):
+    return int(
+        math.ceil(
+            float(total_steps)
+            / int(train_batch_size)
+            / int(gradient_accumulation_steps)
+            * int(epoch)
+            * int(reg_factor)
+        )
+    )
+
 def check_if_model_exist(
     output_name: str, output_dir: str, save_model_as: str, headless: bool = False
 ) -> bool:
@@ -1077,6 +1094,11 @@ def run_cmd_advanced_training(**kwargs):
     if color_aug:
         run_cmd += " --color_aug"
 
+    dataset_config = kwargs.get("dataset_config")
+    if dataset_config:
+        dataset_config = os.path.abspath(os.path.normpath(dataset_config))
+        run_cmd += f' --dataset_config="{dataset_config}"'
+
     dataset_repeats = kwargs.get("dataset_repeats")
     if dataset_repeats:
         run_cmd += f' --dataset_repeats="{dataset_repeats}"'
@@ -1753,6 +1775,13 @@ def validate_paths(headless: bool = False, **kwargs: Optional[str]) -> bool:
         if key in ["output_dir", "logging_dir"]:
             if not validate_path(value, key, create_if_missing=True):
                 return False
+        elif key in ["vae"]:
+            # Check if it matches the Hugging Face model pattern
+            if re.match(r"^[\w-]+\/[\w-]+$", value):
+                log.info("Checking vae... huggingface.co model, skipping validation")
+            else:
+                if not validate_path(value, key):
+                    return False
         else:
             if key not in ["pretrained_model_name_or_path"]:
                 if not validate_path(value, key):

kohya_gui/dreambooth_gui.py

@@ -58,6 +58,7 @@ def save_configuration(
     train_data_dir,
     reg_data_dir,
     output_dir,
+    dataset_config,
     max_resolution,
     learning_rate,
     learning_rate_te,
@@ -189,6 +190,7 @@ def open_configuration(
     train_data_dir,
     reg_data_dir,
     output_dir,
+    dataset_config,
     max_resolution,
     learning_rate,
     learning_rate_te,
@@ -315,6 +317,7 @@ def train_model(
     train_data_dir,
     reg_data_dir,
     output_dir,
+    dataset_config,
     max_resolution,
     learning_rate,
     learning_rate_te,
@@ -421,6 +424,7 @@ def train_model(
         log_tracker_config=log_tracker_config,
         resume=resume,
         vae=vae,
+        dataset_config=dataset_config,
     ):
         return
 
@@ -429,102 +433,90 @@ def train_model(
     ):
         return
 
-    # if sdxl:
+    if dataset_config:
-    #     output_message(
+        log.info("Dataset config toml file used, skipping total_steps, train_batch_size, gradient_accumulation_steps, epoch, reg_factor, max_train_steps calculations...")
-    #         msg='Dreambooth training is not compatible with SDXL models yet..',
-    #         headless=headless_bool,
-    #     )
-    #     return
-
-    # if optimizer == 'Adafactor' and lr_warmup != '0':
-    #     output_message(
-    #         msg="Warning: lr_scheduler is set to 'Adafactor', so 'LR warmup (% of steps)' will be considered 0.",
-    #         title='Warning',
-    #         headless=headless_bool,
-    #     )
-    #     lr_warmup = '0'
-
-    # Get a list of all subfolders in train_data_dir, excluding hidden folders
-    subfolders = [
-        f
-        for f in os.listdir(train_data_dir)
-        if os.path.isdir(os.path.join(train_data_dir, f)) and not f.startswith(".")
-    ]
-
-    # Check if subfolders are present. If not let the user know and return
-    if not subfolders:
-        log.info(f"No {subfolders} were found in train_data_dir can't train...")
-        return
-
-    total_steps = 0
-
-    # Loop through each subfolder and extract the number of repeats
-    for folder in subfolders:
-        # Extract the number of repeats from the folder name
-        try:
-            repeats = int(folder.split("_")[0])
-        except ValueError:
-            log.info(
-                f"Subfolder {folder} does not have a proper repeat value, please correct the name or remove it... can't train..."
-            )
-            continue
-
-        # Count the number of images in the folder
-        num_images = len(
-            [
-                f
-                for f, lower_f in (
-                    (file, file.lower())
-                    for file in os.listdir(os.path.join(train_data_dir, folder))
-                )
-                if lower_f.endswith((".jpg", ".jpeg", ".png", ".webp"))
-            ]
-        )
-
-        if num_images == 0:
-            log.info(f"{folder} folder contain no images, skipping...")
-        else:
-            # Calculate the total number of steps for this folder
-            steps = repeats * num_images
-            total_steps += steps
-
-            # Print the result
-            log.info(f"Folder {folder} : steps {steps}")
-
-    if total_steps == 0:
-        log.info(f"No images were found in folder {train_data_dir}... please rectify!")
-        return
-
-    # Print the result
-    # log.info(f"{total_steps} total steps")
-
-    if reg_data_dir == "":
-        reg_factor = 1
     else:
-        log.info(
+        # Get a list of all subfolders in train_data_dir, excluding hidden folders
-            f"Regularisation images are used... Will double the number of steps required..."
+        subfolders = [
-        )
+            f
-        reg_factor = 2
+            for f in os.listdir(train_data_dir)
+            if os.path.isdir(os.path.join(train_data_dir, f)) and not f.startswith(".")
+        ]
 
-    if max_train_steps == "" or max_train_steps == "0":
+        # Check if subfolders are present. If not let the user know and return
-        # calculate max_train_steps
+        if not subfolders:
-        max_train_steps = int(
+            log.info(f"No {subfolders} were found in train_data_dir can't train...")
-            math.ceil(
+            return
-                float(total_steps)
+
-                / int(train_batch_size)
+        total_steps = 0
-                / int(gradient_accumulation_steps)
+
-                * int(epoch)
+        # Loop through each subfolder and extract the number of repeats
-                * int(reg_factor)
+        for folder in subfolders:
+            # Extract the number of repeats from the folder name
+            try:
+                repeats = int(folder.split("_")[0])
+            except ValueError:
+                log.info(
+                    f"Subfolder {folder} does not have a proper repeat value, please correct the name or remove it... can't train..."
+                )
+                continue
+
+            # Count the number of images in the folder
+            num_images = len(
+                [
+                    f
+                    for f, lower_f in (
+                        (file, file.lower())
+                        for file in os.listdir(os.path.join(train_data_dir, folder))
+                    )
+                    if lower_f.endswith((".jpg", ".jpeg", ".png", ".webp"))
+                ]
+            )
+
+            if num_images == 0:
+                log.info(f"{folder} folder contain no images, skipping...")
+            else:
+                # Calculate the total number of steps for this folder
+                steps = repeats * num_images
+                total_steps += steps
+
+                # Print the result
+                log.info(f"Folder {folder} : steps {steps}")
+
+        if total_steps == 0:
+            log.info(f"No images were found in folder {train_data_dir}... please rectify!")
+            return
+
+        # Print the result
+        # log.info(f"{total_steps} total steps")
+
+        if reg_data_dir == "":
+            reg_factor = 1
+        else:
+            log.info(
+                f"Regularisation images are used... Will double the number of steps required..."
+            )
+            reg_factor = 2
+
+        if max_train_steps == "" or max_train_steps == "0":
+            # calculate max_train_steps
+            max_train_steps = int(
+                math.ceil(
+                    float(total_steps)
+                    / int(train_batch_size)
+                    / int(gradient_accumulation_steps)
+                    * int(epoch)
+                    * int(reg_factor)
+                )
+            )
+            log.info(
+                f"max_train_steps ({total_steps} / {train_batch_size} / {gradient_accumulation_steps} * {epoch} * {reg_factor}) = {max_train_steps}"
             )
-        )
-        log.info(
-            f"max_train_steps ({total_steps} / {train_batch_size} / {gradient_accumulation_steps} * {epoch} * {reg_factor}) = {max_train_steps}"
-        )
 
     # calculate stop encoder training
     if int(stop_text_encoder_training_pct) == -1:
         stop_text_encoder_training = -1
-    elif stop_text_encoder_training_pct == None:
+    elif stop_text_encoder_training_pct == None or (not max_train_steps == "" or not max_train_steps == "0"):
         stop_text_encoder_training = 0
     else:
         stop_text_encoder_training = math.ceil(
@@ -532,7 +524,10 @@ def train_model(
         )
     log.info(f"stop_text_encoder_training = {stop_text_encoder_training}")
 
-    lr_warmup_steps = round(float(int(lr_warmup) * int(max_train_steps) / 100))
+    if not max_train_steps == "":
+        lr_warmup_steps = round(float(int(lr_warmup) * int(max_train_steps) / 100))
+    else:
+        lr_warmup_steps = 0
     log.info(f"lr_warmup_steps = {lr_warmup_steps}")
 
     # run_cmd = f'accelerate launch --num_cpu_threads_per_process={num_cpu_threads_per_process} "train_db.py"'
@@ -564,6 +559,7 @@ def train_model(
         "caption_extension": caption_extension,
         "clip_skip": clip_skip,
         "color_aug": color_aug,
+        "dataset_config": dataset_config,
         "enable_bucket": enable_bucket,
         "epoch": epoch,
         "flip_aug": flip_aug,
@@ -788,6 +784,7 @@ def dreambooth_tab(
             source_model.train_data_dir,
             folders.reg_data_dir,
             folders.output_dir,
+            source_model.dataset_config,
             basic_training.max_resolution,
             basic_training.learning_rate,
             basic_training.learning_rate_te,

kohya_gui/finetune_gui.py

@@ -62,6 +62,7 @@ def save_configuration(
     train_dir,
     image_folder,
     output_dir,
+    dataset_config,
     logging_dir,
     max_resolution,
     min_bucket_reso,
@@ -108,6 +109,7 @@ def save_configuration(
     output_name,
     max_token_length,
     max_train_epochs,
+    max_train_steps,
     max_data_loader_n_workers,
     full_fp16,
     color_aug,
@@ -200,6 +202,7 @@ def open_configuration(
     train_dir,
     image_folder,
     output_dir,
+    dataset_config,
     logging_dir,
     max_resolution,
     min_bucket_reso,
@@ -246,6 +249,7 @@ def open_configuration(
     output_name,
     max_token_length,
     max_train_epochs,
+    max_train_steps,
     max_data_loader_n_workers,
     full_fp16,
     color_aug,
@@ -345,6 +349,7 @@ def train_model(
     train_dir,
     image_folder,
     output_dir,
+    dataset_config,
     logging_dir,
     max_resolution,
     min_bucket_reso,
@@ -391,6 +396,7 @@ def train_model(
     output_name,
     max_token_length,
     max_train_epochs,
+    max_train_steps,
     max_data_loader_n_workers,
     full_fp16,
     color_aug,
@@ -455,93 +461,99 @@ def train_model(
         logging_dir=logging_dir,
         log_tracker_config=log_tracker_config,
         resume=resume,
+        dataset_config=dataset_config
     ):
         return
 
     if not print_only_bool and check_if_model_exist(output_name, output_dir, save_model_as, headless_bool):
         return
 
-    # create caption json file
+    if dataset_config:
-    if generate_caption_database:
+        log.info("Dataset config toml file used, skipping caption json file, image buckets, total_steps, train_batch_size, gradient_accumulation_steps, epoch, reg_factor, max_train_steps creation...")
-        run_cmd = fr'"{PYTHON}" "{scriptdir}/sd-scripts/finetune/merge_captions_to_metadata.py"'
+    else:   
-        if caption_extension == "":
+        # create caption json file
-            run_cmd += f' --caption_extension=".caption"'
+        if generate_caption_database:
-        else:
+            run_cmd = fr'"{PYTHON}" "{scriptdir}/sd-scripts/finetune/merge_captions_to_metadata.py"'
-            run_cmd += f" --caption_extension={caption_extension}"
+            if caption_extension == "":
-        run_cmd += fr' "{image_folder}"'
+                run_cmd += f' --caption_extension=".caption"'
-        run_cmd += fr' "{train_dir}/{caption_metadata_filename}"'
+            else:
-        if full_path:
+                run_cmd += f" --caption_extension={caption_extension}"
-            run_cmd += f" --full_path"
+            run_cmd += fr' "{image_folder}"'
+            run_cmd += fr' "{train_dir}/{caption_metadata_filename}"'
+            if full_path:
+                run_cmd += f" --full_path"
 
-        log.info(run_cmd)
+            log.info(run_cmd)
 
-        env = os.environ.copy()
+            env = os.environ.copy()
-        env['PYTHONPATH'] = fr"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}"
+            env['PYTHONPATH'] = fr"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}"
 
-        if not print_only_bool:
+            if not print_only_bool:
-            # Run the command
+                # Run the command
-            subprocess.run(run_cmd, shell=True, env=env)
+                subprocess.run(run_cmd, shell=True, env=env)
 
-    # create images buckets
+        # create images buckets
-    if generate_image_buckets:
+        if generate_image_buckets:
-        run_cmd = fr'"{PYTHON}" "{scriptdir}/sd-scripts/finetune/prepare_buckets_latents.py"'
+            run_cmd = fr'"{PYTHON}" "{scriptdir}/sd-scripts/finetune/prepare_buckets_latents.py"'
-        run_cmd += fr' "{image_folder}"'
+            run_cmd += fr' "{image_folder}"'
-        run_cmd += fr' "{train_dir}/{caption_metadata_filename}"'
+            run_cmd += fr' "{train_dir}/{caption_metadata_filename}"'
-        run_cmd += fr' "{train_dir}/{latent_metadata_filename}"'
+            run_cmd += fr' "{train_dir}/{latent_metadata_filename}"'
-        run_cmd += fr' "{pretrained_model_name_or_path}"'
+            run_cmd += fr' "{pretrained_model_name_or_path}"'
-        run_cmd += f" --batch_size={batch_size}"
+            run_cmd += f" --batch_size={batch_size}"
-        run_cmd += f" --max_resolution={max_resolution}"
+            run_cmd += f" --max_resolution={max_resolution}"
-        run_cmd += f" --min_bucket_reso={min_bucket_reso}"
+            run_cmd += f" --min_bucket_reso={min_bucket_reso}"
-        run_cmd += f" --max_bucket_reso={max_bucket_reso}"
+            run_cmd += f" --max_bucket_reso={max_bucket_reso}"
-        run_cmd += f" --mixed_precision={mixed_precision}"
+            run_cmd += f" --mixed_precision={mixed_precision}"
-        # if flip_aug:
+            # if flip_aug:
-        #     run_cmd += f' --flip_aug'
+            #     run_cmd += f' --flip_aug'
-        if full_path:
+            if full_path:
-            run_cmd += f" --full_path"
+                run_cmd += f" --full_path"
-        if sdxl_checkbox and sdxl_no_half_vae:
+            if sdxl_checkbox and sdxl_no_half_vae:
-            log.info("Using mixed_precision = no because no half vae is selected...")
+                log.info("Using mixed_precision = no because no half vae is selected...")
-            run_cmd += f' --mixed_precision="no"'
+                run_cmd += f' --mixed_precision="no"'
 
-        log.info(run_cmd)
+            log.info(run_cmd)
 
-        env = os.environ.copy()
+            env = os.environ.copy()
-        env['PYTHONPATH'] = fr"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}"
+            env['PYTHONPATH'] = fr"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}"
 
-        if not print_only_bool:
+            if not print_only_bool:
-            # Run the command
+                # Run the command
-            subprocess.run(run_cmd, shell=True, env=env)
+                subprocess.run(run_cmd, shell=True, env=env)
 
-    image_num = len(
+        image_num = len(
-        [
+            [
-            f
+                f
-            for f, lower_f in (
+                for f, lower_f in (
-                (file, file.lower()) for file in os.listdir(image_folder)
+                    (file, file.lower()) for file in os.listdir(image_folder)
-            )
+                )
-            if lower_f.endswith((".jpg", ".jpeg", ".png", ".webp"))
+                if lower_f.endswith((".jpg", ".jpeg", ".png", ".webp"))
-        ]
+            ]
-    )
-    log.info(f"image_num = {image_num}")
-
-    repeats = int(image_num) * int(dataset_repeats)
-    log.info(f"repeats = {str(repeats)}")
-
-    # calculate max_train_steps
-    max_train_steps = int(
-        math.ceil(
-            float(repeats)
-            / int(train_batch_size)
-            / int(gradient_accumulation_steps)
-            * int(epoch)
         )
-    )
+        log.info(f"image_num = {image_num}")
 
-    # Divide by two because flip augmentation create two copied of the source images
+        repeats = int(image_num) * int(dataset_repeats)
-    if flip_aug:
+        log.info(f"repeats = {str(repeats)}")
-        max_train_steps = int(math.ceil(float(max_train_steps) / 2))
 
-    log.info(f"max_train_steps = {max_train_steps}")
+        # calculate max_train_steps
+        max_train_steps = int(
+            math.ceil(
+                float(repeats)
+                / int(train_batch_size)
+                / int(gradient_accumulation_steps)
+                * int(epoch)
+            )
+        )
 
-    lr_warmup_steps = round(float(int(lr_warmup) * int(max_train_steps) / 100))
+        # Divide by two because flip augmentation create two copied of the source images
+        if flip_aug and max_train_steps:
+            max_train_steps = int(math.ceil(float(max_train_steps) / 2))
+
+    if max_train_steps != "":
+        log.info(f"max_train_steps = {max_train_steps}")
+        lr_warmup_steps = round(float(int(lr_warmup) * int(max_train_steps) / 100))
+    else:
+        lr_warmup_steps = 0
     log.info(f"lr_warmup_steps = {lr_warmup_steps}")
 
     run_cmd = "accelerate launch"
@@ -581,6 +593,7 @@ def train_model(
         "caption_extension": caption_extension,
         "clip_skip": clip_skip,
         "color_aug": color_aug,
+        "dataset_config": dataset_config,
         "dataset_repeats": dataset_repeats,
         "enable_bucket": True,
         "flip_aug": flip_aug,
@@ -861,6 +874,7 @@ def finetune_tab(headless=False, config: dict = {}):
             train_dir,
             image_folder,
             output_dir,
+            source_model.dataset_config,
             logging_dir,
             max_resolution,
             min_bucket_reso,
@@ -906,6 +920,7 @@ def finetune_tab(headless=False, config: dict = {}):
             output_name,
             advanced_training.max_token_length,
             basic_training.max_train_epochs,
+            basic_training.max_train_steps,
             advanced_training.max_data_loader_n_workers,
             advanced_training.full_fp16,
             advanced_training.color_aug,

kohya_gui/textual_inversion_gui.py

@@ -58,6 +58,7 @@ def save_configuration(
     train_data_dir,
     reg_data_dir,
     output_dir,
+    dataset_config,
     max_resolution,
     learning_rate,
     lr_scheduler,
@@ -192,6 +193,7 @@ def open_configuration(
     train_data_dir,
     reg_data_dir,
     output_dir,
+    dataset_config,
     max_resolution,
     learning_rate,
     lr_scheduler,
@@ -319,6 +321,7 @@ def train_model(
     train_data_dir,
     reg_data_dir,
     output_dir,
+    dataset_config,
     max_resolution,
     learning_rate,
     lr_scheduler,
@@ -424,6 +427,7 @@ def train_model(
         log_tracker_config=log_tracker_config,
         resume=resume,
         vae=vae,
+        dataset_config=dataset_config,
     ):
         return
 
@@ -440,68 +444,71 @@ def train_model(
     ):
         return
 
-    # Get a list of all subfolders in train_data_dir
+    if dataset_config:
-    subfolders = [
+        log.info("Dataset config toml file used, skipping total_steps, train_batch_size, gradient_accumulation_steps, epoch, reg_factor, max_train_steps calculations...")
-        f
+    else:
-        for f in os.listdir(train_data_dir)
+        # Get a list of all subfolders in train_data_dir
-        if os.path.isdir(os.path.join(train_data_dir, f))
+        subfolders = [
-    ]
+            f
+            for f in os.listdir(train_data_dir)
+            if os.path.isdir(os.path.join(train_data_dir, f))
+        ]
 
-    total_steps = 0
+        total_steps = 0
 
-    # Loop through each subfolder and extract the number of repeats
+        # Loop through each subfolder and extract the number of repeats
-    for folder in subfolders:
+        for folder in subfolders:
-        # Extract the number of repeats from the folder name
+            # Extract the number of repeats from the folder name
-        repeats = int(folder.split("_")[0])
+            repeats = int(folder.split("_")[0])
 
-        # Count the number of images in the folder
+            # Count the number of images in the folder
-        num_images = len(
+            num_images = len(
-            [
+                [
-                f
+                    f
-                for f, lower_f in (
+                    for f, lower_f in (
-                    (file, file.lower())
+                        (file, file.lower())
-                    for file in os.listdir(os.path.join(train_data_dir, folder))
+                        for file in os.listdir(os.path.join(train_data_dir, folder))
-                )
+                    )
-                if lower_f.endswith((".jpg", ".jpeg", ".png", ".webp"))
+                    if lower_f.endswith((".jpg", ".jpeg", ".png", ".webp"))
-            ]
+                ]
-        )
+            )
 
-        # Calculate the total number of steps for this folder
+            # Calculate the total number of steps for this folder
-        steps = repeats * num_images
+            steps = repeats * num_images
-        total_steps += steps
+            total_steps += steps
+
+            # Print the result
+            log.info(f"Folder {folder}: {steps} steps")
 
         # Print the result
-        log.info(f"Folder {folder}: {steps} steps")
+        # log.info(f"{total_steps} total steps")
 
-    # Print the result
+        if reg_data_dir == "":
-    # log.info(f"{total_steps} total steps")
+            reg_factor = 1
-
+        else:
-    if reg_data_dir == "":
+            log.info(
-        reg_factor = 1
+                "Regularisation images are used... Will double the number of steps required..."
-    else:
-        log.info(
-            "Regularisation images are used... Will double the number of steps required..."
-        )
-        reg_factor = 2
-
-    # calculate max_train_steps
-    if max_train_steps == "" or max_train_steps == "0":
-        max_train_steps = int(
-            math.ceil(
-                float(total_steps)
-                / int(train_batch_size)
-                / int(gradient_accumulation_steps)
-                * int(epoch)
-                * int(reg_factor)
             )
-        )
+            reg_factor = 2
-    else:
-        max_train_steps = int(max_train_steps)
 
-    log.info(f"max_train_steps = {max_train_steps}")
+        # calculate max_train_steps
+        if max_train_steps == "" or max_train_steps == "0":
+            max_train_steps = int(
+                math.ceil(
+                    float(total_steps)
+                    / int(train_batch_size)
+                    / int(gradient_accumulation_steps)
+                    * int(epoch)
+                    * int(reg_factor)
+                )
+            )
+        else:
+            max_train_steps = int(max_train_steps)
+
+        log.info(f"max_train_steps = {max_train_steps}")
 
     # calculate stop encoder training
-    if stop_text_encoder_training_pct == None:
+    if stop_text_encoder_training_pct == None or (not max_train_steps == "" or not max_train_steps == "0"):
         stop_text_encoder_training = 0
     else:
         stop_text_encoder_training = math.ceil(
@@ -509,7 +516,10 @@ def train_model(
         )
     log.info(f"stop_text_encoder_training = {stop_text_encoder_training}")
 
-    lr_warmup_steps = round(float(int(lr_warmup) * int(max_train_steps) / 100))
+    if not max_train_steps == "":
+        lr_warmup_steps = round(float(int(lr_warmup) * int(max_train_steps) / 100))
+    else:
+        lr_warmup_steps = 0
     log.info(f"lr_warmup_steps = {lr_warmup_steps}")
 
     run_cmd = "accelerate launch"
@@ -538,6 +548,7 @@ def train_model(
         caption_extension=caption_extension,
         clip_skip=clip_skip,
         color_aug=color_aug,
+        dataset_config=dataset_config,
         enable_bucket=enable_bucket,
         epoch=epoch,
         flip_aug=flip_aug,
@@ -843,6 +854,7 @@ def ti_tab(headless=False, default_output_dir=None, config: dict = {}):
             source_model.train_data_dir,
             folders.reg_data_dir,
             folders.output_dir,
+            source_model.dataset_config,
             basic_training.max_resolution,
             basic_training.learning_rate,
             basic_training.lr_scheduler,

setup/setup_windows.py

@@ -24,7 +24,10 @@ def cudnn_install():
     )
 
     # Original path with "..\\venv"
-    original_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..\\venv\\Lib\\site-packages\\nvidia\\cudnn\\bin")
+    original_path = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)),
+        "..\\venv\\Lib\\site-packages\\nvidia\\cudnn\\bin",
+    )
     # Normalize the path to resolve "..\\venv"
     cudnn_src = os.path.abspath(original_path)
     cudnn_dest = os.path.join(sysconfig.get_paths()["purelib"], "torch", "lib")
@@ -35,7 +38,7 @@ def cudnn_install():
             # check for different files
             filecmp.clear_cache()
             for file in os.listdir(cudnn_src):
-                if file.lower().endswith('.dll'):  # Check if the file is a .dll file
+                if file.lower().endswith(".dll"):  # Check if the file is a .dll file
                     src_file = os.path.join(cudnn_src, file)
                     dest_file = os.path.join(cudnn_dest, file)
                     # if dest file exists, check if it's different
@@ -119,7 +122,9 @@ def install_kohya_ss_torch2(headless: bool = False):
         "requirements_windows_torch2.txt", check_no_verify_flag=False
     )
 
-    setup_common.configure_accelerate(run_accelerate=not headless) # False if headless is True and vice versa
+    setup_common.configure_accelerate(
+        run_accelerate=not headless
+    )  # False if headless is True and vice versa
 
 
 def install_bitsandbytes_0_35_0():
@@ -147,6 +152,7 @@ def install_bitsandbytes_0_41_1():
         reinstall=True,
     )
 
+
 def install_bitsandbytes_0_41_2():
     log.info("Installing bitsandbytes 0.41.2...")
     setup_common.install(
@@ -155,21 +161,34 @@ def install_bitsandbytes_0_41_2():
         reinstall=True,
     )
 
+
+def install_triton_2_1_0():
+    log.info("Installing triton 2.1.0...")
+    setup_common.install(
+        "--upgrade https://huggingface.co/Rodeszones/CogVLM-grounding-generalist-hf-quant4/resolve/main/triton-2.1.0-cp310-cp310-win_amd64.whl?download=true",
+        "triton 2.1.0",
+        reinstall=True,
+    )
+
+
 def main_menu(headless: bool = False):
     if headless:
         install_kohya_ss_torch2(headless=headless)
     else:
         setup_common.clear_screen()
         while True:
-            print("\nKohya_ss GUI setup menu:\n")
+            print("\nKohya_ss setup menu:\n")
-            print("1. Install kohya_ss gui")
+            print("1. Install kohya_ss GUI")
-            print("2. (Optional) Install cudnn files (if you want to use latest supported cudnn version)")
+            print(
-            print("3. (Optional) Install specific bitsandbytes versions")
+                "2. (Optional) Install CuDNN files (to use the latest supported CuDNN version)"
-            print("4. (Optional) Manually configure accelerate")
+            )
-            print("5. (Optional) Start Kohya_ss GUI in browser")
+            print("3. (Optional) Install Triton 2.1.0 for Windows")
-            print("6. Quit")
+            print("4. (Optional) Install specific version of bitsandbytes")
+            print("5. (Optional) Manually configure Accelerate")
+            print("6. (Optional) Launch Kohya_ss GUI in browser")
+            print("7. Exit Setup")
 
-            choice = input("\nEnter your choice: ")
+            choice = input("\nSelect an option: ")
             print("")
 
             if choice == "1":
@@ -177,22 +196,25 @@ def main_menu(headless: bool = False):
             elif choice == "2":
                 cudnn_install()
             elif choice == "3":
+                install_triton_2_1_0()
+            elif choice == "4":
                 while True:
-                    print("1. (Optional) Force installation of bitsandbytes 0.35.0")
+                    print("\nBitsandBytes Installation Menu:")
+                    print("1. Force install Bitsandbytes 0.35.0")
                     print(
-                        "2. (Optional) Force installation of bitsandbytes 0.40.1 for new optimizer options support and pre-bugfix results"
+                        "2. Force install Bitsandbytes 0.40.1 (supports new optimizer options, pre-bugfix results)"
                     )
                     print(
-                        "3. (Optional) Force installation of bitsandbytes 0.41.1 for new optimizer options support"
+                        "3. Force installation Bitsandbytes 0.41.1 (supports new optimizer options)"
                     )
                     print(
-                        "4. (Recommended) Force installation of bitsandbytes 0.41.2 for new optimizer options support"
+                        "4. (Recommended) Force install Bitsandbytes 0.41.2 (supports new optimizer options)"
                     )
                     print(
-                        "5. (Danger) Install bitsandbytes-windows (this package has been reported to cause issues for most... avoid...)"
+                        "5. (Warning) Install bitsandbytes-windows (may cause issues, use with caution)"
                     )
-                    print("6. Exit")
+                    print("6. Return to Previous Menu:")
-                    choice_torch = input("\nEnter your choice: ")
+                    choice_torch = input("\nSelect an option: ")
                     print("")
 
                     if choice_torch == "1":
@@ -215,18 +237,18 @@ def main_menu(headless: bool = False):
                     elif choice_torch == "6":
                         break
                     else:
-                        print("Invalid choice. Please enter a number between 1-3.")
+                        print("Invalid choice. Please chose an option between 1-6.")
-            elif choice == "4":
-                setup_common.run_cmd("accelerate config")
             elif choice == "5":
+                setup_common.run_cmd("accelerate config")
+            elif choice == "6":
                 subprocess.Popen(
                     "start cmd /k .\gui.bat --inbrowser", shell=True
                 )  # /k keep the terminal open on quit. /c would close the terminal instead
-            elif choice == "6":
+            elif choice == "7":
-                print("Quitting the program.")
+                print("Exiting setup.")
                 break
             else:
-                print("Invalid choice. Please enter a number between 1-5.")
+                print("Invalid selection. Please choose an option between 1-7.")
 
 
 if __name__ == "__main__":
@@ -235,7 +257,7 @@ if __name__ == "__main__":
 
     # Setup argument parser
     parser = argparse.ArgumentParser(description="Your Script Description")
-    parser.add_argument('--headless', action='store_true', help='Run in headless mode')
+    parser.add_argument("--headless", action="store_true", help="Run in headless mode")
 
     # Parse arguments
     args = parser.parse_args()

test/config/Diag-OFT-AdamW8bit-toml.json

@@ -0,0 +1,131 @@
+{
+  "LoRA_type": "LyCORIS/Diag-OFT",
+  "LyCORIS_preset": "full",
+  "adaptive_noise_scale": 0,
+  "additional_parameters": "",
+  "block_alphas": "",
+  "block_dims": "",
+  "block_lr_zero_threshold": "",
+  "bucket_no_upscale": true,
+  "bucket_reso_steps": 64,
+  "bypass_mode": true,
+  "cache_latents": true,
+  "cache_latents_to_disk": false,
+  "caption_dropout_every_n_epochs": 0.0,
+  "caption_dropout_rate": 0.05,
+  "caption_extension": "",
+  "clip_skip": 2,
+  "color_aug": false,
+  "constrain": 0.0,
+  "conv_alpha": 4,
+  "conv_block_alphas": "",
+  "conv_block_dims": "",
+  "conv_dim": 8,
+  "dataset_config": "D:/kohya_ss/test/config/dataset.toml",
+  "debiased_estimation_loss": false,
+  "decompose_both": false,
+  "dim_from_weights": false,
+  "down_lr_weight": "",
+  "enable_bucket": true,
+  "epoch": 1,
+  "factor": -1,
+  "flip_aug": false,
+  "fp8_base": false,
+  "full_bf16": false,
+  "full_fp16": false,
+  "gpu_ids": "",
+  "gradient_accumulation_steps": 1,
+  "gradient_checkpointing": false,
+  "keep_tokens": "0",
+  "learning_rate": 0.0005,
+  "log_tracker_config": "",
+  "log_tracker_name": "",
+  "logging_dir": "./test/logs",
+  "lora_network_weights": "",
+  "lr_scheduler": "constant",
+  "lr_scheduler_args": "",
+  "lr_scheduler_num_cycles": "",
+  "lr_scheduler_power": "",
+  "lr_warmup": 0,
+  "max_bucket_reso": 2048,
+  "max_data_loader_n_workers": "0",
+  "max_grad_norm": 1,
+  "max_resolution": "512,512",
+  "max_timestep": 1000,
+  "max_token_length": "75",
+  "max_train_epochs": "",
+  "max_train_steps": "",
+  "mem_eff_attn": false,
+  "mid_lr_weight": "",
+  "min_bucket_reso": 256,
+  "min_snr_gamma": 0,
+  "min_timestep": 0,
+  "mixed_precision": "bf16",
+  "model_list": "runwayml/stable-diffusion-v1-5",
+  "module_dropout": 0,
+  "multi_gpu": false,
+  "multires_noise_discount": 0,
+  "multires_noise_iterations": 0,
+  "network_alpha": 16,
+  "network_dim": 32,
+  "network_dropout": 0,
+  "noise_offset": 0.05,
+  "noise_offset_type": "Original",
+  "num_cpu_threads_per_process": 2,
+  "num_machines": 1,
+  "num_processes": 1,
+  "optimizer": "AdamW8bit",
+  "optimizer_args": "",
+  "output_dir": "./test/output",
+  "output_name": "Diag-OFT-AdamW8bit-toml",
+  "persistent_data_loader_workers": false,
+  "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
+  "prior_loss_weight": 1.0,
+  "random_crop": false,
+  "rank_dropout": 0,
+  "rank_dropout_scale": false,
+  "reg_data_dir": "",
+  "rescaled": false,
+  "resume": "",
+  "sample_every_n_epochs": 0,
+  "sample_every_n_steps": 25,
+  "sample_prompts": "a painting of a gas mask , by darius kawasaki",
+  "sample_sampler": "euler_a",
+  "save_every_n_epochs": 1,
+  "save_every_n_steps": 0,
+  "save_last_n_steps": 0,
+  "save_last_n_steps_state": 0,
+  "save_model_as": "safetensors",
+  "save_precision": "fp16",
+  "save_state": false,
+  "scale_v_pred_loss_like_noise_pred": false,
+  "scale_weight_norms": 0,
+  "sdxl": false,
+  "sdxl_cache_text_encoder_outputs": false,
+  "sdxl_no_half_vae": true,
+  "seed": "1234",
+  "shuffle_caption": false,
+  "stop_text_encoder_training": 0,
+  "text_encoder_lr": 0.0,
+  "train_batch_size": 4,
+  "train_data_dir": "",
+  "train_norm": false,
+  "train_on_input": true,
+  "training_comment": "",
+  "unet_lr": 0.0,
+  "unit": 1,
+  "up_lr_weight": "",
+  "use_cp": false,
+  "use_scalar": false,
+  "use_tucker": false,
+  "use_wandb": false,
+  "v2": false,
+  "v_parameterization": false,
+  "v_pred_like_loss": 0,
+  "vae": "",
+  "vae_batch_size": 0,
+  "wandb_api_key": "",
+  "wandb_run_name": "",
+  "weighted_captions": false,
+  "xformers": "xformers"
+}

test/config/LoKR-AdamW8bit-toml.json

@@ -0,0 +1,132 @@
+{
+  "LoRA_type": "LyCORIS/LoKr",
+  "LyCORIS_preset": "full",
+  "adaptive_noise_scale": 0,
+  "additional_parameters": "--lr_scheduler_type \"CosineAnnealingLR\" --lr_scheduler_args \"T_max=1000\" \"eta_min=0e-0\"",
+  "block_alphas": "",
+  "block_dims": "",
+  "block_lr_zero_threshold": "",
+  "bucket_no_upscale": true,
+  "bucket_reso_steps": 1,
+  "bypass_mode": false,
+  "cache_latents": true,
+  "cache_latents_to_disk": true,
+  "caption_dropout_every_n_epochs": 0.0,
+  "caption_dropout_rate": 0.1,
+  "caption_extension": ".txt",
+  "clip_skip": "1",
+  "color_aug": false,
+  "constrain": 0.0,
+  "conv_alpha": 1,
+  "conv_block_alphas": "",
+  "conv_block_dims": "",
+  "conv_dim": 100000,
+  "dataset_config": "D:/kohya_ss/test/config/dataset.toml",
+  "debiased_estimation_loss": false,
+  "decompose_both": false,
+  "dim_from_weights": false,
+  "dora_wd": false,
+  "down_lr_weight": "",
+  "enable_bucket": true,
+  "epoch": 150,
+  "factor": 6,
+  "flip_aug": false,
+  "fp8_base": false,
+  "full_bf16": false,
+  "full_fp16": false,
+  "gpu_ids": "",
+  "gradient_accumulation_steps": 1,
+  "gradient_checkpointing": false,
+  "keep_tokens": 1,
+  "learning_rate": 1.0,
+  "log_tracker_config": "",
+  "log_tracker_name": "",
+  "logging_dir": "./test/logs",
+  "lora_network_weights": "",
+  "lr_scheduler": "cosine",
+  "lr_scheduler_args": "",
+  "lr_scheduler_num_cycles": "",
+  "lr_scheduler_power": "",
+  "lr_warmup": 0,
+  "max_bucket_reso": 2048,
+  "max_data_loader_n_workers": "0",
+  "max_grad_norm": 1,
+  "max_resolution": "512,512",
+  "max_timestep": 1000,
+  "max_token_length": "75",
+  "max_train_epochs": "",
+  "max_train_steps": "",
+  "mem_eff_attn": false,
+  "mid_lr_weight": "",
+  "min_bucket_reso": 256,
+  "min_snr_gamma": 5,
+  "min_timestep": 0,
+  "mixed_precision": "bf16",
+  "model_list": "custom",
+  "module_dropout": 0,
+  "multi_gpu": false,
+  "multires_noise_discount": 0.1,
+  "multires_noise_iterations": 6,
+  "network_alpha": 1,
+  "network_dim": 100000,
+  "network_dropout": 0,
+  "noise_offset": 0,
+  "noise_offset_type": "Multires",
+  "num_cpu_threads_per_process": 2,
+  "num_machines": 1,
+  "num_processes": 1,
+  "optimizer": "Prodigy",
+  "optimizer_args": "\"d0=1e-5\" \"d_coef=1.0\" \"weight_decay=0.4\" \"decouple=True\" \"safeguard_warmup=True\" \"use_bias_correction=True\"",
+  "output_dir": "./test/output",
+  "output_name": "LoKR-AdamW8bit-toml",
+  "persistent_data_loader_workers": false,
+  "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
+  "prior_loss_weight": 1.0,
+  "random_crop": false,
+  "rank_dropout": 0,
+  "rank_dropout_scale": false,
+  "reg_data_dir": "",
+  "rescaled": false,
+  "resume": "",
+  "sample_every_n_epochs": 0,
+  "sample_every_n_steps": 25,
+  "sample_prompts": "a painting of a gas mask , by darius kawasaki",
+  "sample_sampler": "euler_a",
+  "save_every_n_epochs": 15,
+  "save_every_n_steps": 0,
+  "save_last_n_steps": 0,
+  "save_last_n_steps_state": 0,
+  "save_model_as": "safetensors",
+  "save_precision": "bf16",
+  "save_state": false,
+  "scale_v_pred_loss_like_noise_pred": false,
+  "scale_weight_norms": 0,
+  "sdxl": false,
+  "sdxl_cache_text_encoder_outputs": false,
+  "sdxl_no_half_vae": true,
+  "seed": "",
+  "shuffle_caption": true,
+  "stop_text_encoder_training": 0,
+  "text_encoder_lr": 1.0,
+  "train_batch_size": 2,
+  "train_data_dir": "",
+  "train_norm": false,
+  "train_on_input": false,
+  "training_comment": "KoopaTroopa",
+  "unet_lr": 1.0,
+  "unit": 1,
+  "up_lr_weight": "",
+  "use_cp": false,
+  "use_scalar": false,
+  "use_tucker": false,
+  "use_wandb": false,
+  "v2": false,
+  "v_parameterization": false,
+  "v_pred_like_loss": 0,
+  "vae": "",
+  "vae_batch_size": 0,
+  "wandb_api_key": "",
+  "wandb_run_name": "",
+  "weighted_captions": false,
+  "xformers": "xformers"
+}

test/config/TI-AdamW8bit-toml.json

@@ -0,0 +1,96 @@
+{
+  "adaptive_noise_scale": 0,
+  "additional_parameters": "",
+  "bucket_no_upscale": true,
+  "bucket_reso_steps": 1,
+  "cache_latents": true,
+  "cache_latents_to_disk": false,
+  "caption_dropout_every_n_epochs": 0.0,
+  "caption_dropout_rate": 0.05,
+  "caption_extension": "",
+  "clip_skip": 2,
+  "color_aug": false,
+  "dataset_config": "D:/kohya_ss/test/config/dataset.toml",
+  "enable_bucket": true,
+  "epoch": 4,
+  "flip_aug": false,
+  "full_fp16": false,
+  "gpu_ids": "",
+  "gradient_accumulation_steps": 1,
+  "gradient_checkpointing": false,
+  "init_word": "*",
+  "keep_tokens": "0",
+  "learning_rate": 0.0001,
+  "log_tracker_config": "",
+  "log_tracker_name": "",
+  "logging_dir": "./test/logs",
+  "lr_scheduler": "cosine",
+  "lr_scheduler_args": "",
+  "lr_scheduler_num_cycles": "",
+  "lr_scheduler_power": "",
+  "lr_warmup": 0,
+  "max_bucket_reso": 2048,
+  "max_data_loader_n_workers": "0",
+  "max_resolution": "512,512",
+  "max_timestep": 1000,
+  "max_token_length": "75",
+  "max_train_epochs": "",
+  "max_train_steps": "80",
+  "mem_eff_attn": false,
+  "min_bucket_reso": 256,
+  "min_snr_gamma": 10,
+  "min_timestep": 0,
+  "mixed_precision": "bf16",
+  "model_list": "runwayml/stable-diffusion-v1-5",
+  "multi_gpu": false,
+  "multires_noise_discount": 0.2,
+  "multires_noise_iterations": 8,
+  "no_token_padding": false,
+  "noise_offset": 0.05,
+  "noise_offset_type": "Multires",
+  "num_cpu_threads_per_process": 2,
+  "num_machines": 1,
+  "num_processes": 1,
+  "num_vectors_per_token": 8,
+  "optimizer": "AdamW8bit",
+  "optimizer_args": "",
+  "output_dir": "./test/output",
+  "output_name": "TI-Adamw8bit-toml",
+  "persistent_data_loader_workers": false,
+  "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
+  "prior_loss_weight": 1.0,
+  "random_crop": false,
+  "reg_data_dir": "",
+  "resume": "",
+  "sample_every_n_epochs": 0,
+  "sample_every_n_steps": 20,
+  "sample_prompts": "a painting of man wearing a gas mask , by darius kawasaki",
+  "sample_sampler": "euler_a",
+  "save_every_n_epochs": 1,
+  "save_every_n_steps": 0,
+  "save_last_n_steps": 0,
+  "save_last_n_steps_state": 0,
+  "save_model_as": "safetensors",
+  "save_precision": "fp16",
+  "save_state": false,
+  "scale_v_pred_loss_like_noise_pred": false,
+  "sdxl": false,
+  "sdxl_no_half_vae": false,
+  "seed": "1234",
+  "shuffle_caption": false,
+  "stop_text_encoder_training": 0,
+  "template": "style template",
+  "token_string": "zxc",
+  "train_batch_size": 4,
+  "train_data_dir": "",
+  "use_wandb": false,
+  "v2": false,
+  "v_parameterization": false,
+  "v_pred_like_loss": 0,
+  "vae": "",
+  "vae_batch_size": 0,
+  "wandb_api_key": "",
+  "wandb_run_name": "",
+  "weights": "",
+  "xformers": "xformers"
+}

test/config/TI-AdamW8bit.json

@@ -0,0 +1,96 @@
+{
+  "adaptive_noise_scale": 0,
+  "additional_parameters": "",
+  "bucket_no_upscale": true,
+  "bucket_reso_steps": 1,
+  "cache_latents": true,
+  "cache_latents_to_disk": false,
+  "caption_dropout_every_n_epochs": 0.0,
+  "caption_dropout_rate": 0.05,
+  "caption_extension": "",
+  "clip_skip": 2,
+  "color_aug": false,
+  "dataset_config": "",
+  "enable_bucket": true,
+  "epoch": 4,
+  "flip_aug": false,
+  "full_fp16": false,
+  "gpu_ids": "",
+  "gradient_accumulation_steps": 1,
+  "gradient_checkpointing": false,
+  "init_word": "*",
+  "keep_tokens": "0",
+  "learning_rate": 0.0001,
+  "log_tracker_config": "",
+  "log_tracker_name": "",
+  "logging_dir": "./test/logs",
+  "lr_scheduler": "cosine",
+  "lr_scheduler_args": "",
+  "lr_scheduler_num_cycles": "",
+  "lr_scheduler_power": "",
+  "lr_warmup": 0,
+  "max_bucket_reso": 2048,
+  "max_data_loader_n_workers": "0",
+  "max_resolution": "512,512",
+  "max_timestep": 1000,
+  "max_token_length": "75",
+  "max_train_epochs": "",
+  "max_train_steps": "",
+  "mem_eff_attn": false,
+  "min_bucket_reso": 256,
+  "min_snr_gamma": 10,
+  "min_timestep": 0,
+  "mixed_precision": "bf16",
+  "model_list": "runwayml/stable-diffusion-v1-5",
+  "multi_gpu": false,
+  "multires_noise_discount": 0.2,
+  "multires_noise_iterations": 8,
+  "no_token_padding": false,
+  "noise_offset": 0.05,
+  "noise_offset_type": "Multires",
+  "num_cpu_threads_per_process": 2,
+  "num_machines": 1,
+  "num_processes": 1,
+  "num_vectors_per_token": 8,
+  "optimizer": "AdamW8bit",
+  "optimizer_args": "",
+  "output_dir": "./test/output",
+  "output_name": "TI-Adamw8bit",
+  "persistent_data_loader_workers": false,
+  "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
+  "prior_loss_weight": 1.0,
+  "random_crop": false,
+  "reg_data_dir": "",
+  "resume": "",
+  "sample_every_n_epochs": 0,
+  "sample_every_n_steps": 20,
+  "sample_prompts": "a painting of man wearing a gas mask , by darius kawasaki",
+  "sample_sampler": "euler_a",
+  "save_every_n_epochs": 1,
+  "save_every_n_steps": 0,
+  "save_last_n_steps": 0,
+  "save_last_n_steps_state": 0,
+  "save_model_as": "safetensors",
+  "save_precision": "fp16",
+  "save_state": false,
+  "scale_v_pred_loss_like_noise_pred": false,
+  "sdxl": false,
+  "sdxl_no_half_vae": false,
+  "seed": "1234",
+  "shuffle_caption": false,
+  "stop_text_encoder_training": 0,
+  "template": "style template",
+  "token_string": "zxc",
+  "train_batch_size": 4,
+  "train_data_dir": "./test/img",
+  "use_wandb": false,
+  "v2": false,
+  "v_parameterization": false,
+  "v_pred_like_loss": 0,
+  "vae": "",
+  "vae_batch_size": 0,
+  "wandb_api_key": "",
+  "wandb_run_name": "",
+  "weights": "",
+  "xformers": "xformers"
+}

test/config/dataset-finetune.toml

@@ -0,0 +1,14 @@
+[[datasets]]
+resolution = 512
+batch_size = 4
+keep_tokens = 1
+enable_bucket = true
+min_bucket_reso = 64
+max_bucket_reso = 1024
+bucket_reso_steps = 32
+bucket_no_upscale = false
+
+  [[datasets.subsets]]
+  image_dir = '.\test\img\10_darius kawasaki person'
+  num_repeats = 10
+  metadata_file = '.\test\config\meta-1_lat.json'

test/config/dataset.toml

@@ -0,0 +1,15 @@
+[[datasets]]
+resolution = 512
+batch_size = 4
+keep_tokens = 1
+enable_bucket = true
+min_bucket_reso = 64
+max_bucket_reso = 1024
+bucket_reso_steps = 32
+bucket_no_upscale = true
+
+  [[datasets.subsets]]
+  image_dir = '.\test\img\10_darius kawasaki person'
+  num_repeats = 10
+  class_tokens = 'darius kawasaki person'
+  caption_extension = '.txt'

test/config/dreambooth-AdamW8bit-toml.json

@@ -0,0 +1,95 @@
+{
+  "adaptive_noise_scale": 0,
+  "additional_parameters": "",
+  "bucket_no_upscale": true,
+  "bucket_reso_steps": 64,
+  "cache_latents": true,
+  "cache_latents_to_disk": false,
+  "caption_dropout_every_n_epochs": 0.0,
+  "caption_dropout_rate": 0.05,
+  "caption_extension": "",
+  "clip_skip": 2,
+  "color_aug": false,
+  "dataset_config": "D:/kohya_ss/test/config/dataset.toml",
+  "enable_bucket": true,
+  "epoch": 1,
+  "flip_aug": false,
+  "full_bf16": false,
+  "full_fp16": false,
+  "gpu_ids": "",
+  "gradient_accumulation_steps": 1,
+  "gradient_checkpointing": false,
+  "keep_tokens": "0",
+  "learning_rate": 5e-05,
+  "learning_rate_te": 1e-05,
+  "learning_rate_te1": 1e-05,
+  "learning_rate_te2": 1e-05,
+  "log_tracker_config": "",
+  "log_tracker_name": "",
+  "logging_dir": "./test/logs",
+  "lr_scheduler": "constant",
+  "lr_scheduler_args": "",
+  "lr_scheduler_num_cycles": "",
+  "lr_scheduler_power": "",
+  "lr_warmup": 0,
+  "max_bucket_reso": 2048,
+  "max_data_loader_n_workers": "0",
+  "max_resolution": "512,512",
+  "max_timestep": 1000,
+  "max_token_length": "75",
+  "max_train_epochs": "",
+  "max_train_steps": "",
+  "mem_eff_attn": false,
+  "min_bucket_reso": 256,
+  "min_snr_gamma": 0,
+  "min_timestep": 0,
+  "mixed_precision": "bf16",
+  "model_list": "runwayml/stable-diffusion-v1-5",
+  "multi_gpu": false,
+  "multires_noise_discount": 0,
+  "multires_noise_iterations": 0,
+  "no_token_padding": false,
+  "noise_offset": 0.05,
+  "noise_offset_type": "Original",
+  "num_cpu_threads_per_process": 2,
+  "num_machines": 1,
+  "num_processes": 1,
+  "optimizer": "AdamW8bit",
+  "optimizer_args": "",
+  "output_dir": "./test/output",
+  "output_name": "db-AdamW8bit-toml",
+  "persistent_data_loader_workers": false,
+  "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
+  "prior_loss_weight": 1.0,
+  "random_crop": false,
+  "reg_data_dir": "",
+  "resume": "",
+  "sample_every_n_epochs": 0,
+  "sample_every_n_steps": 25,
+  "sample_prompts": "a painting of a gas mask , by darius kawasaki",
+  "sample_sampler": "euler_a",
+  "save_every_n_epochs": 1,
+  "save_every_n_steps": 0,
+  "save_last_n_steps": 0,
+  "save_last_n_steps_state": 0,
+  "save_model_as": "safetensors",
+  "save_precision": "fp16",
+  "save_state": false,
+  "scale_v_pred_loss_like_noise_pred": false,
+  "sdxl": false,
+  "seed": "1234",
+  "shuffle_caption": false,
+  "stop_text_encoder_training": 0,
+  "train_batch_size": 4,
+  "train_data_dir": "",
+  "use_wandb": false,
+  "v2": false,
+  "v_parameterization": false,
+  "v_pred_like_loss": 0,
+  "vae": "",
+  "vae_batch_size": 0,
+  "wandb_api_key": "",
+  "wandb_run_name": "",
+  "weighted_captions": false,
+  "xformers": "xformers"
+}

test/config/dreambooth-AdamW8bit.json

@@ -1,11 +1,6 @@
 {
-  "LoRA_type": "Standard",
-  "LyCORIS_preset": "full",
   "adaptive_noise_scale": 0,
   "additional_parameters": "",
-  "block_alphas": "",
-  "block_dims": "",
-  "block_lr_zero_threshold": "",
   "bucket_no_upscale": true,
   "bucket_reso_steps": 64,
   "cache_latents": true,
@@ -15,20 +10,10 @@
   "caption_extension": "",
   "clip_skip": 2,
   "color_aug": false,
-  "constrain": 0.0,
+  "dataset_config": "",
-  "conv_alpha": 1,
-  "conv_block_alphas": "",
-  "conv_block_dims": "",
-  "conv_dim": 1,
-  "debiased_estimation_loss": false,
-  "decompose_both": false,
-  "dim_from_weights": false,
-  "down_lr_weight": "",
   "enable_bucket": true,
   "epoch": 1,
-  "factor": -1,
   "flip_aug": false,
-  "fp8_base": false,
   "full_bf16": false,
   "full_fp16": false,
   "gpu_ids": "",
@@ -36,10 +21,12 @@
   "gradient_checkpointing": false,
   "keep_tokens": "0",
   "learning_rate": 5e-05,
+  "learning_rate_te": 1e-05,
+  "learning_rate_te1": 1e-05,
+  "learning_rate_te2": 1e-05,
   "log_tracker_config": "",
   "log_tracker_name": "",
   "logging_dir": "./test/logs",
-  "lora_network_weights": "",
   "lr_scheduler": "constant",
   "lr_scheduler_args": "",
   "lr_scheduler_num_cycles": "",
@@ -47,26 +34,21 @@
   "lr_warmup": 0,
   "max_bucket_reso": 2048,
   "max_data_loader_n_workers": "0",
-  "max_grad_norm": 1,
   "max_resolution": "512,512",
   "max_timestep": 1000,
   "max_token_length": "75",
   "max_train_epochs": "",
   "max_train_steps": "",
   "mem_eff_attn": false,
-  "mid_lr_weight": "",
   "min_bucket_reso": 256,
   "min_snr_gamma": 0,
   "min_timestep": 0,
   "mixed_precision": "bf16",
   "model_list": "runwayml/stable-diffusion-v1-5",
-  "module_dropout": 0,
   "multi_gpu": false,
   "multires_noise_discount": 0,
   "multires_noise_iterations": 0,
-  "network_alpha": 1,
+  "no_token_padding": false,
-  "network_dim": 8,
-  "network_dropout": 0,
   "noise_offset": 0.05,
   "noise_offset_type": "Original",
   "num_cpu_threads_per_process": 2,
@@ -80,10 +62,7 @@
   "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
   "prior_loss_weight": 1.0,
   "random_crop": false,
-  "rank_dropout": 0,
-  "rank_dropout_scale": false,
   "reg_data_dir": "",
-  "rescaled": false,
   "resume": "",
   "sample_every_n_epochs": 0,
   "sample_every_n_steps": 25,
@@ -97,30 +76,17 @@
   "save_precision": "fp16",
   "save_state": false,
   "scale_v_pred_loss_like_noise_pred": false,
-  "scale_weight_norms": 0,
   "sdxl": false,
-  "sdxl_cache_text_encoder_outputs": false,
-  "sdxl_no_half_vae": true,
   "seed": "1234",
   "shuffle_caption": false,
   "stop_text_encoder_training": 0,
-  "text_encoder_lr": 0.0,
   "train_batch_size": 4,
   "train_data_dir": "./test/img",
-  "train_norm": false,
-  "train_on_input": true,
-  "training_comment": "",
-  "unet_lr": 0.0,
-  "unit": 1,
-  "up_lr_weight": "",
-  "use_cp": false,
-  "use_scalar": false,
-  "use_tucker": false,
   "use_wandb": false,
   "v2": false,
   "v_parameterization": false,
   "v_pred_like_loss": 0,
-  "vae": "",
+  "vae": "stabilityai/vae",
   "vae_batch_size": 0,
   "wandb_api_key": "",
   "wandb_run_name": "",

test/config/finetune-AdamW-toml.json

@@ -0,0 +1,100 @@
+{
+  "adaptive_noise_scale": 0,
+  "additional_parameters": "",
+  "batch_size": "8",
+  "block_lr": "",
+  "bucket_no_upscale": false,
+  "bucket_reso_steps": 1,
+  "cache_latents": true,
+  "cache_latents_to_disk": false,
+  "caption_dropout_every_n_epochs": 0.0,
+  "caption_dropout_rate": 0,
+  "caption_extension": ".txt",
+  "caption_metadata_filename": "meta-1_cap.json",
+  "clip_skip": 1,
+  "color_aug": false,
+  "create_buckets": false,
+  "create_caption": true,
+  "dataset_config": "D:/kohya_ss/test/config/dataset-finetune.toml",
+  "dataset_repeats": "50",
+  "epoch": 2,
+  "flip_aug": false,
+  "full_bf16": false,
+  "full_fp16": false,
+  "full_path": true,
+  "gpu_ids": "",
+  "gradient_accumulation_steps": 1.0,
+  "gradient_checkpointing": false,
+  "image_folder": "",
+  "keep_tokens": 0,
+  "latent_metadata_filename": "meta-1_lat.json",
+  "learning_rate": 1e-05,
+  "learning_rate_te": 5e-06,
+  "learning_rate_te1": 5e-06,
+  "learning_rate_te2": 0.0,
+  "log_tracker_config": "",
+  "log_tracker_name": "",
+  "logging_dir": "./test/ft",
+  "lr_scheduler": "cosine_with_restarts",
+  "lr_scheduler_args": "",
+  "lr_warmup": 10,
+  "max_bucket_reso": "1024",
+  "max_data_loader_n_workers": "0",
+  "max_resolution": "512,512",
+  "max_timestep": 1000,
+  "max_token_length": "75",
+  "max_train_epochs": "",
+  "max_train_steps": "20",
+  "mem_eff_attn": false,
+  "min_bucket_reso": "256",
+  "min_snr_gamma": 0,
+  "min_timestep": 0,
+  "mixed_precision": "bf16",
+  "model_list": "runwayml/stable-diffusion-v1-5",
+  "multi_gpu": false,
+  "multires_noise_discount": 0,
+  "multires_noise_iterations": 0,
+  "noise_offset": 0,
+  "noise_offset_type": "Original",
+  "num_cpu_threads_per_process": 2,
+  "num_machines": 1,
+  "num_processes": 1,
+  "optimizer": "AdamW",
+  "optimizer_args": "",
+  "output_dir": "./test/output",
+  "output_name": "test_ft-toml",
+  "persistent_data_loader_workers": false,
+  "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
+  "random_crop": false,
+  "resume": "",
+  "sample_every_n_epochs": 0,
+  "sample_every_n_steps": 0,
+  "sample_prompts": "",
+  "sample_sampler": "euler_a",
+  "save_every_n_epochs": 1,
+  "save_every_n_steps": 0,
+  "save_last_n_steps": 0,
+  "save_last_n_steps_state": 0,
+  "save_model_as": "safetensors",
+  "save_precision": "bf16",
+  "save_state": false,
+  "scale_v_pred_loss_like_noise_pred": false,
+  "sdxl_cache_text_encoder_outputs": false,
+  "sdxl_checkbox": false,
+  "sdxl_no_half_vae": false,
+  "seed": "1234",
+  "shuffle_caption": false,
+  "train_batch_size": 4,
+  "train_dir": "./test",
+  "train_text_encoder": true,
+  "use_latent_files": "No",
+  "use_wandb": false,
+  "v2": false,
+  "v_parameterization": false,
+  "v_pred_like_loss": 0,
+  "vae_batch_size": 0,
+  "wandb_api_key": "",
+  "wandb_run_name": "",
+  "weighted_captions": false,
+  "xformers": "xformers"
+}

test/config/finetune-AdamW.json

@@ -15,12 +15,14 @@
   "color_aug": false,
   "create_buckets": false,
   "create_caption": true,
+  "dataset_config": "",
   "dataset_repeats": "50",
   "epoch": 2,
   "flip_aug": false,
   "full_bf16": false,
   "full_fp16": false,
   "full_path": true,
+  "gpu_ids": "",
   "gradient_accumulation_steps": 1.0,
   "gradient_checkpointing": false,
   "image_folder": ".\\test\\img\\10_darius kawasaki person",
@@ -30,6 +32,8 @@
   "learning_rate_te": 5e-06,
   "learning_rate_te1": 5e-06,
   "learning_rate_te2": 0.0,
+  "log_tracker_config": "",
+  "log_tracker_name": "",
   "logging_dir": "./test/ft",
   "lr_scheduler": "cosine_with_restarts",
   "lr_scheduler_args": "",
@@ -40,17 +44,21 @@
   "max_timestep": 1000,
   "max_token_length": "75",
   "max_train_epochs": "",
+  "max_train_steps": "",
   "mem_eff_attn": false,
   "min_bucket_reso": "256",
   "min_snr_gamma": 0,
   "min_timestep": 0,
   "mixed_precision": "bf16",
   "model_list": "runwayml/stable-diffusion-v1-5",
+  "multi_gpu": false,
   "multires_noise_discount": 0,
   "multires_noise_iterations": 0,
   "noise_offset": 0,
   "noise_offset_type": "Original",
   "num_cpu_threads_per_process": 2,
+  "num_machines": 1,
+  "num_processes": 1,
   "optimizer": "AdamW",
   "optimizer_args": "",
   "output_dir": "./test/output",
@@ -86,6 +94,7 @@
   "v_pred_like_loss": 0,
   "vae_batch_size": 0,
   "wandb_api_key": "",
+  "wandb_run_name": "",
   "weighted_captions": false,
   "xformers": "xformers"
 }

test/config/locon-AdamW8bit-toml.json

@@ -0,0 +1,130 @@
+{
+  "LoRA_type": "Standard",
+  "LyCORIS_preset": "full",
+  "adaptive_noise_scale": 0,
+  "additional_parameters": "",
+  "block_alphas": "",
+  "block_dims": "",
+  "block_lr_zero_threshold": "",
+  "bucket_no_upscale": true,
+  "bucket_reso_steps": 64,
+  "cache_latents": true,
+  "cache_latents_to_disk": false,
+  "caption_dropout_every_n_epochs": 0.0,
+  "caption_dropout_rate": 0.05,
+  "caption_extension": "",
+  "clip_skip": 2,
+  "color_aug": false,
+  "constrain": 0.0,
+  "conv_alpha": 1,
+  "conv_block_alphas": "",
+  "conv_block_dims": "",
+  "conv_dim": 1,
+  "dataset_config": "D:/kohya_ss/test/config/dataset.toml",
+  "debiased_estimation_loss": false,
+  "decompose_both": false,
+  "dim_from_weights": false,
+  "down_lr_weight": "",
+  "enable_bucket": true,
+  "epoch": 1,
+  "factor": -1,
+  "flip_aug": false,
+  "fp8_base": false,
+  "full_bf16": false,
+  "full_fp16": false,
+  "gpu_ids": "",
+  "gradient_accumulation_steps": 1,
+  "gradient_checkpointing": false,
+  "keep_tokens": "0",
+  "learning_rate": 0.0005,
+  "log_tracker_config": "",
+  "log_tracker_name": "",
+  "logging_dir": "./test/logs",
+  "lora_network_weights": "",
+  "lr_scheduler": "constant",
+  "lr_scheduler_args": "",
+  "lr_scheduler_num_cycles": "",
+  "lr_scheduler_power": "",
+  "lr_warmup": 0,
+  "max_bucket_reso": 2048,
+  "max_data_loader_n_workers": "0",
+  "max_grad_norm": 1,
+  "max_resolution": "512,512",
+  "max_timestep": 1000,
+  "max_token_length": "75",
+  "max_train_epochs": "",
+  "max_train_steps": "",
+  "mem_eff_attn": false,
+  "mid_lr_weight": "",
+  "min_bucket_reso": 256,
+  "min_snr_gamma": 0,
+  "min_timestep": 0,
+  "mixed_precision": "bf16",
+  "model_list": "runwayml/stable-diffusion-v1-5",
+  "module_dropout": 0,
+  "multi_gpu": false,
+  "multires_noise_discount": 0,
+  "multires_noise_iterations": 0,
+  "network_alpha": 1,
+  "network_dim": 8,
+  "network_dropout": 0,
+  "noise_offset": 0.05,
+  "noise_offset_type": "Original",
+  "num_cpu_threads_per_process": 2,
+  "num_machines": 1,
+  "num_processes": 1,
+  "optimizer": "AdamW8bit",
+  "optimizer_args": "",
+  "output_dir": "./test/output",
+  "output_name": "locon-AdamW8bit-toml",
+  "persistent_data_loader_workers": false,
+  "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
+  "prior_loss_weight": 1.0,
+  "random_crop": false,
+  "rank_dropout": 0,
+  "rank_dropout_scale": false,
+  "reg_data_dir": "",
+  "rescaled": false,
+  "resume": "",
+  "sample_every_n_epochs": 0,
+  "sample_every_n_steps": 25,
+  "sample_prompts": "a painting of a gas mask , by darius kawasaki",
+  "sample_sampler": "euler_a",
+  "save_every_n_epochs": 1,
+  "save_every_n_steps": 0,
+  "save_last_n_steps": 0,
+  "save_last_n_steps_state": 0,
+  "save_model_as": "safetensors",
+  "save_precision": "fp16",
+  "save_state": false,
+  "scale_v_pred_loss_like_noise_pred": false,
+  "scale_weight_norms": 0,
+  "sdxl": false,
+  "sdxl_cache_text_encoder_outputs": false,
+  "sdxl_no_half_vae": true,
+  "seed": "1234",
+  "shuffle_caption": false,
+  "stop_text_encoder_training": 0,
+  "text_encoder_lr": 0.0,
+  "train_batch_size": 4,
+  "train_data_dir": "",
+  "train_norm": false,
+  "train_on_input": true,
+  "training_comment": "",
+  "unet_lr": 0.0,
+  "unit": 1,
+  "up_lr_weight": "",
+  "use_cp": false,
+  "use_scalar": false,
+  "use_tucker": false,
+  "use_wandb": false,
+  "v2": false,
+  "v_parameterization": false,
+  "v_pred_like_loss": 0,
+  "vae": "",
+  "vae_batch_size": 0,
+  "wandb_api_key": "",
+  "wandb_run_name": "",
+  "weighted_captions": false,
+  "xformers": "xformers"
+}

test/config/meta-1_lat.json

@@ -0,0 +1,58 @@
+{
+  "test\\img\\10_darius kawasaki person\\Dariusz_Zawadzki.jpg": {
+    "caption": "a painting of a steam punk skull with a gas mask , by darius kawasaki",
+    "train_resolution": [
+      1024,
+      1024
+    ]
+  },
+  "test\\img\\10_darius kawasaki person\\Dariusz_Zawadzki_2.jpg": {
+    "caption": "a painting of a man with a skull on his head , by darius kawasaki",
+    "train_resolution": [
+      1024,
+      1024
+    ]
+  },
+  "test\\img\\10_darius kawasaki person\\Dariusz_Zawadzki_3.jpg": {
+    "caption": "a painting of a woman with a helmet on her head , by darius kawasaki",
+    "train_resolution": [
+      1024,
+      1024
+    ]
+  },
+  "test\\img\\10_darius kawasaki person\\Dariusz_Zawadzki_4.jpg": {
+    "caption": "a painting of a horned man with a goat head , by darius kawasaki",
+    "train_resolution": [
+      1024,
+      1024
+    ]
+  },
+  "test\\img\\10_darius kawasaki person\\Dariusz_Zawadzki_5.jpg": {
+    "caption": "a painting of a man playing a piano , by darius kawasaki",
+    "train_resolution": [
+      1024,
+      1024
+    ]
+  },
+  "test\\img\\10_darius kawasaki person\\Dariusz_Zawadzki_6.jpg": {
+    "caption": "a painting of a robot sitting on a rock , by darius kawasaki",
+    "train_resolution": [
+      1024,
+      1024
+    ]
+  },
+  "test\\img\\10_darius kawasaki person\\Dariusz_Zawadzki_7.jpg": {
+    "caption": "a painting of a soldier with a helmet on , by darius kawasaki",
+    "train_resolution": [
+      1024,
+      1024
+    ]
+  },
+  "test\\img\\10_darius kawasaki person\\Dariusz_Zawadzki_8.jpg": {
+    "caption": "a painting of a giant crab with a large body , by darius kawasaki",
+    "train_resolution": [
+      1024,
+      1024
+    ]
+  }
+}