diff --git a/README.md b/README.md index c1db449..6b5caa9 100644 --- a/README.md +++ b/README.md @@ -404,10 +404,12 @@ ControlNet dataset is used to specify the mask. The mask images should be the RG ### 2024/04/12 (v23.1.6) -- Rewrote a lot of the code to fix issues resulting from the security upgrades to remove shell=True from process calls. -- Improved the training and tensorboard buttons -- Upgrade the gradio version to 4.20.0 to fix a bug with runpod. -- Various other minor fixes. +- Rewrote significant portions of the code to address security vulnerabilities and remove the `shell=True` parameter from process calls. +- Enhanced the training and tensorboard buttons to provide a more intuitive and user-friendly experience. +- Upgraded the gradio version to 4.20.0 to address a bug that was causing issues with the runpod platform. +- Added a HuggingFace section to all trainers tabs, enabling users to authenticate and utilize HuggingFace's powerful AI models. +- Converted the Graphical User Interface (GUI) to use the configuration TOML file format to pass arguments to sd-scripts. This change improves security by eliminating the need for sensitive information to be passed through the command-line interface (CLI). +- Made various other minor improvements and bug fixes to enhance the overall functionality and user experience. ### 2024/04/10 (v23.1.5) diff --git a/kohya_gui/class_command_executor.py b/kohya_gui/class_command_executor.py index f81f27e..b16db0f 100644 --- a/kohya_gui/class_command_executor.py +++ b/kohya_gui/class_command_executor.py @@ -32,8 +32,8 @@ class CommandExecutor: if self.process and self.process.poll() is None: log.info("The command is already running. Please wait for it to finish.") else: - for i, item in enumerate(run_cmd): - log.info(f"{i}: {item}") + # for i, item in enumerate(run_cmd): + # log.info(f"{i}: {item}") # Reconstruct the safe command string for display command_to_run = ' '.join(run_cmd) diff --git a/kohya_gui/class_huggingface.py b/kohya_gui/class_huggingface.py new file mode 100644 index 0000000..93001fc --- /dev/null +++ b/kohya_gui/class_huggingface.py @@ -0,0 +1,82 @@ +import gradio as gr +import toml +from .class_gui_config import KohyaSSGUIConfig + +class HuggingFace: + def __init__( + self, + config: KohyaSSGUIConfig = {}, + ) -> None: + self.config = config + + # Initialize the UI components + self.initialize_ui_components() + + def initialize_ui_components(self) -> None: + # --huggingface_repo_id HUGGINGFACE_REPO_ID + # huggingface repo name to upload / huggingfaceにアップロードするリポジトリ名 + # --huggingface_repo_type HUGGINGFACE_REPO_TYPE + # huggingface repo type to upload / huggingfaceにアップロードするリポジトリの種類 + # --huggingface_path_in_repo HUGGINGFACE_PATH_IN_REPO + # huggingface model path to upload files / huggingfaceにアップロードするファイルのパス + # --huggingface_token HUGGINGFACE_TOKEN + # huggingface token / huggingfaceのトークン + # --huggingface_repo_visibility HUGGINGFACE_REPO_VISIBILITY + # huggingface repository visibility ('public' for public, 'private' or None for private) / huggingfaceにアップロードするリポジトリの公開設定（'public'で公開、'private'またはNoneで非公開） + # --save_state_to_huggingface + # save state to huggingface / huggingfaceにstateを保存する + # --resume_from_huggingface + # resume from huggingface (ex: --resume {repo_id}/{path_in_repo}:{revision}:{repo_type}) / huggingfaceから学習を再開する(例: --resume {repo_id}/{path_in_repo}:{revision}:{repo_type}) + # --async_upload upload to huggingface asynchronously / huggingfaceに非同期でアップロードする + with gr.Row(): + self.huggingface_repo_id = gr.Textbox( + label="Huggingface repo id", + placeholder="huggingface repo id", + value=self.config.get("huggingface.repo_id", ""), + ) + + self.huggingface_token = gr.Textbox( + label="Huggingface token", + placeholder="huggingface token", + value=self.config.get("huggingface.token", ""), + ) + + with gr.Row(): + # Repository settings + self.huggingface_repo_type = gr.Textbox( + label="Huggingface repo type", + placeholder="huggingface repo type", + value=self.config.get("huggingface.repo_type", ""), + ) + + self.huggingface_repo_visibility = gr.Textbox( + label="Huggingface repo visibility", + placeholder="huggingface repo visibility", + value=self.config.get("huggingface.repo_visibility", ""), + ) + + with gr.Row(): + # File location in the repository + self.huggingface_path_in_repo = gr.Textbox( + label="Huggingface path in repo", + placeholder="huggingface path in repo", + value=self.config.get("huggingface.path_in_repo", ""), + ) + + with gr.Row(): + # Functions + self.save_state_to_huggingface = gr.Checkbox( + label="Save state to huggingface", + value=self.config.get("huggingface.save_state_to_huggingface", False), + ) + + self.resume_from_huggingface = gr.Textbox( + label="Resume from huggingface", + placeholder="resume from huggingface", + value=self.config.get("huggingface.resume_from_huggingface", ""), + ) + + self.async_upload = gr.Checkbox( + label="Async upload", + value=self.config.get("huggingface.async_upload", False), + ) \ No newline at end of file diff --git a/kohya_gui/class_sample_images.py b/kohya_gui/class_sample_images.py index ff98041..d77628a 100644 --- a/kohya_gui/class_sample_images.py +++ b/kohya_gui/class_sample_images.py @@ -17,6 +17,23 @@ document_symbol = "\U0001F4C4" # 📄 ### ### Gradio common sampler GUI section ### +def create_prompt_file(sample_prompts, output_dir): + """ + Creates a prompt file for image sampling. + + Args: + sample_prompts (str): The prompts to use for image sampling. + output_dir (str): The directory where the output images will be saved. + + Returns: + str: The path to the prompt file. + """ + sample_prompts_path = os.path.join(output_dir, "prompt.txt") + + with open(sample_prompts_path, "w") as f: + f.write(sample_prompts) + + return sample_prompts_path def run_cmd_sample( diff --git a/kohya_gui/common_gui.py b/kohya_gui/common_gui.py index 22af67d..5c7f6c6 100644 --- a/kohya_gui/common_gui.py +++ b/kohya_gui/common_gui.py @@ -1575,7 +1575,8 @@ def run_cmd_advanced_training(run_cmd: list = [], **kwargs): # Use Weights and Biases logging if kwargs.get("use_wandb"): - run_cmd.append("--log_with wandb") + run_cmd.append("--log_with") + run_cmd.append("wandb") # V parameterization if kwargs.get("v_parameterization"): diff --git a/kohya_gui/dreambooth_gui.py b/kohya_gui/dreambooth_gui.py index b624dcd..a8ffcc0 100644 --- a/kohya_gui/dreambooth_gui.py +++ b/kohya_gui/dreambooth_gui.py @@ -4,9 +4,9 @@ import math import os import time import sys +import toml from datetime import datetime from .common_gui import ( - get_executable_path, get_file_path, get_saveasfile_path, color_aug_changed, @@ -26,12 +26,13 @@ from .class_basic_training import BasicTraining from .class_advanced_training import AdvancedTraining from .class_folders import Folders from .class_command_executor import CommandExecutor +from .class_huggingface import HuggingFace from .dreambooth_folder_creation_gui import ( gradio_dreambooth_folder_creation_tab, ) from .dataset_balancing_gui import gradio_dataset_balancing_tab -from .class_sample_images import SampleImages, run_cmd_sample +from .class_sample_images import SampleImages, run_cmd_sample, create_prompt_file from .class_tensorboard import TensorboardManager from .custom_logging import setup_logging @@ -42,6 +43,9 @@ log = setup_logging() # Setup command executor executor = CommandExecutor() +# Setup huggingface +huggingface = None + PYTHON = sys.executable TRAIN_BUTTON_VISIBLE = [gr.Button(visible=True), gr.Button(visible=False)] @@ -155,6 +159,14 @@ def save_configuration( max_timestep, debiased_estimation_loss, extra_accelerate_launch_args, + huggingface_repo_id, + huggingface_token, + huggingface_repo_type, + huggingface_repo_visibility, + huggingface_path_in_repo, + save_state_to_huggingface, + resume_from_huggingface, + async_upload, ): # Get list of function parameters and values parameters = list(locals().items()) @@ -296,6 +308,14 @@ def open_configuration( max_timestep, debiased_estimation_loss, extra_accelerate_launch_args, + huggingface_repo_id, + huggingface_token, + huggingface_repo_type, + huggingface_repo_visibility, + huggingface_path_in_repo, + save_state_to_huggingface, + resume_from_huggingface, + async_upload, ): # Get list of function parameters and values parameters = list(locals().items()) @@ -432,6 +452,14 @@ def train_model( max_timestep, debiased_estimation_loss, extra_accelerate_launch_args, + huggingface_repo_id, + huggingface_token, + huggingface_repo_type, + huggingface_repo_visibility, + huggingface_path_in_repo, + save_state_to_huggingface, + resume_from_huggingface, + async_upload, ): # Get list of function parameters and values parameters = list(locals().items()) @@ -587,6 +615,136 @@ def train_model( # Initialize a dictionary with always-included keyword arguments kwargs_for_training = { + # "adaptive_noise_scale": adaptive_noise_scale, + # "bucket_no_upscale": bucket_no_upscale, + # "bucket_reso_steps": bucket_reso_steps, + # "cache_latents": cache_latents, + # "cache_latents_to_disk": cache_latents_to_disk, + # "caption_dropout_every_n_epochs": caption_dropout_every_n_epochs, + # "caption_dropout_rate": caption_dropout_rate, + # "caption_extension": caption_extension, + # "clip_skip": clip_skip, + # "color_aug": color_aug, + # "dataset_config": dataset_config, + # "debiased_estimation_loss": debiased_estimation_loss, + # "enable_bucket": enable_bucket, + # "epoch": epoch, + # "flip_aug": flip_aug, + # "masked_loss": masked_loss, + # "full_bf16": full_bf16, + # "full_fp16": full_fp16, + # "gradient_accumulation_steps": gradient_accumulation_steps, + # "gradient_checkpointing": gradient_checkpointing, + # "ip_noise_gamma": ip_noise_gamma, + # "ip_noise_gamma_random_strength": ip_noise_gamma_random_strength, + # "keep_tokens": keep_tokens, + # "learning_rate": learning_rate, + # "logging_dir": logging_dir, + # "log_tracker_name": log_tracker_name, + # "log_tracker_config": log_tracker_config, + # "lr_scheduler": lr_scheduler, + # "lr_scheduler_args": lr_scheduler_args, + # "lr_scheduler_num_cycles": lr_scheduler_num_cycles, + # "lr_scheduler_power": lr_scheduler_power, + # "lr_warmup_steps": lr_warmup_steps, + # "max_bucket_reso": max_bucket_reso, + "max_data_loader_n_workers": max_data_loader_n_workers, + # "max_resolution": max_resolution, + # "max_timestep": max_timestep, + # "max_token_length": max_token_length, + # "max_train_epochs": max_train_epochs, + # "max_train_steps": max_train_steps, + # "mem_eff_attn": mem_eff_attn, + # "min_bucket_reso": min_bucket_reso, + # "min_snr_gamma": min_snr_gamma, + # "min_timestep": min_timestep, + # "mixed_precision": mixed_precision, + # "multires_noise_discount": multires_noise_discount, + # "multires_noise_iterations": multires_noise_iterations, + # "no_token_padding": no_token_padding, + # "noise_offset": noise_offset, + # "noise_offset_random_strength": noise_offset_random_strength, + # "noise_offset_type": noise_offset_type, + # "optimizer": optimizer, + # "optimizer_args": optimizer_args, + # "output_dir": output_dir, + # "output_name": output_name, + # "persistent_data_loader_workers": persistent_data_loader_workers, + # "pretrained_model_name_or_path": pretrained_model_name_or_path, + # "prior_loss_weight": prior_loss_weight, + # "random_crop": random_crop, + # "reg_data_dir": reg_data_dir, + # "resume": resume, + # "save_every_n_epochs": save_every_n_epochs, + # "save_every_n_steps": save_every_n_steps, + # "save_last_n_steps": save_last_n_steps, + # "save_last_n_steps_state": save_last_n_steps_state, + # "save_model_as": save_model_as, + # "save_precision": save_precision, + # "save_state": save_state, + # "save_state_on_train_end": save_state_on_train_end, + # "scale_v_pred_loss_like_noise_pred": scale_v_pred_loss_like_noise_pred, + # "seed": seed, + # "shuffle_caption": shuffle_caption, + # "stop_text_encoder_training": stop_text_encoder_training, + # "train_batch_size": train_batch_size, + # "train_data_dir": train_data_dir, + # "use_wandb": use_wandb, + # "v2": v2, + # "v_parameterization": v_parameterization, + # "v_pred_like_loss": v_pred_like_loss, + # "vae": vae, + # "vae_batch_size": vae_batch_size, + # "wandb_api_key": wandb_api_key, + # "wandb_run_name": wandb_run_name, + # "weighted_captions": weighted_captions, + # "xformers": xformers, + "additional_parameters": additional_parameters, + # "loss_type": loss_type, + # "huber_schedule": huber_schedule, + # "huber_c": huber_c, + } + + # Conditionally include specific keyword arguments based on sdxl + # if sdxl: + # kwargs_for_training["learning_rate_te1"] = learning_rate_te1 + # kwargs_for_training["learning_rate_te2"] = learning_rate_te2 + # else: + # kwargs_for_training["learning_rate_te"] = learning_rate_te + + # Pass the dynamically constructed keyword arguments to the function + run_cmd = run_cmd_advanced_training(run_cmd=run_cmd, **kwargs_for_training) + + # run_cmd = run_cmd_sample( + # run_cmd, + # sample_every_n_steps, + # sample_every_n_epochs, + # sample_sampler, + # sample_prompts, + # output_dir, + # ) + + if max_data_loader_n_workers == "" or None: + max_data_loader_n_workers = 0 + else: + max_data_loader_n_workers = int(max_data_loader_n_workers) + + if max_train_steps == "" or None: + max_train_steps = 0 + else: + max_train_steps = int(max_train_steps) + + # def save_huggingface_to_toml(self, toml_file_path: str): + config_toml_data = { + # Update the values in the TOML data + "huggingface_repo_id": huggingface_repo_id, + "huggingface_token": huggingface_token, + "huggingface_repo_type": huggingface_repo_type, + "huggingface_repo_visibility": huggingface_repo_visibility, + "huggingface_path_in_repo": huggingface_path_in_repo, + "save_state_to_huggingface": save_state_to_huggingface, + "resume_from_huggingface": resume_from_huggingface, + "async_upload": async_upload, "adaptive_noise_scale": adaptive_noise_scale, "bucket_no_upscale": bucket_no_upscale, "bucket_reso_steps": bucket_reso_steps, @@ -595,12 +753,12 @@ def train_model( "caption_dropout_every_n_epochs": caption_dropout_every_n_epochs, "caption_dropout_rate": caption_dropout_rate, "caption_extension": caption_extension, - "clip_skip": clip_skip, + "clip_skip": int(clip_skip), "color_aug": color_aug, "dataset_config": dataset_config, "debiased_estimation_loss": debiased_estimation_loss, "enable_bucket": enable_bucket, - "epoch": epoch, + "epoch": int(epoch), "flip_aug": flip_aug, "masked_loss": masked_loss, "full_bf16": full_bf16, @@ -609,27 +767,29 @@ def train_model( "gradient_checkpointing": gradient_checkpointing, "ip_noise_gamma": ip_noise_gamma, "ip_noise_gamma_random_strength": ip_noise_gamma_random_strength, - "keep_tokens": keep_tokens, + "keep_tokens": int(keep_tokens), "learning_rate": learning_rate, + "learning_rate_te": learning_rate_te, + "learning_rate_te1": learning_rate_te1, + "learning_rate_te2": learning_rate_te2, "logging_dir": logging_dir, "log_tracker_name": log_tracker_name, "log_tracker_config": log_tracker_config, "lr_scheduler": lr_scheduler, - "lr_scheduler_args": lr_scheduler_args, + "lr_scheduler_args": str(lr_scheduler_args).replace('"', '').split(), "lr_scheduler_num_cycles": lr_scheduler_num_cycles, "lr_scheduler_power": lr_scheduler_power, "lr_warmup_steps": lr_warmup_steps, "max_bucket_reso": max_bucket_reso, "max_data_loader_n_workers": max_data_loader_n_workers, - "max_resolution": max_resolution, "max_timestep": max_timestep, - "max_token_length": max_token_length, + "max_token_length": int(max_token_length), "max_train_epochs": max_train_epochs, - "max_train_steps": max_train_steps, + "max_train_steps": int(max_train_steps), "mem_eff_attn": mem_eff_attn, - "min_bucket_reso": min_bucket_reso, + "min_bucket_reso": int(min_bucket_reso), "min_snr_gamma": min_snr_gamma, - "min_timestep": min_timestep, + "min_timestep": int(min_timestep), "mixed_precision": mixed_precision, "multires_noise_discount": multires_noise_discount, "multires_noise_iterations": multires_noise_iterations, @@ -637,8 +797,8 @@ def train_model( "noise_offset": noise_offset, "noise_offset_random_strength": noise_offset_random_strength, "noise_offset_type": noise_offset_type, - "optimizer": optimizer, - "optimizer_args": optimizer_args, + "optimizer_type": optimizer, + "optimizer_args": str(optimizer_args).replace('"', '').split(), "output_dir": output_dir, "output_name": output_name, "persistent_data_loader_workers": persistent_data_loader_workers, @@ -646,7 +806,12 @@ def train_model( "prior_loss_weight": prior_loss_weight, "random_crop": random_crop, "reg_data_dir": reg_data_dir, + "resolution": max_resolution, "resume": resume, + "sample_every_n_epochs": sample_every_n_epochs, + "sample_every_n_steps": sample_every_n_steps, + "sample_prompts": create_prompt_file(output_dir, output_dir), + "sample_sampler": sample_sampler, "save_every_n_epochs": save_every_n_epochs, "save_every_n_steps": save_every_n_steps, "save_last_n_steps": save_last_n_steps, @@ -656,7 +821,7 @@ def train_model( "save_state": save_state, "save_state_on_train_end": save_state_on_train_end, "scale_v_pred_loss_like_noise_pred": scale_v_pred_loss_like_noise_pred, - "seed": seed, + "seed": int(seed), "shuffle_caption": shuffle_caption, "stop_text_encoder_training": stop_text_encoder_training, "train_batch_size": train_batch_size, @@ -677,24 +842,30 @@ def train_model( "huber_c": huber_c, } - # Conditionally include specific keyword arguments based on sdxl - if sdxl: - kwargs_for_training["learning_rate_te1"] = learning_rate_te1 - kwargs_for_training["learning_rate_te2"] = learning_rate_te2 - else: - kwargs_for_training["learning_rate_te"] = learning_rate_te + # Given dictionary `config_toml_data` + # Remove all values = "" + config_toml_data = { + key: value + for key, value in config_toml_data.items() + if value != "" and value != False + } - # Pass the dynamically constructed keyword arguments to the function - run_cmd = run_cmd_advanced_training(run_cmd=run_cmd, **kwargs_for_training) + tmpfilename = "./outputs/tmpfiledbooth.toml" + # Save the updated TOML data back to the file + with open(tmpfilename, "w") as toml_file: + toml.dump(config_toml_data, toml_file) - run_cmd = run_cmd_sample( - run_cmd, - sample_every_n_steps, - sample_every_n_epochs, - sample_sampler, - sample_prompts, - output_dir, - ) + if not os.path.exists(toml_file.name): + log.error(f"Failed to write TOML file: {toml_file.name}") + + toml_file_path = ( + os.path.abspath(os.path.normpath(toml_file.name)).replace("\\", "/") + if os.name == "nt" + else toml_file.name + ) + + run_cmd.append(f"--config_file") + run_cmd.append(rf"{toml_file_path}") if print_only: log.warning( @@ -806,6 +977,10 @@ def dreambooth_tab( with gr.Accordion("Samples", open=False, elem_id="samples_tab"): sample = SampleImages(config=config) + global huggingface + with gr.Accordion("HuggingFace", open=False): + huggingface = HuggingFace(config=config) + with gr.Column(), gr.Group(): with gr.Row(): button_run = gr.Button("Start training", variant="primary") @@ -927,6 +1102,14 @@ def dreambooth_tab( advanced_training.max_timestep, advanced_training.debiased_estimation_loss, accelerate_launch.extra_accelerate_launch_args, + huggingface.huggingface_repo_id, + huggingface.huggingface_token, + huggingface.huggingface_repo_type, + huggingface.huggingface_repo_visibility, + huggingface.huggingface_path_in_repo, + huggingface.save_state_to_huggingface, + huggingface.resume_from_huggingface, + huggingface.async_upload, ] configuration.button_open_config.click( diff --git a/kohya_gui/finetune_gui.py b/kohya_gui/finetune_gui.py index e69e7e5..6c0bc6c 100644 --- a/kohya_gui/finetune_gui.py +++ b/kohya_gui/finetune_gui.py @@ -5,9 +5,9 @@ import os import subprocess import time import sys +import toml from datetime import datetime from .common_gui import ( - get_executable_path, get_file_path, get_saveasfile_path, run_cmd_advanced_training, @@ -28,7 +28,8 @@ from .class_folders import Folders from .class_sdxl_parameters import SDXLParameters from .class_command_executor import CommandExecutor from .class_tensorboard import TensorboardManager -from .class_sample_images import SampleImages, run_cmd_sample +from .class_sample_images import SampleImages, run_cmd_sample, create_prompt_file +from .class_huggingface import HuggingFace from .custom_logging import setup_logging @@ -38,6 +39,9 @@ log = setup_logging() # Setup command executor executor = CommandExecutor() +# Setup huggingface +huggingface = None + # from easygui import msgbox folder_symbol = "\U0001f4c2" # 📂 @@ -163,6 +167,14 @@ def save_configuration( min_timestep, max_timestep, extra_accelerate_launch_args, + huggingface_repo_id, + huggingface_token, + huggingface_repo_type, + huggingface_repo_visibility, + huggingface_path_in_repo, + save_state_to_huggingface, + resume_from_huggingface, + async_upload, ): # Get list of function parameters and values parameters = list(locals().items()) @@ -311,6 +323,14 @@ def open_configuration( min_timestep, max_timestep, extra_accelerate_launch_args, + huggingface_repo_id, + huggingface_token, + huggingface_repo_type, + huggingface_repo_visibility, + huggingface_path_in_repo, + save_state_to_huggingface, + resume_from_huggingface, + async_upload, training_preset, ): # Get list of function parameters and values @@ -465,6 +485,14 @@ def train_model( min_timestep, max_timestep, extra_accelerate_launch_args, + huggingface_repo_id, + huggingface_token, + huggingface_repo_type, + huggingface_repo_visibility, + huggingface_path_in_repo, + save_state_to_huggingface, + resume_from_huggingface, + async_upload, ): # Get list of function parameters and values parameters = list(locals().items()) @@ -651,19 +679,146 @@ def train_model( # Initialize a dictionary with always-included keyword arguments kwargs_for_training = { + # "adaptive_noise_scale": adaptive_noise_scale, + # "block_lr": block_lr, + # "bucket_no_upscale": bucket_no_upscale, + # "bucket_reso_steps": bucket_reso_steps, + # "cache_latents": cache_latents, + # "cache_latents_to_disk": cache_latents_to_disk, + # "caption_dropout_every_n_epochs": caption_dropout_every_n_epochs, + # "caption_dropout_rate": caption_dropout_rate, + # "caption_extension": caption_extension, + # "clip_skip": clip_skip, + # "color_aug": color_aug, + # "dataset_config": dataset_config, + # "dataset_repeats": dataset_repeats, + # "enable_bucket": True, + # "flip_aug": flip_aug, + # "masked_loss": masked_loss, + # "full_bf16": full_bf16, + # "full_fp16": full_fp16, + # "gradient_accumulation_steps": gradient_accumulation_steps, + # "gradient_checkpointing": gradient_checkpointing, + # "in_json": in_json, + # "ip_noise_gamma": ip_noise_gamma, + # "ip_noise_gamma_random_strength": ip_noise_gamma_random_strength, + # "keep_tokens": keep_tokens, + # "learning_rate": learning_rate, + # "logging_dir": logging_dir, + # "log_tracker_name": log_tracker_name, + # "log_tracker_config": log_tracker_config, + # "lr_scheduler": lr_scheduler, + # "lr_scheduler_args": lr_scheduler_args, + # "lr_warmup_steps": lr_warmup_steps, + # "max_bucket_reso": max_bucket_reso, + "max_data_loader_n_workers": max_data_loader_n_workers, + # "max_resolution": max_resolution, + # "max_timestep": max_timestep, + # "max_token_length": max_token_length, + # "max_train_epochs": max_train_epochs, + # "max_train_steps": max_train_steps, + # "mem_eff_attn": mem_eff_attn, + # "min_bucket_reso": min_bucket_reso, + # "min_snr_gamma": min_snr_gamma, + # "min_timestep": min_timestep, + # "mixed_precision": mixed_precision, + # "multires_noise_discount": multires_noise_discount, + # "multires_noise_iterations": multires_noise_iterations, + # "noise_offset": noise_offset, + # "noise_offset_random_strength": noise_offset_random_strength, + # "noise_offset_type": noise_offset_type, + # "optimizer": optimizer, + # "optimizer_args": optimizer_args, + # "output_dir": output_dir, + # "output_name": output_name, + # "persistent_data_loader_workers": persistent_data_loader_workers, + # "pretrained_model_name_or_path": pretrained_model_name_or_path, + # "random_crop": random_crop, + # "resume": resume, + # "save_every_n_epochs": save_every_n_epochs, + # "save_every_n_steps": save_every_n_steps, + # "save_last_n_steps": save_last_n_steps, + # "save_last_n_steps_state": save_last_n_steps_state, + # "save_model_as": save_model_as, + # "save_precision": save_precision, + # "save_state": save_state, + # "save_state_on_train_end": save_state_on_train_end, + # "scale_v_pred_loss_like_noise_pred": scale_v_pred_loss_like_noise_pred, + # "seed": seed, + # "shuffle_caption": shuffle_caption, + # "train_batch_size": train_batch_size, + # "train_data_dir": image_folder, + # "train_text_encoder": train_text_encoder, + # "use_wandb": use_wandb, + # "v2": v2, + # "v_parameterization": v_parameterization, + # "v_pred_like_loss": v_pred_like_loss, + # "vae_batch_size": vae_batch_size, + # "wandb_api_key": wandb_api_key, + # "wandb_run_name": wandb_run_name, + # "weighted_captions": weighted_captions, + # "xformers": xformers, + "additional_parameters": additional_parameters, + # "loss_type": loss_type, + # "huber_schedule": huber_schedule, + # "huber_c": huber_c, + } + + # Conditionally include specific keyword arguments based on sdxl_checkbox + # if sdxl_checkbox: + # kwargs_for_training["cache_text_encoder_outputs"] = cache_text_encoder_outputs + # kwargs_for_training["learning_rate_te1"] = learning_rate_te1 + # kwargs_for_training["learning_rate_te2"] = learning_rate_te2 + # kwargs_for_training["no_half_vae"] = no_half_vae + # else: + # kwargs_for_training["learning_rate_te"] = learning_rate_te + + # Pass the dynamically constructed keyword arguments to the function + run_cmd = run_cmd_advanced_training(run_cmd=run_cmd, **kwargs_for_training) + + # run_cmd = run_cmd_sample( + # run_cmd, + # sample_every_n_steps, + # sample_every_n_epochs, + # sample_sampler, + # sample_prompts, + # output_dir, + # ) + + if max_data_loader_n_workers == "" or None: + max_data_loader_n_workers = 0 + else: + max_data_loader_n_workers = int(max_data_loader_n_workers) + + if max_train_steps == "" or None: + max_train_steps = 0 + else: + max_train_steps = int(max_train_steps) + + config_toml_data = { + # Update the values in the TOML data + "huggingface_repo_id": huggingface_repo_id, + "huggingface_token": huggingface_token, + "huggingface_repo_type": huggingface_repo_type, + "huggingface_repo_visibility": huggingface_repo_visibility, + "huggingface_path_in_repo": huggingface_path_in_repo, + "save_state_to_huggingface": save_state_to_huggingface, + "resume_from_huggingface": resume_from_huggingface, + "async_upload": async_upload, "adaptive_noise_scale": adaptive_noise_scale, "block_lr": block_lr, "bucket_no_upscale": bucket_no_upscale, "bucket_reso_steps": bucket_reso_steps, "cache_latents": cache_latents, "cache_latents_to_disk": cache_latents_to_disk, + "cache_text_encoder_outputs": cache_text_encoder_outputs, "caption_dropout_every_n_epochs": caption_dropout_every_n_epochs, "caption_dropout_rate": caption_dropout_rate, "caption_extension": caption_extension, - "clip_skip": clip_skip, + "clip_skip": int(clip_skip), "color_aug": color_aug, "dataset_config": dataset_config, - "dataset_repeats": dataset_repeats, + "dataset_repeats": int(dataset_repeats), "enable_bucket": True, "flip_aug": flip_aug, "masked_loss": masked_loss, @@ -674,39 +829,47 @@ def train_model( "in_json": in_json, "ip_noise_gamma": ip_noise_gamma, "ip_noise_gamma_random_strength": ip_noise_gamma_random_strength, - "keep_tokens": keep_tokens, + "keep_tokens": int(keep_tokens), "learning_rate": learning_rate, + "learning_rate_te": learning_rate_te, + "learning_rate_te1": learning_rate_te1, + "learning_rate_te2": learning_rate_te2, "logging_dir": logging_dir, "log_tracker_name": log_tracker_name, "log_tracker_config": log_tracker_config, "lr_scheduler": lr_scheduler, - "lr_scheduler_args": lr_scheduler_args, + "lr_scheduler_args": str(lr_scheduler_args).replace('"', '').split(), "lr_warmup_steps": lr_warmup_steps, - "max_bucket_reso": max_bucket_reso, + "max_bucket_reso": int(max_bucket_reso), "max_data_loader_n_workers": max_data_loader_n_workers, - "max_resolution": max_resolution, "max_timestep": max_timestep, - "max_token_length": max_token_length, + "max_token_length": int(max_token_length), "max_train_epochs": max_train_epochs, - "max_train_steps": max_train_steps, + "max_train_steps": int(max_train_steps), "mem_eff_attn": mem_eff_attn, - "min_bucket_reso": min_bucket_reso, + "min_bucket_reso": int(min_bucket_reso), "min_snr_gamma": min_snr_gamma, - "min_timestep": min_timestep, + "min_timestep": int(min_timestep), "mixed_precision": mixed_precision, "multires_noise_discount": multires_noise_discount, "multires_noise_iterations": multires_noise_iterations, + "no_half_vae": no_half_vae, "noise_offset": noise_offset, "noise_offset_random_strength": noise_offset_random_strength, "noise_offset_type": noise_offset_type, - "optimizer": optimizer, - "optimizer_args": optimizer_args, + "optimizer_type": optimizer, + "optimizer_args": str(optimizer_args).replace('"', '').split(), "output_dir": output_dir, "output_name": output_name, "persistent_data_loader_workers": persistent_data_loader_workers, "pretrained_model_name_or_path": pretrained_model_name_or_path, "random_crop": random_crop, + "resolution": max_resolution, "resume": resume, + "sample_every_n_epochs": sample_every_n_epochs, + "sample_every_n_steps": sample_every_n_steps, + "sample_prompts": create_prompt_file(output_dir, output_dir), + "sample_sampler": sample_sampler, "save_every_n_epochs": save_every_n_epochs, "save_every_n_steps": save_every_n_steps, "save_last_n_steps": save_last_n_steps, @@ -716,7 +879,7 @@ def train_model( "save_state": save_state, "save_state_on_train_end": save_state_on_train_end, "scale_v_pred_loss_like_noise_pred": scale_v_pred_loss_like_noise_pred, - "seed": seed, + "seed": int(seed), "shuffle_caption": shuffle_caption, "train_batch_size": train_batch_size, "train_data_dir": image_folder, @@ -736,26 +899,30 @@ def train_model( "huber_c": huber_c, } - # Conditionally include specific keyword arguments based on sdxl_checkbox - if sdxl_checkbox: - kwargs_for_training["cache_text_encoder_outputs"] = cache_text_encoder_outputs - kwargs_for_training["learning_rate_te1"] = learning_rate_te1 - kwargs_for_training["learning_rate_te2"] = learning_rate_te2 - kwargs_for_training["no_half_vae"] = no_half_vae - else: - kwargs_for_training["learning_rate_te"] = learning_rate_te + # Given dictionary `config_toml_data` + # Remove all values = "" + config_toml_data = { + key: value + for key, value in config_toml_data.items() + if value != "" and value != False + } - # Pass the dynamically constructed keyword arguments to the function - run_cmd = run_cmd_advanced_training(run_cmd=run_cmd, **kwargs_for_training) + tmpfilename = "./outputs/tmpfilefinetune.toml" + # Save the updated TOML data back to the file + with open(tmpfilename, "w") as toml_file: + toml.dump(config_toml_data, toml_file) - run_cmd = run_cmd_sample( - run_cmd, - sample_every_n_steps, - sample_every_n_epochs, - sample_sampler, - sample_prompts, - output_dir, - ) + if not os.path.exists(toml_file.name): + log.error(f"Failed to write TOML file: {toml_file.name}") + + toml_file_path = ( + os.path.abspath(os.path.normpath(toml_file.name)).replace("\\", "/") + if os.name == "nt" + else toml_file.name + ) + + run_cmd.append(f"--config_file") + run_cmd.append(rf"{toml_file_path}") if print_only: log.warning( @@ -937,6 +1104,10 @@ def finetune_tab(headless=False, config: dict = {}): with gr.Accordion("Samples", open=False, elem_id="samples_tab"): sample = SampleImages(config=config) + global huggingface + with gr.Accordion("HuggingFace", open=False): + huggingface = HuggingFace(config=config) + with gr.Column(), gr.Group(): with gr.Row(): button_run = gr.Button("Start training", variant="primary") @@ -1062,6 +1233,14 @@ def finetune_tab(headless=False, config: dict = {}): advanced_training.min_timestep, advanced_training.max_timestep, accelerate_launch.extra_accelerate_launch_args, + huggingface.huggingface_repo_id, + huggingface.huggingface_token, + huggingface.huggingface_repo_type, + huggingface.huggingface_repo_visibility, + huggingface.huggingface_path_in_repo, + huggingface.save_state_to_huggingface, + huggingface.resume_from_huggingface, + huggingface.async_upload, ] configuration.button_open_config.click( diff --git a/kohya_gui/lora_gui.py b/kohya_gui/lora_gui.py index 4451c23..1ff2470 100644 --- a/kohya_gui/lora_gui.py +++ b/kohya_gui/lora_gui.py @@ -3,10 +3,10 @@ import json import math import os import time +import toml from datetime import datetime from .common_gui import ( - get_executable_path, get_file_path, get_any_file_path, get_saveasfile_path, @@ -29,8 +29,9 @@ from .class_sdxl_parameters import SDXLParameters from .class_folders import Folders from .class_command_executor import CommandExecutor from .class_tensorboard import TensorboardManager -from .class_sample_images import SampleImages, run_cmd_sample +from .class_sample_images import SampleImages, run_cmd_sample, create_prompt_file from .class_lora_tab import LoRATools +from .class_huggingface import HuggingFace from .dreambooth_folder_creation_gui import ( gradio_dreambooth_folder_creation_tab, @@ -45,6 +46,9 @@ log = setup_logging() # Setup command executor executor = CommandExecutor() +# Setup huggingface +huggingface = None + button_run = gr.Button("Start training", variant="primary") button_stop_training = gr.Button("Stop training", visible=False) @@ -79,7 +83,7 @@ def update_network_args_with_kohya_lora_vars( # Iterate over the Kohya LoRA variables and append them to the network arguments for key, value in kohya_lora_vars.items(): # Append each variable as a key-value pair to the network_args - network_args += f' {key}="{value}"' + network_args += f' {key}={value}' return network_args @@ -228,6 +232,14 @@ def save_configuration( LyCORIS_preset, debiased_estimation_loss, extra_accelerate_launch_args, + huggingface_repo_id, + huggingface_token, + huggingface_repo_type, + huggingface_repo_visibility, + huggingface_path_in_repo, + save_state_to_huggingface, + resume_from_huggingface, + async_upload, ): # Get list of function parameters and values parameters = list(locals().items()) @@ -416,6 +428,14 @@ def open_configuration( LyCORIS_preset, debiased_estimation_loss, extra_accelerate_launch_args, + huggingface_repo_id, + huggingface_token, + huggingface_repo_type, + huggingface_repo_visibility, + huggingface_path_in_repo, + save_state_to_huggingface, + resume_from_huggingface, + async_upload, training_preset, ): # Get list of function parameters and values @@ -634,6 +654,14 @@ def train_model( LyCORIS_preset, debiased_estimation_loss, extra_accelerate_launch_args, + huggingface_repo_id, + huggingface_token, + huggingface_repo_type, + huggingface_repo_visibility, + huggingface_path_in_repo, + save_state_to_huggingface, + resume_from_huggingface, + async_upload, ): # Get list of function parameters and values parameters = list(locals().items()) @@ -816,39 +844,39 @@ def train_model( if LoRA_type == "LyCORIS/BOFT": network_module = "lycoris.kohya" - network_args = f' preset="{LyCORIS_preset}" conv_dim="{conv_dim}" conv_alpha="{conv_alpha}" module_dropout="{module_dropout}" use_tucker="{use_tucker}" use_scalar="{use_scalar}" rank_dropout="{rank_dropout}" rank_dropout_scale="{rank_dropout_scale}" constrain="{constrain}" rescaled="{rescaled}" algo="boft" train_norm="{train_norm}"' + network_args = f' preset={LyCORIS_preset} conv_dim={conv_dim} conv_alpha={conv_alpha} module_dropout={module_dropout} use_tucker={use_tucker} use_scalar={use_scalar} rank_dropout={rank_dropout} rank_dropout_scale={rank_dropout_scale} constrain={constrain} rescaled={rescaled} algo=boft train_norm={train_norm}' if LoRA_type == "LyCORIS/Diag-OFT": network_module = "lycoris.kohya" - network_args = f' preset="{LyCORIS_preset}" conv_dim="{conv_dim}" conv_alpha="{conv_alpha}" module_dropout="{module_dropout}" use_tucker="{use_tucker}" use_scalar="{use_scalar}" rank_dropout="{rank_dropout}" rank_dropout_scale="{rank_dropout_scale}" constrain="{constrain}" rescaled="{rescaled}" algo="diag-oft" train_norm="{train_norm}"' + network_args = f' preset={LyCORIS_preset} conv_dim={conv_dim} conv_alpha={conv_alpha} module_dropout={module_dropout} use_tucker={use_tucker} use_scalar={use_scalar} rank_dropout={rank_dropout} rank_dropout_scale={rank_dropout_scale} constrain={constrain} rescaled={rescaled} algo=diag-oft train_norm={train_norm}' if LoRA_type == "LyCORIS/DyLoRA": network_module = "lycoris.kohya" - network_args = f' preset="{LyCORIS_preset}" conv_dim="{conv_dim}" conv_alpha="{conv_alpha}" use_tucker="{use_tucker}" block_size="{unit}" rank_dropout="{rank_dropout}" module_dropout="{module_dropout}" algo="dylora" train_norm="{train_norm}"' + network_args = f' preset={LyCORIS_preset} conv_dim={conv_dim} conv_alpha={conv_alpha} use_tucker={use_tucker} block_size={unit} rank_dropout={rank_dropout} module_dropout={module_dropout} algo="dylora" train_norm={train_norm}' if LoRA_type == "LyCORIS/GLoRA": network_module = "lycoris.kohya" - network_args = f' preset="{LyCORIS_preset}" conv_dim="{conv_dim}" conv_alpha="{conv_alpha}" rank_dropout="{rank_dropout}" module_dropout="{module_dropout}" rank_dropout_scale="{rank_dropout_scale}" algo="glora" train_norm="{train_norm}"' + network_args = f' preset={LyCORIS_preset} conv_dim={conv_dim} conv_alpha={conv_alpha} rank_dropout={rank_dropout} module_dropout={module_dropout} rank_dropout_scale={rank_dropout_scale} algo="glora" train_norm={train_norm}' if LoRA_type == "LyCORIS/iA3": network_module = "lycoris.kohya" - network_args = f' preset="{LyCORIS_preset}" conv_dim="{conv_dim}" conv_alpha="{conv_alpha}" train_on_input="{train_on_input}" algo="ia3"' + network_args = f' preset={LyCORIS_preset} conv_dim={conv_dim} conv_alpha={conv_alpha} train_on_input={train_on_input} algo=ia3' if LoRA_type == "LoCon" or LoRA_type == "LyCORIS/LoCon": network_module = "lycoris.kohya" - network_args = f' preset="{LyCORIS_preset}" conv_dim="{conv_dim}" conv_alpha="{conv_alpha}" rank_dropout="{rank_dropout}" bypass_mode="{bypass_mode}" dora_wd="{dora_wd}" module_dropout="{module_dropout}" use_tucker="{use_tucker}" use_scalar="{use_scalar}" rank_dropout_scale="{rank_dropout_scale}" algo="locon" train_norm="{train_norm}"' + network_args = f' preset={LyCORIS_preset} conv_dim={conv_dim} conv_alpha={conv_alpha} rank_dropout={rank_dropout} bypass_mode={bypass_mode} dora_wd={dora_wd} module_dropout={module_dropout} use_tucker={use_tucker} use_scalar={use_scalar} rank_dropout_scale={rank_dropout_scale} algo=locon train_norm={train_norm}' if LoRA_type == "LyCORIS/LoHa": network_module = "lycoris.kohya" - network_args = f' preset="{LyCORIS_preset}" conv_dim="{conv_dim}" conv_alpha="{conv_alpha}" rank_dropout="{rank_dropout}" bypass_mode="{bypass_mode}" dora_wd="{dora_wd}" module_dropout="{module_dropout}" use_tucker="{use_tucker}" use_scalar="{use_scalar}" rank_dropout_scale="{rank_dropout_scale}" algo="loha" train_norm="{train_norm}"' + network_args = f' preset={LyCORIS_preset} conv_dim={conv_dim} conv_alpha={conv_alpha} rank_dropout={rank_dropout} bypass_mode={bypass_mode} dora_wd={dora_wd} module_dropout={module_dropout} use_tucker={use_tucker} use_scalar={use_scalar} rank_dropout_scale={rank_dropout_scale} algo="loha" train_norm={train_norm}' if LoRA_type == "LyCORIS/LoKr": network_module = "lycoris.kohya" - network_args = f' preset="{LyCORIS_preset}" conv_dim="{conv_dim}" conv_alpha="{conv_alpha}" rank_dropout="{rank_dropout}" bypass_mode="{bypass_mode}" dora_wd="{dora_wd}" module_dropout="{module_dropout}" factor="{factor}" use_cp="{use_cp}" use_scalar="{use_scalar}" decompose_both="{decompose_both}" rank_dropout_scale="{rank_dropout_scale}" algo="lokr" train_norm="{train_norm}"' + network_args = f' preset={LyCORIS_preset} conv_dim={conv_dim} conv_alpha={conv_alpha} rank_dropout={rank_dropout} bypass_mode={bypass_mode} dora_wd={dora_wd} module_dropout={module_dropout} factor={factor} use_cp={use_cp} use_scalar={use_scalar} decompose_both={decompose_both} rank_dropout_scale={rank_dropout_scale} algo=lokr train_norm={train_norm}' if LoRA_type == "LyCORIS/Native Fine-Tuning": network_module = "lycoris.kohya" - network_args = f' preset="{LyCORIS_preset}" rank_dropout="{rank_dropout}" module_dropout="{module_dropout}" use_tucker="{use_tucker}" use_scalar="{use_scalar}" rank_dropout_scale="{rank_dropout_scale}" algo="full" train_norm="{train_norm}"' + network_args = f' preset={LyCORIS_preset} rank_dropout={rank_dropout} module_dropout={module_dropout} use_tucker={use_tucker} use_scalar={use_scalar} rank_dropout_scale={rank_dropout_scale} algo=full train_norm={train_norm}' if LoRA_type in ["Kohya LoCon", "Standard"]: kohya_lora_var_list = [ @@ -931,6 +959,150 @@ def train_model( # Define a dictionary of parameters run_cmd_params = { + # "adaptive_noise_scale": adaptive_noise_scale, + # "bucket_no_upscale": bucket_no_upscale, + # "bucket_reso_steps": bucket_reso_steps, + # "cache_latents": cache_latents, + # "cache_latents_to_disk": cache_latents_to_disk, + # "cache_text_encoder_outputs": ( + # True if sdxl and sdxl_cache_text_encoder_outputs else None + # ), + # "caption_dropout_every_n_epochs": caption_dropout_every_n_epochs, + # "caption_dropout_rate": caption_dropout_rate, + # "caption_extension": caption_extension, + # "clip_skip": clip_skip, + # "color_aug": color_aug, + # "dataset_config": dataset_config, + # "debiased_estimation_loss": debiased_estimation_loss, + # "dim_from_weights": dim_from_weights, + # "enable_bucket": enable_bucket, + # "epoch": epoch, + # "flip_aug": flip_aug, + # "masked_loss": masked_loss, + # "fp8_base": fp8_base, + # "full_bf16": full_bf16, + # "full_fp16": full_fp16, + # "gradient_accumulation_steps": gradient_accumulation_steps, + # "gradient_checkpointing": gradient_checkpointing, + # "ip_noise_gamma": ip_noise_gamma, + # "ip_noise_gamma_random_strength": ip_noise_gamma_random_strength, + # "keep_tokens": keep_tokens, + # "learning_rate": learning_rate, + # "logging_dir": logging_dir, + # "log_tracker_name": log_tracker_name, + # "log_tracker_config": log_tracker_config, + # "lora_network_weights": lora_network_weights, + # "lr_scheduler": lr_scheduler, + # "lr_scheduler_args": lr_scheduler_args, + # "lr_scheduler_num_cycles": lr_scheduler_num_cycles, + # "lr_scheduler_power": lr_scheduler_power, + # "lr_warmup_steps": lr_warmup_steps, + # "max_bucket_reso": max_bucket_reso, + "max_data_loader_n_workers": max_data_loader_n_workers, + # "max_grad_norm": max_grad_norm, + # "max_resolution": max_resolution, + # "max_timestep": max_timestep, + # "max_token_length": max_token_length, + # "max_train_epochs": max_train_epochs, + # "max_train_steps": max_train_steps, + # "mem_eff_attn": mem_eff_attn, + # "min_bucket_reso": min_bucket_reso, + # "min_snr_gamma": min_snr_gamma, + # "min_timestep": min_timestep, + # "mixed_precision": mixed_precision, + # "multires_noise_discount": multires_noise_discount, + # "multires_noise_iterations": multires_noise_iterations, + # "network_alpha": network_alpha, + # "network_args": network_args, + # "network_dim": network_dim, + # "network_dropout": network_dropout, + # "network_module": network_module, + # "network_train_unet_only": network_train_unet_only, + # "network_train_text_encoder_only": network_train_text_encoder_only, + # "no_half_vae": True if sdxl and sdxl_no_half_vae else None, + # "noise_offset": noise_offset, + # "noise_offset_random_strength": noise_offset_random_strength, + # "noise_offset_type": noise_offset_type, + # "optimizer": optimizer, + # "optimizer_args": optimizer_args, + # "output_dir": output_dir, + # "output_name": output_name, + # "persistent_data_loader_workers": persistent_data_loader_workers, + # "pretrained_model_name_or_path": pretrained_model_name_or_path, + # "prior_loss_weight": prior_loss_weight, + # "random_crop": random_crop, + # "reg_data_dir": reg_data_dir, + # "resume": resume, + # "save_every_n_epochs": save_every_n_epochs, + # "save_every_n_steps": save_every_n_steps, + # "save_last_n_steps": save_last_n_steps, + # "save_last_n_steps_state": save_last_n_steps_state, + # "save_model_as": save_model_as, + # "save_precision": save_precision, + # "save_state": save_state, + # "save_state_on_train_end": save_state_on_train_end, + # "scale_v_pred_loss_like_noise_pred": scale_v_pred_loss_like_noise_pred, + # "scale_weight_norms": scale_weight_norms, + # "seed": seed, + # "shuffle_caption": shuffle_caption, + # "stop_text_encoder_training": stop_text_encoder_training, + # "text_encoder_lr": text_encoder_lr, + # "train_batch_size": train_batch_size, + # "train_data_dir": train_data_dir, + # "training_comment": training_comment, + # "unet_lr": unet_lr, + # "use_wandb": use_wandb, + # "v2": v2, + # "v_parameterization": v_parameterization, + # "v_pred_like_loss": v_pred_like_loss, + # "vae": vae, + # "vae_batch_size": vae_batch_size, + # "wandb_api_key": wandb_api_key, + # "wandb_run_name": wandb_run_name, + # "weighted_captions": weighted_captions, + # "xformers": xformers, + "additional_parameters": additional_parameters, + # "loss_type": loss_type, + # "huber_schedule": huber_schedule, + # "huber_c": huber_c, + } + + # Use the ** syntax to unpack the dictionary when calling the function + run_cmd = run_cmd_advanced_training(run_cmd=run_cmd, **run_cmd_params) + + # run_cmd = run_cmd_sample( + # run_cmd, + # sample_every_n_steps, + # sample_every_n_epochs, + # sample_sampler, + # sample_prompts, + # output_dir, + # ) + + if max_data_loader_n_workers == "" or None: + max_data_loader_n_workers = 0 + else: + max_data_loader_n_workers = int(max_data_loader_n_workers) + + if max_train_steps == "" or None: + max_train_steps = 0 + else: + max_train_steps = int(max_train_steps) + + if seed == "": + seed = 0 + else: + seed = int(seed) + + config_toml_data = { + "huggingface_repo_id": huggingface_repo_id, + "huggingface_token": huggingface_token, + "huggingface_repo_type": huggingface_repo_type, + "huggingface_repo_visibility": huggingface_repo_visibility, + "huggingface_path_in_repo": huggingface_path_in_repo, + "save_state_to_huggingface": save_state_to_huggingface, + "resume_from_huggingface": resume_from_huggingface, + "async_upload": async_upload, "adaptive_noise_scale": adaptive_noise_scale, "bucket_no_upscale": bucket_no_upscale, "bucket_reso_steps": bucket_reso_steps, @@ -942,13 +1114,13 @@ def train_model( "caption_dropout_every_n_epochs": caption_dropout_every_n_epochs, "caption_dropout_rate": caption_dropout_rate, "caption_extension": caption_extension, - "clip_skip": clip_skip, + "clip_skip": int(clip_skip), "color_aug": color_aug, "dataset_config": dataset_config, "debiased_estimation_loss": debiased_estimation_loss, "dim_from_weights": dim_from_weights, "enable_bucket": enable_bucket, - "epoch": epoch, + "epoch": int(epoch), "flip_aug": flip_aug, "masked_loss": masked_loss, "fp8_base": fp8_base, @@ -958,34 +1130,33 @@ def train_model( "gradient_checkpointing": gradient_checkpointing, "ip_noise_gamma": ip_noise_gamma, "ip_noise_gamma_random_strength": ip_noise_gamma_random_strength, - "keep_tokens": keep_tokens, + "keep_tokens": int(keep_tokens), "learning_rate": learning_rate, "logging_dir": logging_dir, "log_tracker_name": log_tracker_name, "log_tracker_config": log_tracker_config, "lora_network_weights": lora_network_weights, "lr_scheduler": lr_scheduler, - "lr_scheduler_args": lr_scheduler_args, + "lr_scheduler_args": str(lr_scheduler_args).replace('"', '').split(), "lr_scheduler_num_cycles": lr_scheduler_num_cycles, "lr_scheduler_power": lr_scheduler_power, "lr_warmup_steps": lr_warmup_steps, "max_bucket_reso": max_bucket_reso, "max_data_loader_n_workers": max_data_loader_n_workers, "max_grad_norm": max_grad_norm, - "max_resolution": max_resolution, "max_timestep": max_timestep, - "max_token_length": max_token_length, + "max_token_length": int(max_token_length), "max_train_epochs": max_train_epochs, - "max_train_steps": max_train_steps, + "max_train_steps": int(max_train_steps), "mem_eff_attn": mem_eff_attn, - "min_bucket_reso": min_bucket_reso, + "min_bucket_reso": int(min_bucket_reso), "min_snr_gamma": min_snr_gamma, - "min_timestep": min_timestep, + "min_timestep": int(min_timestep), "mixed_precision": mixed_precision, "multires_noise_discount": multires_noise_discount, "multires_noise_iterations": multires_noise_iterations, "network_alpha": network_alpha, - "network_args": network_args, + "network_args": str(network_args).replace('"', '').split(), "network_dim": network_dim, "network_dropout": network_dropout, "network_module": network_module, @@ -995,8 +1166,8 @@ def train_model( "noise_offset": noise_offset, "noise_offset_random_strength": noise_offset_random_strength, "noise_offset_type": noise_offset_type, - "optimizer": optimizer, - "optimizer_args": optimizer_args, + "optimizer_type": optimizer, + "optimizer_args": str(optimizer_args).replace('"', '').split(), "output_dir": output_dir, "output_name": output_name, "persistent_data_loader_workers": persistent_data_loader_workers, @@ -1004,7 +1175,12 @@ def train_model( "prior_loss_weight": prior_loss_weight, "random_crop": random_crop, "reg_data_dir": reg_data_dir, + "resolution": max_resolution, "resume": resume, + "sample_every_n_epochs": sample_every_n_epochs, + "sample_every_n_steps": sample_every_n_steps, + "sample_prompts": create_prompt_file(output_dir, output_dir), + "sample_sampler": sample_sampler, "save_every_n_epochs": save_every_n_epochs, "save_every_n_steps": save_every_n_steps, "save_last_n_steps": save_last_n_steps, @@ -1015,7 +1191,7 @@ def train_model( "save_state_on_train_end": save_state_on_train_end, "scale_v_pred_loss_like_noise_pred": scale_v_pred_loss_like_noise_pred, "scale_weight_norms": scale_weight_norms, - "seed": seed, + "seed": int(seed), "shuffle_caption": shuffle_caption, "stop_text_encoder_training": stop_text_encoder_training, "text_encoder_lr": text_encoder_lr, @@ -1039,17 +1215,30 @@ def train_model( "huber_c": huber_c, } - # Use the ** syntax to unpack the dictionary when calling the function - run_cmd = run_cmd_advanced_training(run_cmd=run_cmd, **run_cmd_params) + # Given dictionary `config_toml_data` + # Remove all values = "" + config_toml_data = { + key: value + for key, value in config_toml_data.items() + if value != "" and value != False + } - run_cmd = run_cmd_sample( - run_cmd, - sample_every_n_steps, - sample_every_n_epochs, - sample_sampler, - sample_prompts, - output_dir, - ) + tmpfilename = "./outputs/tmpfilelora.toml" + # Save the updated TOML data back to the file + with open(tmpfilename, "w") as toml_file: + toml.dump(config_toml_data, toml_file) + + if not os.path.exists(toml_file.name): + log.error(f"Failed to write TOML file: {toml_file.name}") + + toml_file_path = ( + os.path.abspath(os.path.normpath(toml_file.name)).replace("\\", "/") + if os.name == "nt" + else toml_file.name + ) + + run_cmd.append(f"--config_file") + run_cmd.append(rf"{toml_file_path}") if print_only: log.warning( @@ -1898,6 +2087,10 @@ def lora_tab( with gr.Accordion("Samples", open=False, elem_id="samples_tab"): sample = SampleImages(config=config) + global huggingface + with gr.Accordion("HuggingFace", open=False): + huggingface = HuggingFace(config=config) + LoRA_type.change( update_LoRA_settings, inputs=[ @@ -2094,6 +2287,14 @@ def lora_tab( LyCORIS_preset, advanced_training.debiased_estimation_loss, accelerate_launch.extra_accelerate_launch_args, + huggingface.huggingface_repo_id, + huggingface.huggingface_token, + huggingface.huggingface_repo_type, + huggingface.huggingface_repo_visibility, + huggingface.huggingface_path_in_repo, + huggingface.save_state_to_huggingface, + huggingface.resume_from_huggingface, + huggingface.async_upload, ] configuration.button_open_config.click( diff --git a/kohya_gui/textual_inversion_gui.py b/kohya_gui/textual_inversion_gui.py index 9c7f675..75bd043 100644 --- a/kohya_gui/textual_inversion_gui.py +++ b/kohya_gui/textual_inversion_gui.py @@ -2,7 +2,7 @@ import gradio as gr import json import math import os -import shlex +import toml import time from datetime import datetime from .common_gui import ( @@ -29,12 +29,13 @@ from .class_advanced_training import AdvancedTraining from .class_folders import Folders from .class_sdxl_parameters import SDXLParameters from .class_command_executor import CommandExecutor +from .class_huggingface import HuggingFace from .class_tensorboard import TensorboardManager from .dreambooth_folder_creation_gui import ( gradio_dreambooth_folder_creation_tab, ) from .dataset_balancing_gui import gradio_dataset_balancing_tab -from .class_sample_images import SampleImages, run_cmd_sample +from .class_sample_images import SampleImages, run_cmd_sample, create_prompt_file from .custom_logging import setup_logging @@ -44,6 +45,9 @@ log = setup_logging() # Setup command executor executor = CommandExecutor() +# Setup huggingface +huggingface = None + TRAIN_BUTTON_VISIBLE = [gr.Button(visible=True), gr.Button(visible=False)] @@ -154,6 +158,14 @@ def save_configuration( max_timestep, sdxl_no_half_vae, extra_accelerate_launch_args, + huggingface_repo_id, + huggingface_token, + huggingface_repo_type, + huggingface_repo_visibility, + huggingface_path_in_repo, + save_state_to_huggingface, + resume_from_huggingface, + async_upload, ): # Get list of function parameters and values parameters = list(locals().items()) @@ -296,6 +308,14 @@ def open_configuration( max_timestep, sdxl_no_half_vae, extra_accelerate_launch_args, + huggingface_repo_id, + huggingface_token, + huggingface_repo_type, + huggingface_repo_visibility, + huggingface_path_in_repo, + save_state_to_huggingface, + resume_from_huggingface, + async_upload, ): # Get list of function parameters and values parameters = list(locals().items()) @@ -431,6 +451,14 @@ def train_model( max_timestep, sdxl_no_half_vae, extra_accelerate_launch_args, + huggingface_repo_id, + huggingface_token, + huggingface_repo_type, + huggingface_repo_visibility, + huggingface_path_in_repo, + save_state_to_huggingface, + resume_from_huggingface, + async_upload, ): # Get list of function parameters and values parameters = list(locals().items()) @@ -565,111 +593,175 @@ def train_model( else: run_cmd.append(f"{scriptdir}/sd-scripts/train_textual_inversion.py") - run_cmd = run_cmd_advanced_training( - run_cmd=run_cmd, - adaptive_noise_scale=adaptive_noise_scale, - bucket_no_upscale=bucket_no_upscale, - bucket_reso_steps=bucket_reso_steps, - cache_latents=cache_latents, - cache_latents_to_disk=cache_latents_to_disk, - caption_dropout_every_n_epochs=caption_dropout_every_n_epochs, - caption_extension=caption_extension, - clip_skip=clip_skip, - color_aug=color_aug, - dataset_config=dataset_config, - enable_bucket=enable_bucket, - epoch=epoch, - flip_aug=flip_aug, - full_fp16=full_fp16, - gradient_accumulation_steps=gradient_accumulation_steps, - gradient_checkpointing=gradient_checkpointing, - ip_noise_gamma=ip_noise_gamma, - ip_noise_gamma_random_strength=ip_noise_gamma_random_strength, - keep_tokens=keep_tokens, - learning_rate=learning_rate, - logging_dir=logging_dir, - log_tracker_name=log_tracker_name, - log_tracker_config=log_tracker_config, - lr_scheduler=lr_scheduler, - lr_scheduler_args=lr_scheduler_args, - lr_scheduler_num_cycles=lr_scheduler_num_cycles, - lr_scheduler_power=lr_scheduler_power, - lr_warmup_steps=lr_warmup_steps, - max_bucket_reso=max_bucket_reso, - max_data_loader_n_workers=max_data_loader_n_workers, - max_resolution=max_resolution, - max_timestep=max_timestep, - max_token_length=max_token_length, - max_train_epochs=max_train_epochs, - max_train_steps=max_train_steps, - mem_eff_attn=mem_eff_attn, - min_bucket_reso=min_bucket_reso, - min_snr_gamma=min_snr_gamma, - min_timestep=min_timestep, - mixed_precision=mixed_precision, - multires_noise_discount=multires_noise_discount, - multires_noise_iterations=multires_noise_iterations, - no_half_vae=True if sdxl and sdxl_no_half_vae else None, - no_token_padding=no_token_padding, - noise_offset=noise_offset, - noise_offset_random_strength=noise_offset_random_strength, - noise_offset_type=noise_offset_type, - optimizer=optimizer, - optimizer_args=optimizer_args, - output_dir=output_dir, - output_name=output_name, - persistent_data_loader_workers=persistent_data_loader_workers, - pretrained_model_name_or_path=pretrained_model_name_or_path, - prior_loss_weight=prior_loss_weight, - random_crop=random_crop, - reg_data_dir=reg_data_dir, - resume=resume, - save_every_n_epochs=save_every_n_epochs, - save_every_n_steps=save_every_n_steps, - save_last_n_steps=save_last_n_steps, - save_last_n_steps_state=save_last_n_steps_state, - save_model_as=save_model_as, - save_precision=save_precision, - save_state=save_state, - save_state_on_train_end=save_state_on_train_end, - scale_v_pred_loss_like_noise_pred=scale_v_pred_loss_like_noise_pred, - seed=seed, - shuffle_caption=shuffle_caption, - stop_text_encoder_training=stop_text_encoder_training, - train_batch_size=train_batch_size, - train_data_dir=train_data_dir, - use_wandb=use_wandb, - v2=v2, - v_parameterization=v_parameterization, - v_pred_like_loss=v_pred_like_loss, - vae=vae, - vae_batch_size=vae_batch_size, - wandb_api_key=wandb_api_key, - wandb_run_name=wandb_run_name, - xformers=xformers, - additional_parameters=additional_parameters, - loss_type=loss_type, - huber_schedule=huber_schedule, - huber_c=huber_c, - ) - run_cmd.append(f'--token_string="{shlex.quote(token_string)}"') - run_cmd.append(f'--init_word="{shlex.quote(init_word)}"') - run_cmd.append(f"--num_vectors_per_token={int(num_vectors_per_token)}") - if not weights == "": - run_cmd.append(f'--weights="{shlex.quote(weights)}"') - if template == "object template": - run_cmd.append("--use_object_template") - elif template == "style template": - run_cmd.append("--use_style_template") + # Initialize a dictionary with always-included keyword arguments + kwargs_for_training = { + "max_data_loader_n_workers": max_data_loader_n_workers, + "additional_parameters": additional_parameters, + } + + # run_cmd.append(f'--token_string="{shlex.quote(token_string)}"') + # run_cmd.append(f'--init_word="{shlex.quote(init_word)}"') + # run_cmd.append(f"--num_vectors_per_token={int(num_vectors_per_token)}") + # if not weights == "": + # run_cmd.append(f'--weights="{shlex.quote(weights)}"') + # if template == "object template": + # run_cmd.append("--use_object_template") + # elif template == "style template": + # run_cmd.append("--use_style_template") - run_cmd = run_cmd_sample( - run_cmd, - sample_every_n_steps, - sample_every_n_epochs, - sample_sampler, - sample_prompts, - output_dir, - ) + # Pass the dynamically constructed keyword arguments to the function + run_cmd = run_cmd_advanced_training(run_cmd=run_cmd, **kwargs_for_training) + + # run_cmd = run_cmd_sample( + # run_cmd, + # sample_every_n_steps, + # sample_every_n_epochs, + # sample_sampler, + # sample_prompts, + # output_dir, + # ) + + if max_data_loader_n_workers == "" or None: + max_data_loader_n_workers = 0 + else: + max_data_loader_n_workers = int(max_data_loader_n_workers) + + if max_train_steps == "" or None: + max_train_steps = 0 + else: + max_train_steps = int(max_train_steps) + + # def save_huggingface_to_toml(self, toml_file_path: str): + config_toml_data = { + # Update the values in the TOML data + "huggingface_repo_id": huggingface_repo_id, + "huggingface_token": huggingface_token, + "huggingface_repo_type": huggingface_repo_type, + "huggingface_repo_visibility": huggingface_repo_visibility, + "huggingface_path_in_repo": huggingface_path_in_repo, + "save_state_to_huggingface": save_state_to_huggingface, + "resume_from_huggingface": resume_from_huggingface, + "async_upload": async_upload, + "adaptive_noise_scale": adaptive_noise_scale, + "bucket_no_upscale": bucket_no_upscale, + "bucket_reso_steps": bucket_reso_steps, + "cache_latents": cache_latents, + "cache_latents_to_disk": cache_latents_to_disk, + "caption_dropout_every_n_epochs": caption_dropout_every_n_epochs, + "caption_extension": caption_extension, + "clip_skip": int(clip_skip), + "color_aug": color_aug, + "dataset_config": dataset_config, + "enable_bucket": enable_bucket, + "epoch": int(epoch), + "flip_aug": flip_aug, + "full_fp16": full_fp16, + "gradient_accumulation_steps": gradient_accumulation_steps, + "gradient_checkpointing": gradient_checkpointing, + "init_word": init_word, + "ip_noise_gamma": ip_noise_gamma, + "ip_noise_gamma_random_strength": ip_noise_gamma_random_strength, + "keep_tokens": int(keep_tokens), + "learning_rate": learning_rate, + "logging_dir": logging_dir, + "log_tracker_name": log_tracker_name, + "log_tracker_config": log_tracker_config, + "lr_scheduler": lr_scheduler, + "lr_scheduler_args": str(lr_scheduler_args).replace('"', '').split(), + "lr_scheduler_num_cycles": lr_scheduler_num_cycles, + "lr_scheduler_power": lr_scheduler_power, + "lr_warmup_steps": lr_warmup_steps, + "max_bucket_reso": max_bucket_reso, + "max_data_loader_n_workers": max_data_loader_n_workers, + "max_timestep": max_timestep, + "max_token_length": int(max_token_length), + "max_train_epochs": max_train_epochs, + "max_train_steps": int(max_train_steps), + "mem_eff_attn": mem_eff_attn, + "min_bucket_reso": int(min_bucket_reso), + "min_snr_gamma": min_snr_gamma, + "min_timestep": int(min_timestep), + "mixed_precision": mixed_precision, + "multires_noise_discount": multires_noise_discount, + "multires_noise_iterations": multires_noise_iterations, + "no_half_vae": sdxl_no_half_vae, + "no_token_padding": no_token_padding, + "noise_offset": noise_offset, + "noise_offset_random_strength": noise_offset_random_strength, + "noise_offset_type": noise_offset_type, + "num_vectors_per_token": int(num_vectors_per_token), + "optimizer_type": optimizer, + "optimizer_args": str(optimizer_args).replace('"', '').split(), + "output_dir": output_dir, + "output_name": output_name, + "persistent_data_loader_workers": persistent_data_loader_workers, + "pretrained_model_name_or_path": pretrained_model_name_or_path, + "prior_loss_weight": prior_loss_weight, + "random_crop": random_crop, + "reg_data_dir": reg_data_dir, + "resolution": max_resolution, + "resume": resume, + "sample_every_n_epochs": sample_every_n_epochs, + "sample_every_n_steps": sample_every_n_steps, + "sample_prompts": create_prompt_file(output_dir, output_dir), + "sample_sampler": sample_sampler, + "save_every_n_epochs": save_every_n_epochs, + "save_every_n_steps": save_every_n_steps, + "save_last_n_steps": save_last_n_steps, + "save_last_n_steps_state": save_last_n_steps_state, + "save_model_as": save_model_as, + "save_precision": save_precision, + "save_state": save_state, + "save_state_on_train_end": save_state_on_train_end, + "scale_v_pred_loss_like_noise_pred": scale_v_pred_loss_like_noise_pred, + "seed": int(seed), + "shuffle_caption": shuffle_caption, + "stop_text_encoder_training": stop_text_encoder_training, + "token_string": token_string, + "train_batch_size": train_batch_size, + "train_data_dir": train_data_dir, + "use_wandb": use_wandb, + "v2": v2, + "v_parameterization": v_parameterization, + "v_pred_like_loss": v_pred_like_loss, + "vae": vae, + "vae_batch_size": vae_batch_size, + "wandb_api_key": wandb_api_key, + "wandb_run_name": wandb_run_name, + "weigts": weights, + "use_object_template": True if template == "object template" else None, + "use_style_template": True if template == "style template" else None, + "xformers": xformers, + "additional_parameters": additional_parameters, + "loss_type": loss_type, + "huber_schedule": huber_schedule, + "huber_c": huber_c, + } + + # Given dictionary `config_toml_data` + # Remove all values = "" + config_toml_data = { + key: value + for key, value in config_toml_data.items() + if value != "" and value != False + } + + tmpfilename = "./outputs/tmpfileti.toml" + # Save the updated TOML data back to the file + with open(tmpfilename, "w") as toml_file: + toml.dump(config_toml_data, toml_file) + + if not os.path.exists(toml_file.name): + log.error(f"Failed to write TOML file: {toml_file.name}") + + toml_file_path = ( + os.path.abspath(os.path.normpath(toml_file.name)).replace("\\", "/") + if os.name == "nt" + else toml_file.name + ) + + run_cmd.append(f"--config_file") + run_cmd.append(rf"{toml_file_path}") if print_only: log.warning( @@ -870,6 +962,10 @@ def ti_tab(headless=False, default_output_dir=None, config: dict = {}): with gr.Accordion("Samples", open=False, elem_id="samples_tab"): sample = SampleImages(config=config) + global huggingface + with gr.Accordion("HuggingFace", open=False): + huggingface = HuggingFace(config=config) + with gr.Column(), gr.Group(): with gr.Row(): button_run = gr.Button("Start training", variant="primary") @@ -990,6 +1086,14 @@ def ti_tab(headless=False, default_output_dir=None, config: dict = {}): advanced_training.max_timestep, sdxl_params.sdxl_no_half_vae, accelerate_launch.extra_accelerate_launch_args, + huggingface.huggingface_repo_id, + huggingface.huggingface_token, + huggingface.huggingface_repo_type, + huggingface.huggingface_repo_visibility, + huggingface.huggingface_path_in_repo, + huggingface.save_state_to_huggingface, + huggingface.resume_from_huggingface, + huggingface.async_upload, ] configuration.button_open_config.click( diff --git a/test/config/dreambooth-AdamW8bit.json b/test/config/dreambooth-AdamW8bit.json index 52fea4d..ea1ef35 100644 --- a/test/config/dreambooth-AdamW8bit.json +++ b/test/config/dreambooth-AdamW8bit.json @@ -1,11 +1,12 @@ { "adaptive_noise_scale": 0, - "additional_parameters": "", + "additional_parameters": "--config_file ./outputs/tmpfile.toml", + "async_upload": false, "bucket_no_upscale": true, "bucket_reso_steps": 64, "cache_latents": true, "cache_latents_to_disk": false, - "caption_dropout_every_n_epochs": 0.0, + "caption_dropout_every_n_epochs": 0, "caption_dropout_rate": 0.05, "caption_extension": "", "clip_skip": 2, @@ -23,6 +24,11 @@ "gradient_checkpointing": false, "huber_c": 0.1, "huber_schedule": "snr", + "huggingface_path_in_repo": "", + "huggingface_repo_id": "", + "huggingface_repo_type": "", + "huggingface_repo_visibility": "", + "huggingface_token": "", "ip_noise_gamma": 0.1, "ip_noise_gamma_random_strength": true, "keep_tokens": "0", @@ -52,7 +58,7 @@ "min_bucket_reso": 256, "min_snr_gamma": 0, "min_timestep": 0, - "mixed_precision": "bf16", + "mixed_precision": "fp16", "model_list": "runwayml/stable-diffusion-v1-5", "multi_gpu": false, "multires_noise_discount": 0, @@ -63,21 +69,23 @@ "noise_offset_type": "Original", "num_cpu_threads_per_process": 2, "num_machines": 1, - "num_processes": 1, + "num_processes": 8, "optimizer": "AdamW8bit", "optimizer_args": "", "output_dir": "./test/output", "output_name": "db-AdamW8bit", "persistent_data_loader_workers": false, "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5", - "prior_loss_weight": 1.0, + "prior_loss_weight": 1, "random_crop": false, "reg_data_dir": "", "resume": "", + "resume_from_huggingface": "", "sample_every_n_epochs": 0, "sample_every_n_steps": 20, "sample_prompts": "a painting of a gas mask , by darius kawasaki", "sample_sampler": "euler_a", + "save_as_bool": false, "save_every_n_epochs": 1, "save_every_n_steps": 0, "save_last_n_steps": 0, @@ -86,6 +94,7 @@ "save_precision": "fp16", "save_state": false, "save_state_on_train_end": false, + "save_state_to_huggingface": false, "scale_v_pred_loss_like_noise_pred": false, "sdxl": false, "seed": "1234", @@ -97,7 +106,7 @@ "v2": false, "v_parameterization": false, "v_pred_like_loss": 0, - "vae": "stabilityai/vae", + "vae": "", "vae_batch_size": 0, "wandb_api_key": "", "wandb_run_name": "", diff --git a/test/config/dreambooth-Prodigy.json b/test/config/dreambooth-Prodigy.json index 0bbc337..a7575c9 100644 --- a/test/config/dreambooth-Prodigy.json +++ b/test/config/dreambooth-Prodigy.json @@ -1,54 +1,91 @@ { "adaptive_noise_scale": 0, "additional_parameters": "", + "async_upload": false, "bucket_no_upscale": true, "bucket_reso_steps": 1, "cache_latents": true, "cache_latents_to_disk": false, - "caption_dropout_every_n_epochs": 0.0, + "caption_dropout_every_n_epochs": 0, "caption_dropout_rate": 0, "caption_extension": "", - "clip_skip": 2, + "clip_skip": 1, "color_aug": false, + "dataset_config": "", + "debiased_estimation_loss": true, "enable_bucket": true, "epoch": 1, + "extra_accelerate_launch_args": "", "flip_aug": false, + "full_bf16": false, "full_fp16": false, - "gradient_accumulation_steps": 1.0, + "gpu_ids": "", + "gradient_accumulation_steps": 1, "gradient_checkpointing": false, + "huber_c": 0.1, + "huber_schedule": "snr", + "huggingface_path_in_repo": "", + "huggingface_repo_id": "", + "huggingface_repo_type": "", + "huggingface_repo_visibility": "", + "huggingface_token": "", + "ip_noise_gamma": 0.1, + "ip_noise_gamma_random_strength": true, "keep_tokens": "0", - "learning_rate": 1.0, + "learning_rate": 1, + "learning_rate_te": 1e-05, + "learning_rate_te1": 1e-05, + "learning_rate_te2": 0, + "log_tracker_config": "", + "log_tracker_name": "", "logging_dir": "./test/logs", + "loss_type": "l2", "lr_scheduler": "cosine", + "lr_scheduler_args": "", + "lr_scheduler_num_cycles": "", + "lr_scheduler_power": "", "lr_warmup": 0, + "main_process_port": 0, + "masked_loss": false, + "max_bucket_reso": 2048, "max_data_loader_n_workers": "0", "max_resolution": "512,512", + "max_timestep": 1000, "max_token_length": "75", "max_train_epochs": "", + "max_train_steps": "30", "mem_eff_attn": false, + "min_bucket_reso": 256, "min_snr_gamma": 0, + "min_timestep": 0, "mixed_precision": "bf16", "model_list": "runwayml/stable-diffusion-v1-5", + "multi_gpu": false, "multires_noise_discount": 0.2, "multires_noise_iterations": 8, "no_token_padding": false, - "noise_offset": "0.05", + "noise_offset": 0.05, + "noise_offset_random_strength": true, "noise_offset_type": "Multires", "num_cpu_threads_per_process": 2, + "num_machines": 1, + "num_processes": 1, "optimizer": "Prodigy", "optimizer_args": "decouple=True weight_decay=0.6 betas=0.9,0.99 use_bias_correction=True", "output_dir": "./test/output", "output_name": "db-Prodigy", "persistent_data_loader_workers": false, "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5", - "prior_loss_weight": 1.0, + "prior_loss_weight": 1, "random_crop": false, "reg_data_dir": "", "resume": "", + "resume_from_huggingface": "", "sample_every_n_epochs": 0, "sample_every_n_steps": 25, "sample_prompts": "a painting of a gas mask , by darius kawasaki", "sample_sampler": "euler_a", + "save_as_bool": false, "save_every_n_epochs": 1, "save_every_n_steps": 0, "save_last_n_steps": 0, @@ -56,7 +93,10 @@ "save_model_as": "safetensors", "save_precision": "fp16", "save_state": false, + "save_state_on_train_end": false, + "save_state_to_huggingface": false, "scale_v_pred_loss_like_noise_pred": false, + "sdxl": false, "seed": "1234", "shuffle_caption": false, "stop_text_encoder_training": 0, @@ -65,9 +105,11 @@ "use_wandb": false, "v2": false, "v_parameterization": false, + "v_pred_like_loss": 0, "vae": "", "vae_batch_size": 0, "wandb_api_key": "", + "wandb_run_name": "", "weighted_captions": false, - "xformers": true + "xformers": "xformers" } \ No newline at end of file