1580158
/
kohya_ss
mirror of https://github.com/bmaltais/kohya_ss
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

- Implement HuggingFace inputs in all training tabs (#2287) 

- Switch to sd-scripts configuration files to pass most of the parameters in a secure way.
pull/2292/head
bmaltais 2024-04-14 18:43:45 -04:00 committed by GitHub
parent fa5c65eade
commit e98fc40a63
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 1044 additions and 224 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
README.md
View File

@ -404,10 +404,12 @@ ControlNet dataset is used to specify the mask. The mask images should be the RG
### 2024/04/12 (v23.1.6)
- Rewrote a lot of the code to fix issues resulting from the security upgrades to remove shell=True from process calls.
- Improved the training and tensorboard buttons
- Upgrade the gradio version to 4.20.0 to fix a bug with runpod.
- Various other minor fixes.
- Rewrote significant portions of the code to address security vulnerabilities and remove the `shell=True` parameter from process calls.
- Enhanced the training and tensorboard buttons to provide a more intuitive and user-friendly experience.
- Upgraded the gradio version to 4.20.0 to address a bug that was causing issues with the runpod platform.
- Added a HuggingFace section to all trainers tabs, enabling users to authenticate and utilize HuggingFace's powerful AI models.
- Converted the Graphical User Interface (GUI) to use the configuration TOML file format to pass arguments to sd-scripts. This change improves security by eliminating the need for sensitive information to be passed through the command-line interface (CLI).
- Made various other minor improvements and bug fixes to enhance the overall functionality and user experience.
### 2024/04/10 (v23.1.5)

4
kohya_gui/class_command_executor.py
View File

@ -32,8 +32,8 @@ class CommandExecutor:
if self.process and self.process.poll() is None:
log.info("The command is already running. Please wait for it to finish.")
else:
for i, item in enumerate(run_cmd):
log.info(f"{i}: {item}")
# for i, item in enumerate(run_cmd):
# log.info(f"{i}: {item}")
# Reconstruct the safe command string for display
command_to_run = ' '.join(run_cmd)

82
kohya_gui/class_huggingface.py Normal file
View File

@ -0,0 +1,82 @@
import gradio as gr
import toml
from .class_gui_config import KohyaSSGUIConfig
class HuggingFace:
def __init__(
self,
config: KohyaSSGUIConfig = {},
) -> None:
self.config = config
# Initialize the UI components
self.initialize_ui_components()
def initialize_ui_components(self) -> None:
# --huggingface_repo_id HUGGINGFACE_REPO_ID
# huggingface repo name to upload / huggingfaceにアップロードするリポジトリ名
# --huggingface_repo_type HUGGINGFACE_REPO_TYPE
# huggingface repo type to upload / huggingfaceにアップロードするリポジトリの種類
# --huggingface_path_in_repo HUGGINGFACE_PATH_IN_REPO
# huggingface model path to upload files / huggingfaceにアップロードするファイルのパス
# --huggingface_token HUGGINGFACE_TOKEN
# huggingface token / huggingfaceのトークン
# --huggingface_repo_visibility HUGGINGFACE_REPO_VISIBILITY
# huggingface repository visibility ('public' for public, 'private' or None for private) / huggingfaceにアップロードするリポジトリの公開設定（'public'で公開、'private'またはNoneで非公開）
# --save_state_to_huggingface
# save state to huggingface / huggingfaceにstateを保存する
# --resume_from_huggingface
# resume from huggingface (ex: --resume {repo_id}/{path_in_repo}:{revision}:{repo_type}) / huggingfaceから学習を再開する(例: --resume {repo_id}/{path_in_repo}:{revision}:{repo_type})
# --async_upload upload to huggingface asynchronously / huggingfaceに非同期でアップロードする
with gr.Row():
self.huggingface_repo_id = gr.Textbox(
label="Huggingface repo id",
placeholder="huggingface repo id",
value=self.config.get("huggingface.repo_id", ""),
)
self.huggingface_token = gr.Textbox(
label="Huggingface token",
placeholder="huggingface token",
value=self.config.get("huggingface.token", ""),
)
with gr.Row():
# Repository settings
self.huggingface_repo_type = gr.Textbox(
label="Huggingface repo type",
placeholder="huggingface repo type",
value=self.config.get("huggingface.repo_type", ""),
)
self.huggingface_repo_visibility = gr.Textbox(
label="Huggingface repo visibility",
placeholder="huggingface repo visibility",
value=self.config.get("huggingface.repo_visibility", ""),
)
with gr.Row():
# File location in the repository
self.huggingface_path_in_repo = gr.Textbox(
label="Huggingface path in repo",
placeholder="huggingface path in repo",
value=self.config.get("huggingface.path_in_repo", ""),
)
with gr.Row():
# Functions
self.save_state_to_huggingface = gr.Checkbox(
label="Save state to huggingface",
value=self.config.get("huggingface.save_state_to_huggingface", False),
)
self.resume_from_huggingface = gr.Textbox(
label="Resume from huggingface",
placeholder="resume from huggingface",
value=self.config.get("huggingface.resume_from_huggingface", ""),
)
self.async_upload = gr.Checkbox(
label="Async upload",
value=self.config.get("huggingface.async_upload", False),
)

17
kohya_gui/class_sample_images.py
View File

@ -17,6 +17,23 @@ document_symbol = "\U0001F4C4" # 📄
###
### Gradio common sampler GUI section
###
def create_prompt_file(sample_prompts, output_dir):
"""
Creates a prompt file for image sampling.
Args:
sample_prompts (str): The prompts to use for image sampling.
output_dir (str): The directory where the output images will be saved.
Returns:
str: The path to the prompt file.
"""
sample_prompts_path = os.path.join(output_dir, "prompt.txt")
with open(sample_prompts_path, "w") as f:
f.write(sample_prompts)
return sample_prompts_path
def run_cmd_sample(

3
kohya_gui/common_gui.py
View File

@ -1575,7 +1575,8 @@ def run_cmd_advanced_training(run_cmd: list = [], **kwargs):
# Use Weights and Biases logging
if kwargs.get("use_wandb"):
run_cmd.append("--log_with wandb")
run_cmd.append("--log_with")
run_cmd.append("wandb")
# V parameterization
if kwargs.get("v_parameterization"):

243
kohya_gui/dreambooth_gui.py
View File

@ -4,9 +4,9 @@ import math
import os
import time
import sys
import toml
from datetime import datetime
from .common_gui import (
get_executable_path,
get_file_path,
get_saveasfile_path,
color_aug_changed,
@ -26,12 +26,13 @@ from .class_basic_training import BasicTraining
from .class_advanced_training import AdvancedTraining
from .class_folders import Folders
from .class_command_executor import CommandExecutor
from .class_huggingface import HuggingFace
from .dreambooth_folder_creation_gui import (
gradio_dreambooth_folder_creation_tab,
)
from .dataset_balancing_gui import gradio_dataset_balancing_tab
from .class_sample_images import SampleImages, run_cmd_sample
from .class_sample_images import SampleImages, run_cmd_sample, create_prompt_file
from .class_tensorboard import TensorboardManager
from .custom_logging import setup_logging
@ -42,6 +43,9 @@ log = setup_logging()
# Setup command executor
executor = CommandExecutor()
# Setup huggingface
huggingface = None
PYTHON = sys.executable
TRAIN_BUTTON_VISIBLE = [gr.Button(visible=True), gr.Button(visible=False)]
@ -155,6 +159,14 @@ def save_configuration(
max_timestep,
debiased_estimation_loss,
extra_accelerate_launch_args,
huggingface_repo_id,
huggingface_token,
huggingface_repo_type,
huggingface_repo_visibility,
huggingface_path_in_repo,
save_state_to_huggingface,
resume_from_huggingface,
async_upload,
):
# Get list of function parameters and values
parameters = list(locals().items())
@ -296,6 +308,14 @@ def open_configuration(
max_timestep,
debiased_estimation_loss,
extra_accelerate_launch_args,
huggingface_repo_id,
huggingface_token,
huggingface_repo_type,
huggingface_repo_visibility,
huggingface_path_in_repo,
save_state_to_huggingface,
resume_from_huggingface,
async_upload,
):
# Get list of function parameters and values
parameters = list(locals().items())
@ -432,6 +452,14 @@ def train_model(
max_timestep,
debiased_estimation_loss,
extra_accelerate_launch_args,
huggingface_repo_id,
huggingface_token,
huggingface_repo_type,
huggingface_repo_visibility,
huggingface_path_in_repo,
save_state_to_huggingface,
resume_from_huggingface,
async_upload,
):
# Get list of function parameters and values
parameters = list(locals().items())
@ -587,6 +615,136 @@ def train_model(
# Initialize a dictionary with always-included keyword arguments
kwargs_for_training = {
# "adaptive_noise_scale": adaptive_noise_scale,
# "bucket_no_upscale": bucket_no_upscale,
# "bucket_reso_steps": bucket_reso_steps,
# "cache_latents": cache_latents,
# "cache_latents_to_disk": cache_latents_to_disk,
# "caption_dropout_every_n_epochs": caption_dropout_every_n_epochs,
# "caption_dropout_rate": caption_dropout_rate,
# "caption_extension": caption_extension,
# "clip_skip": clip_skip,
# "color_aug": color_aug,
# "dataset_config": dataset_config,
# "debiased_estimation_loss": debiased_estimation_loss,
# "enable_bucket": enable_bucket,
# "epoch": epoch,
# "flip_aug": flip_aug,
# "masked_loss": masked_loss,
# "full_bf16": full_bf16,
# "full_fp16": full_fp16,
# "gradient_accumulation_steps": gradient_accumulation_steps,
# "gradient_checkpointing": gradient_checkpointing,
# "ip_noise_gamma": ip_noise_gamma,
# "ip_noise_gamma_random_strength": ip_noise_gamma_random_strength,
# "keep_tokens": keep_tokens,
# "learning_rate": learning_rate,
# "logging_dir": logging_dir,
# "log_tracker_name": log_tracker_name,
# "log_tracker_config": log_tracker_config,
# "lr_scheduler": lr_scheduler,
# "lr_scheduler_args": lr_scheduler_args,
# "lr_scheduler_num_cycles": lr_scheduler_num_cycles,
# "lr_scheduler_power": lr_scheduler_power,
# "lr_warmup_steps": lr_warmup_steps,
# "max_bucket_reso": max_bucket_reso,
"max_data_loader_n_workers": max_data_loader_n_workers,
# "max_resolution": max_resolution,
# "max_timestep": max_timestep,
# "max_token_length": max_token_length,
# "max_train_epochs": max_train_epochs,
# "max_train_steps": max_train_steps,
# "mem_eff_attn": mem_eff_attn,
# "min_bucket_reso": min_bucket_reso,
# "min_snr_gamma": min_snr_gamma,
# "min_timestep": min_timestep,
# "mixed_precision": mixed_precision,
# "multires_noise_discount": multires_noise_discount,
# "multires_noise_iterations": multires_noise_iterations,
# "no_token_padding": no_token_padding,
# "noise_offset": noise_offset,
# "noise_offset_random_strength": noise_offset_random_strength,
# "noise_offset_type": noise_offset_type,
# "optimizer": optimizer,
# "optimizer_args": optimizer_args,
# "output_dir": output_dir,
# "output_name": output_name,
# "persistent_data_loader_workers": persistent_data_loader_workers,
# "pretrained_model_name_or_path": pretrained_model_name_or_path,
# "prior_loss_weight": prior_loss_weight,
# "random_crop": random_crop,
# "reg_data_dir": reg_data_dir,
# "resume": resume,
# "save_every_n_epochs": save_every_n_epochs,
# "save_every_n_steps": save_every_n_steps,
# "save_last_n_steps": save_last_n_steps,
# "save_last_n_steps_state": save_last_n_steps_state,
# "save_model_as": save_model_as,
# "save_precision": save_precision,
# "save_state": save_state,
# "save_state_on_train_end": save_state_on_train_end,
# "scale_v_pred_loss_like_noise_pred": scale_v_pred_loss_like_noise_pred,
# "seed": seed,
# "shuffle_caption": shuffle_caption,
# "stop_text_encoder_training": stop_text_encoder_training,
# "train_batch_size": train_batch_size,
# "train_data_dir": train_data_dir,
# "use_wandb": use_wandb,
# "v2": v2,
# "v_parameterization": v_parameterization,
# "v_pred_like_loss": v_pred_like_loss,
# "vae": vae,
# "vae_batch_size": vae_batch_size,
# "wandb_api_key": wandb_api_key,
# "wandb_run_name": wandb_run_name,
# "weighted_captions": weighted_captions,
# "xformers": xformers,
"additional_parameters": additional_parameters,
# "loss_type": loss_type,
# "huber_schedule": huber_schedule,
# "huber_c": huber_c,
}
# Conditionally include specific keyword arguments based on sdxl
# if sdxl:
# kwargs_for_training["learning_rate_te1"] = learning_rate_te1
# kwargs_for_training["learning_rate_te2"] = learning_rate_te2
# else:
# kwargs_for_training["learning_rate_te"] = learning_rate_te
# Pass the dynamically constructed keyword arguments to the function
run_cmd = run_cmd_advanced_training(run_cmd=run_cmd, **kwargs_for_training)
# run_cmd = run_cmd_sample(
# run_cmd,
# sample_every_n_steps,
# sample_every_n_epochs,
# sample_sampler,
# sample_prompts,
# output_dir,
# )
if max_data_loader_n_workers == "" or None:
max_data_loader_n_workers = 0
else:
max_data_loader_n_workers = int(max_data_loader_n_workers)
if max_train_steps == "" or None:
max_train_steps = 0
else:
max_train_steps = int(max_train_steps)
# def save_huggingface_to_toml(self, toml_file_path: str):
config_toml_data = {
# Update the values in the TOML data
"huggingface_repo_id": huggingface_repo_id,
"huggingface_token": huggingface_token,
"huggingface_repo_type": huggingface_repo_type,
"huggingface_repo_visibility": huggingface_repo_visibility,
"huggingface_path_in_repo": huggingface_path_in_repo,
"save_state_to_huggingface": save_state_to_huggingface,
"resume_from_huggingface": resume_from_huggingface,
"async_upload": async_upload,
"adaptive_noise_scale": adaptive_noise_scale,
"bucket_no_upscale": bucket_no_upscale,
"bucket_reso_steps": bucket_reso_steps,
@ -595,12 +753,12 @@ def train_model(
"caption_dropout_every_n_epochs": caption_dropout_every_n_epochs,
"caption_dropout_rate": caption_dropout_rate,
"caption_extension": caption_extension,
"clip_skip": clip_skip,
"clip_skip": int(clip_skip),
"color_aug": color_aug,
"dataset_config": dataset_config,
"debiased_estimation_loss": debiased_estimation_loss,
"enable_bucket": enable_bucket,
"epoch": epoch,
"epoch": int(epoch),
"flip_aug": flip_aug,
"masked_loss": masked_loss,
"full_bf16": full_bf16,
@ -609,27 +767,29 @@ def train_model(
"gradient_checkpointing": gradient_checkpointing,
"ip_noise_gamma": ip_noise_gamma,
"ip_noise_gamma_random_strength": ip_noise_gamma_random_strength,
"keep_tokens": keep_tokens,
"keep_tokens": int(keep_tokens),
"learning_rate": learning_rate,
"learning_rate_te": learning_rate_te,
"learning_rate_te1": learning_rate_te1,
"learning_rate_te2": learning_rate_te2,
"logging_dir": logging_dir,
"log_tracker_name": log_tracker_name,
"log_tracker_config": log_tracker_config,
"lr_scheduler": lr_scheduler,
"lr_scheduler_args": lr_scheduler_args,
"lr_scheduler_args": str(lr_scheduler_args).replace('"', '').split(),
"lr_scheduler_num_cycles": lr_scheduler_num_cycles,
"lr_scheduler_power": lr_scheduler_power,
"lr_warmup_steps": lr_warmup_steps,
"max_bucket_reso": max_bucket_reso,
"max_data_loader_n_workers": max_data_loader_n_workers,
"max_resolution": max_resolution,
"max_timestep": max_timestep,
"max_token_length": max_token_length,
"max_token_length": int(max_token_length),
"max_train_epochs": max_train_epochs,
"max_train_steps": max_train_steps,
"max_train_steps": int(max_train_steps),
"mem_eff_attn": mem_eff_attn,
"min_bucket_reso": min_bucket_reso,
"min_bucket_reso": int(min_bucket_reso),
"min_snr_gamma": min_snr_gamma,
"min_timestep": min_timestep,
"min_timestep": int(min_timestep),
"mixed_precision": mixed_precision,
"multires_noise_discount": multires_noise_discount,
"multires_noise_iterations": multires_noise_iterations,
@ -637,8 +797,8 @@ def train_model(
"noise_offset": noise_offset,
"noise_offset_random_strength": noise_offset_random_strength,
"noise_offset_type": noise_offset_type,
"optimizer": optimizer,
"optimizer_args": optimizer_args,
"optimizer_type": optimizer,
"optimizer_args": str(optimizer_args).replace('"', '').split(),
"output_dir": output_dir,
"output_name": output_name,
"persistent_data_loader_workers": persistent_data_loader_workers,
@ -646,7 +806,12 @@ def train_model(
"prior_loss_weight": prior_loss_weight,
"random_crop": random_crop,
"reg_data_dir": reg_data_dir,
"resolution": max_resolution,
"resume": resume,
"sample_every_n_epochs": sample_every_n_epochs,
"sample_every_n_steps": sample_every_n_steps,
"sample_prompts": create_prompt_file(output_dir, output_dir),
"sample_sampler": sample_sampler,
"save_every_n_epochs": save_every_n_epochs,
"save_every_n_steps": save_every_n_steps,
"save_last_n_steps": save_last_n_steps,
@ -656,7 +821,7 @@ def train_model(
"save_state": save_state,
"save_state_on_train_end": save_state_on_train_end,
"scale_v_pred_loss_like_noise_pred": scale_v_pred_loss_like_noise_pred,
"seed": seed,
"seed": int(seed),
"shuffle_caption": shuffle_caption,
"stop_text_encoder_training": stop_text_encoder_training,
"train_batch_size": train_batch_size,
@ -677,24 +842,30 @@ def train_model(
"huber_c": huber_c,
}
# Conditionally include specific keyword arguments based on sdxl
if sdxl:
kwargs_for_training["learning_rate_te1"] = learning_rate_te1
kwargs_for_training["learning_rate_te2"] = learning_rate_te2
else:
kwargs_for_training["learning_rate_te"] = learning_rate_te
# Given dictionary `config_toml_data`
# Remove all values = ""
config_toml_data = {
key: value
for key, value in config_toml_data.items()
if value != "" and value != False
}
# Pass the dynamically constructed keyword arguments to the function
run_cmd = run_cmd_advanced_training(run_cmd=run_cmd, **kwargs_for_training)
tmpfilename = "./outputs/tmpfiledbooth.toml"
# Save the updated TOML data back to the file
with open(tmpfilename, "w") as toml_file:
toml.dump(config_toml_data, toml_file)
run_cmd = run_cmd_sample(
run_cmd,
sample_every_n_steps,
sample_every_n_epochs,
sample_sampler,
sample_prompts,
output_dir,
)
if not os.path.exists(toml_file.name):
log.error(f"Failed to write TOML file: {toml_file.name}")
toml_file_path = (
os.path.abspath(os.path.normpath(toml_file.name)).replace("\\", "/")
if os.name == "nt"
else toml_file.name
)
run_cmd.append(f"--config_file")
run_cmd.append(rf"{toml_file_path}")
if print_only:
log.warning(
@ -806,6 +977,10 @@ def dreambooth_tab(
with gr.Accordion("Samples", open=False, elem_id="samples_tab"):
sample = SampleImages(config=config)
global huggingface
with gr.Accordion("HuggingFace", open=False):
huggingface = HuggingFace(config=config)
with gr.Column(), gr.Group():
with gr.Row():
button_run = gr.Button("Start training", variant="primary")
@ -927,6 +1102,14 @@ def dreambooth_tab(
advanced_training.max_timestep,
advanced_training.debiased_estimation_loss,
accelerate_launch.extra_accelerate_launch_args,
huggingface.huggingface_repo_id,
huggingface.huggingface_token,
huggingface.huggingface_repo_type,
huggingface.huggingface_repo_visibility,
huggingface.huggingface_path_in_repo,
huggingface.save_state_to_huggingface,
huggingface.resume_from_huggingface,
huggingface.async_upload,
]
configuration.button_open_config.click(

245
kohya_gui/finetune_gui.py
View File

@ -5,9 +5,9 @@ import os
import subprocess
import time
import sys
import toml
from datetime import datetime
from .common_gui import (
get_executable_path,
get_file_path,
get_saveasfile_path,
run_cmd_advanced_training,
@ -28,7 +28,8 @@ from .class_folders import Folders
from .class_sdxl_parameters import SDXLParameters
from .class_command_executor import CommandExecutor
from .class_tensorboard import TensorboardManager
from .class_sample_images import SampleImages, run_cmd_sample
from .class_sample_images import SampleImages, run_cmd_sample, create_prompt_file
from .class_huggingface import HuggingFace
from .custom_logging import setup_logging
@ -38,6 +39,9 @@ log = setup_logging()
# Setup command executor
executor = CommandExecutor()
# Setup huggingface
huggingface = None
# from easygui import msgbox
folder_symbol = "\U0001f4c2" # 📂
@ -163,6 +167,14 @@ def save_configuration(
min_timestep,
max_timestep,
extra_accelerate_launch_args,
huggingface_repo_id,
huggingface_token,
huggingface_repo_type,
huggingface_repo_visibility,
huggingface_path_in_repo,
save_state_to_huggingface,
resume_from_huggingface,
async_upload,
):
# Get list of function parameters and values
parameters = list(locals().items())
@ -311,6 +323,14 @@ def open_configuration(
min_timestep,
max_timestep,
extra_accelerate_launch_args,
huggingface_repo_id,
huggingface_token,
huggingface_repo_type,
huggingface_repo_visibility,
huggingface_path_in_repo,
save_state_to_huggingface,
resume_from_huggingface,
async_upload,
training_preset,
):
# Get list of function parameters and values
@ -465,6 +485,14 @@ def train_model(
min_timestep,
max_timestep,
extra_accelerate_launch_args,
huggingface_repo_id,
huggingface_token,
huggingface_repo_type,
huggingface_repo_visibility,
huggingface_path_in_repo,
save_state_to_huggingface,
resume_from_huggingface,
async_upload,
):
# Get list of function parameters and values
parameters = list(locals().items())
@ -651,19 +679,146 @@ def train_model(
# Initialize a dictionary with always-included keyword arguments
kwargs_for_training = {
# "adaptive_noise_scale": adaptive_noise_scale,
# "block_lr": block_lr,
# "bucket_no_upscale": bucket_no_upscale,
# "bucket_reso_steps": bucket_reso_steps,
# "cache_latents": cache_latents,
# "cache_latents_to_disk": cache_latents_to_disk,
# "caption_dropout_every_n_epochs": caption_dropout_every_n_epochs,
# "caption_dropout_rate": caption_dropout_rate,
# "caption_extension": caption_extension,
# "clip_skip": clip_skip,
# "color_aug": color_aug,
# "dataset_config": dataset_config,
# "dataset_repeats": dataset_repeats,
# "enable_bucket": True,
# "flip_aug": flip_aug,
# "masked_loss": masked_loss,
# "full_bf16": full_bf16,
# "full_fp16": full_fp16,
# "gradient_accumulation_steps": gradient_accumulation_steps,
# "gradient_checkpointing": gradient_checkpointing,
# "in_json": in_json,
# "ip_noise_gamma": ip_noise_gamma,
# "ip_noise_gamma_random_strength": ip_noise_gamma_random_strength,
# "keep_tokens": keep_tokens,
# "learning_rate": learning_rate,
# "logging_dir": logging_dir,
# "log_tracker_name": log_tracker_name,
# "log_tracker_config": log_tracker_config,
# "lr_scheduler": lr_scheduler,
# "lr_scheduler_args": lr_scheduler_args,
# "lr_warmup_steps": lr_warmup_steps,
# "max_bucket_reso": max_bucket_reso,
"max_data_loader_n_workers": max_data_loader_n_workers,
# "max_resolution": max_resolution,
# "max_timestep": max_timestep,
# "max_token_length": max_token_length,
# "max_train_epochs": max_train_epochs,
# "max_train_steps": max_train_steps,
# "mem_eff_attn": mem_eff_attn,
# "min_bucket_reso": min_bucket_reso,
# "min_snr_gamma": min_snr_gamma,
# "min_timestep": min_timestep,
# "mixed_precision": mixed_precision,
# "multires_noise_discount": multires_noise_discount,
# "multires_noise_iterations": multires_noise_iterations,
# "noise_offset": noise_offset,
# "noise_offset_random_strength": noise_offset_random_strength,
# "noise_offset_type": noise_offset_type,
# "optimizer": optimizer,
# "optimizer_args": optimizer_args,
# "output_dir": output_dir,
# "output_name": output_name,
# "persistent_data_loader_workers": persistent_data_loader_workers,
# "pretrained_model_name_or_path": pretrained_model_name_or_path,
# "random_crop": random_crop,
# "resume": resume,
# "save_every_n_epochs": save_every_n_epochs,
# "save_every_n_steps": save_every_n_steps,
# "save_last_n_steps": save_last_n_steps,
# "save_last_n_steps_state": save_last_n_steps_state,
# "save_model_as": save_model_as,
# "save_precision": save_precision,
# "save_state": save_state,
# "save_state_on_train_end": save_state_on_train_end,
# "scale_v_pred_loss_like_noise_pred": scale_v_pred_loss_like_noise_pred,
# "seed": seed,
# "shuffle_caption": shuffle_caption,
# "train_batch_size": train_batch_size,
# "train_data_dir": image_folder,
# "train_text_encoder": train_text_encoder,
# "use_wandb": use_wandb,
# "v2": v2,
# "v_parameterization": v_parameterization,
# "v_pred_like_loss": v_pred_like_loss,
# "vae_batch_size": vae_batch_size,
# "wandb_api_key": wandb_api_key,
# "wandb_run_name": wandb_run_name,
# "weighted_captions": weighted_captions,
# "xformers": xformers,
"additional_parameters": additional_parameters,
# "loss_type": loss_type,
# "huber_schedule": huber_schedule,
# "huber_c": huber_c,
}
# Conditionally include specific keyword arguments based on sdxl_checkbox
# if sdxl_checkbox:
# kwargs_for_training["cache_text_encoder_outputs"] = cache_text_encoder_outputs
# kwargs_for_training["learning_rate_te1"] = learning_rate_te1
# kwargs_for_training["learning_rate_te2"] = learning_rate_te2
# kwargs_for_training["no_half_vae"] = no_half_vae
# else:
# kwargs_for_training["learning_rate_te"] = learning_rate_te
# Pass the dynamically constructed keyword arguments to the function
run_cmd = run_cmd_advanced_training(run_cmd=run_cmd, **kwargs_for_training)
# run_cmd = run_cmd_sample(
# run_cmd,
# sample_every_n_steps,
# sample_every_n_epochs,
# sample_sampler,
# sample_prompts,
# output_dir,
# )
if max_data_loader_n_workers == "" or None:
max_data_loader_n_workers = 0
else:
max_data_loader_n_workers = int(max_data_loader_n_workers)
if max_train_steps == "" or None:
max_train_steps = 0
else:
max_train_steps = int(max_train_steps)
config_toml_data = {
# Update the values in the TOML data
"huggingface_repo_id": huggingface_repo_id,
"huggingface_token": huggingface_token,
"huggingface_repo_type": huggingface_repo_type,
"huggingface_repo_visibility": huggingface_repo_visibility,
"huggingface_path_in_repo": huggingface_path_in_repo,
"save_state_to_huggingface": save_state_to_huggingface,
"resume_from_huggingface": resume_from_huggingface,
"async_upload": async_upload,
"adaptive_noise_scale": adaptive_noise_scale,
"block_lr": block_lr,
"bucket_no_upscale": bucket_no_upscale,
"bucket_reso_steps": bucket_reso_steps,
"cache_latents": cache_latents,
"cache_latents_to_disk": cache_latents_to_disk,
"cache_text_encoder_outputs": cache_text_encoder_outputs,
"caption_dropout_every_n_epochs": caption_dropout_every_n_epochs,
"caption_dropout_rate": caption_dropout_rate,
"caption_extension": caption_extension,
"clip_skip": clip_skip,
"clip_skip": int(clip_skip),
"color_aug": color_aug,
"dataset_config": dataset_config,
"dataset_repeats": dataset_repeats,
"dataset_repeats": int(dataset_repeats),
"enable_bucket": True,
"flip_aug": flip_aug,
"masked_loss": masked_loss,
@ -674,39 +829,47 @@ def train_model(
"in_json": in_json,
"ip_noise_gamma": ip_noise_gamma,
"ip_noise_gamma_random_strength": ip_noise_gamma_random_strength,
"keep_tokens": keep_tokens,
"keep_tokens": int(keep_tokens),
"learning_rate": learning_rate,
"learning_rate_te": learning_rate_te,
"learning_rate_te1": learning_rate_te1,
"learning_rate_te2": learning_rate_te2,
"logging_dir": logging_dir,
"log_tracker_name": log_tracker_name,
"log_tracker_config": log_tracker_config,
"lr_scheduler": lr_scheduler,
"lr_scheduler_args": lr_scheduler_args,
"lr_scheduler_args": str(lr_scheduler_args).replace('"', '').split(),
"lr_warmup_steps": lr_warmup_steps,
"max_bucket_reso": max_bucket_reso,
"max_bucket_reso": int(max_bucket_reso),
"max_data_loader_n_workers": max_data_loader_n_workers,
"max_resolution": max_resolution,
"max_timestep": max_timestep,
"max_token_length": max_token_length,
"max_token_length": int(max_token_length),
"max_train_epochs": max_train_epochs,
"max_train_steps": max_train_steps,
"max_train_steps": int(max_train_steps),
"mem_eff_attn": mem_eff_attn,
"min_bucket_reso": min_bucket_reso,
"min_bucket_reso": int(min_bucket_reso),
"min_snr_gamma": min_snr_gamma,
"min_timestep": min_timestep,
"min_timestep": int(min_timestep),
"mixed_precision": mixed_precision,
"multires_noise_discount": multires_noise_discount,
"multires_noise_iterations": multires_noise_iterations,
"no_half_vae": no_half_vae,
"noise_offset": noise_offset,
"noise_offset_random_strength": noise_offset_random_strength,
"noise_offset_type": noise_offset_type,
"optimizer": optimizer,
"optimizer_args": optimizer_args,
"optimizer_type": optimizer,
"optimizer_args": str(optimizer_args).replace('"', '').split(),
"output_dir": output_dir,
"output_name": output_name,
"persistent_data_loader_workers": persistent_data_loader_workers,
"pretrained_model_name_or_path": pretrained_model_name_or_path,
"random_crop": random_crop,
"resolution": max_resolution,
"resume": resume,
"sample_every_n_epochs": sample_every_n_epochs,
"sample_every_n_steps": sample_every_n_steps,
"sample_prompts": create_prompt_file(output_dir, output_dir),
"sample_sampler": sample_sampler,
"save_every_n_epochs": save_every_n_epochs,
"save_every_n_steps": save_every_n_steps,
"save_last_n_steps": save_last_n_steps,
@ -716,7 +879,7 @@ def train_model(
"save_state": save_state,
"save_state_on_train_end": save_state_on_train_end,
"scale_v_pred_loss_like_noise_pred": scale_v_pred_loss_like_noise_pred,
"seed": seed,
"seed": int(seed),
"shuffle_caption": shuffle_caption,
"train_batch_size": train_batch_size,
"train_data_dir": image_folder,
@ -736,26 +899,30 @@ def train_model(
"huber_c": huber_c,
}
# Conditionally include specific keyword arguments based on sdxl_checkbox
if sdxl_checkbox:
kwargs_for_training["cache_text_encoder_outputs"] = cache_text_encoder_outputs
kwargs_for_training["learning_rate_te1"] = learning_rate_te1
kwargs_for_training["learning_rate_te2"] = learning_rate_te2
kwargs_for_training["no_half_vae"] = no_half_vae
else:
kwargs_for_training["learning_rate_te"] = learning_rate_te
# Given dictionary `config_toml_data`
# Remove all values = ""
config_toml_data = {
key: value
for key, value in config_toml_data.items()
if value != "" and value != False
}
# Pass the dynamically constructed keyword arguments to the function
run_cmd = run_cmd_advanced_training(run_cmd=run_cmd, **kwargs_for_training)
tmpfilename = "./outputs/tmpfilefinetune.toml"
# Save the updated TOML data back to the file
with open(tmpfilename, "w") as toml_file:
toml.dump(config_toml_data, toml_file)
run_cmd = run_cmd_sample(
run_cmd,
sample_every_n_steps,
sample_every_n_epochs,
sample_sampler,
sample_prompts,
output_dir,
)
if not os.path.exists(toml_file.name):
log.error(f"Failed to write TOML file: {toml_file.name}")
toml_file_path = (
os.path.abspath(os.path.normpath(toml_file.name)).replace("\\", "/")
if os.name == "nt"
else toml_file.name
)
run_cmd.append(f"--config_file")
run_cmd.append(rf"{toml_file_path}")
if print_only:
log.warning(
@ -937,6 +1104,10 @@ def finetune_tab(headless=False, config: dict = {}):
with gr.Accordion("Samples", open=False, elem_id="samples_tab"):
sample = SampleImages(config=config)
global huggingface
with gr.Accordion("HuggingFace", open=False):
huggingface = HuggingFace(config=config)
with gr.Column(), gr.Group():
with gr.Row():
button_run = gr.Button("Start training", variant="primary")
@ -1062,6 +1233,14 @@ def finetune_tab(headless=False, config: dict = {}):
advanced_training.min_timestep,
advanced_training.max_timestep,
accelerate_launch.extra_accelerate_launch_args,
huggingface.huggingface_repo_id,
huggingface.huggingface_token,
huggingface.huggingface_repo_type,
huggingface.huggingface_repo_visibility,
huggingface.huggingface_path_in_repo,
huggingface.save_state_to_huggingface,
huggingface.resume_from_huggingface,
huggingface.async_upload,
]
configuration.button_open_config.click(

271
kohya_gui/lora_gui.py
View File

@ -3,10 +3,10 @@ import json
import math
import os
import time
import toml
from datetime import datetime
from .common_gui import (
get_executable_path,
get_file_path,
get_any_file_path,
get_saveasfile_path,
@ -29,8 +29,9 @@ from .class_sdxl_parameters import SDXLParameters
from .class_folders import Folders
from .class_command_executor import CommandExecutor
from .class_tensorboard import TensorboardManager
from .class_sample_images import SampleImages, run_cmd_sample
from .class_sample_images import SampleImages, run_cmd_sample, create_prompt_file
from .class_lora_tab import LoRATools
from .class_huggingface import HuggingFace
from .dreambooth_folder_creation_gui import (
gradio_dreambooth_folder_creation_tab,
@ -45,6 +46,9 @@ log = setup_logging()
# Setup command executor
executor = CommandExecutor()
# Setup huggingface
huggingface = None
button_run = gr.Button("Start training", variant="primary")
button_stop_training = gr.Button("Stop training", visible=False)
@ -79,7 +83,7 @@ def update_network_args_with_kohya_lora_vars(
# Iterate over the Kohya LoRA variables and append them to the network arguments
for key, value in kohya_lora_vars.items():
# Append each variable as a key-value pair to the network_args
network_args += f' {key}="{value}"'
network_args += f' {key}={value}'
return network_args
@ -228,6 +232,14 @@ def save_configuration(
LyCORIS_preset,
debiased_estimation_loss,
extra_accelerate_launch_args,
huggingface_repo_id,
huggingface_token,
huggingface_repo_type,
huggingface_repo_visibility,
huggingface_path_in_repo,
save_state_to_huggingface,
resume_from_huggingface,
async_upload,
):
# Get list of function parameters and values
parameters = list(locals().items())
@ -416,6 +428,14 @@ def open_configuration(
LyCORIS_preset,
debiased_estimation_loss,
extra_accelerate_launch_args,
huggingface_repo_id,
huggingface_token,
huggingface_repo_type,
huggingface_repo_visibility,
huggingface_path_in_repo,
save_state_to_huggingface,
resume_from_huggingface,
async_upload,
training_preset,
):
# Get list of function parameters and values
@ -634,6 +654,14 @@ def train_model(
LyCORIS_preset,
debiased_estimation_loss,
extra_accelerate_launch_args,
huggingface_repo_id,
huggingface_token,
huggingface_repo_type,
huggingface_repo_visibility,
huggingface_path_in_repo,
save_state_to_huggingface,
resume_from_huggingface,
async_upload,
):
# Get list of function parameters and values
parameters = list(locals().items())
@ -816,39 +844,39 @@ def train_model(
if LoRA_type == "LyCORIS/BOFT":
network_module = "lycoris.kohya"
network_args = f' preset="{LyCORIS_preset}" conv_dim="{conv_dim}" conv_alpha="{conv_alpha}" module_dropout="{module_dropout}" use_tucker="{use_tucker}" use_scalar="{use_scalar}" rank_dropout="{rank_dropout}" rank_dropout_scale="{rank_dropout_scale}" constrain="{constrain}" rescaled="{rescaled}" algo="boft" train_norm="{train_norm}"'
network_args = f' preset={LyCORIS_preset} conv_dim={conv_dim} conv_alpha={conv_alpha} module_dropout={module_dropout} use_tucker={use_tucker} use_scalar={use_scalar} rank_dropout={rank_dropout} rank_dropout_scale={rank_dropout_scale} constrain={constrain} rescaled={rescaled} algo=boft train_norm={train_norm}'
if LoRA_type == "LyCORIS/Diag-OFT":
network_module = "lycoris.kohya"
network_args = f' preset="{LyCORIS_preset}" conv_dim="{conv_dim}" conv_alpha="{conv_alpha}" module_dropout="{module_dropout}" use_tucker="{use_tucker}" use_scalar="{use_scalar}" rank_dropout="{rank_dropout}" rank_dropout_scale="{rank_dropout_scale}" constrain="{constrain}" rescaled="{rescaled}" algo="diag-oft" train_norm="{train_norm}"'
network_args = f' preset={LyCORIS_preset} conv_dim={conv_dim} conv_alpha={conv_alpha} module_dropout={module_dropout} use_tucker={use_tucker} use_scalar={use_scalar} rank_dropout={rank_dropout} rank_dropout_scale={rank_dropout_scale} constrain={constrain} rescaled={rescaled} algo=diag-oft train_norm={train_norm}'
if LoRA_type == "LyCORIS/DyLoRA":
network_module = "lycoris.kohya"
network_args = f' preset="{LyCORIS_preset}" conv_dim="{conv_dim}" conv_alpha="{conv_alpha}" use_tucker="{use_tucker}" block_size="{unit}" rank_dropout="{rank_dropout}" module_dropout="{module_dropout}" algo="dylora" train_norm="{train_norm}"'
network_args = f' preset={LyCORIS_preset} conv_dim={conv_dim} conv_alpha={conv_alpha} use_tucker={use_tucker} block_size={unit} rank_dropout={rank_dropout} module_dropout={module_dropout} algo="dylora" train_norm={train_norm}'
if LoRA_type == "LyCORIS/GLoRA":
network_module = "lycoris.kohya"
network_args = f' preset="{LyCORIS_preset}" conv_dim="{conv_dim}" conv_alpha="{conv_alpha}" rank_dropout="{rank_dropout}" module_dropout="{module_dropout}" rank_dropout_scale="{rank_dropout_scale}" algo="glora" train_norm="{train_norm}"'
network_args = f' preset={LyCORIS_preset} conv_dim={conv_dim} conv_alpha={conv_alpha} rank_dropout={rank_dropout} module_dropout={module_dropout} rank_dropout_scale={rank_dropout_scale} algo="glora" train_norm={train_norm}'
if LoRA_type == "LyCORIS/iA3":
network_module = "lycoris.kohya"
network_args = f' preset="{LyCORIS_preset}" conv_dim="{conv_dim}" conv_alpha="{conv_alpha}" train_on_input="{train_on_input}" algo="ia3"'
network_args = f' preset={LyCORIS_preset} conv_dim={conv_dim} conv_alpha={conv_alpha} train_on_input={train_on_input} algo=ia3'
if LoRA_type == "LoCon" or LoRA_type == "LyCORIS/LoCon":
network_module = "lycoris.kohya"
network_args = f' preset="{LyCORIS_preset}" conv_dim="{conv_dim}" conv_alpha="{conv_alpha}" rank_dropout="{rank_dropout}" bypass_mode="{bypass_mode}" dora_wd="{dora_wd}" module_dropout="{module_dropout}" use_tucker="{use_tucker}" use_scalar="{use_scalar}" rank_dropout_scale="{rank_dropout_scale}" algo="locon" train_norm="{train_norm}"'
network_args = f' preset={LyCORIS_preset} conv_dim={conv_dim} conv_alpha={conv_alpha} rank_dropout={rank_dropout} bypass_mode={bypass_mode} dora_wd={dora_wd} module_dropout={module_dropout} use_tucker={use_tucker} use_scalar={use_scalar} rank_dropout_scale={rank_dropout_scale} algo=locon train_norm={train_norm}'
if LoRA_type == "LyCORIS/LoHa":
network_module = "lycoris.kohya"
network_args = f' preset="{LyCORIS_preset}" conv_dim="{conv_dim}" conv_alpha="{conv_alpha}" rank_dropout="{rank_dropout}" bypass_mode="{bypass_mode}" dora_wd="{dora_wd}" module_dropout="{module_dropout}" use_tucker="{use_tucker}" use_scalar="{use_scalar}" rank_dropout_scale="{rank_dropout_scale}" algo="loha" train_norm="{train_norm}"'
network_args = f' preset={LyCORIS_preset} conv_dim={conv_dim} conv_alpha={conv_alpha} rank_dropout={rank_dropout} bypass_mode={bypass_mode} dora_wd={dora_wd} module_dropout={module_dropout} use_tucker={use_tucker} use_scalar={use_scalar} rank_dropout_scale={rank_dropout_scale} algo="loha" train_norm={train_norm}'
if LoRA_type == "LyCORIS/LoKr":
network_module = "lycoris.kohya"
network_args = f' preset="{LyCORIS_preset}" conv_dim="{conv_dim}" conv_alpha="{conv_alpha}" rank_dropout="{rank_dropout}" bypass_mode="{bypass_mode}" dora_wd="{dora_wd}" module_dropout="{module_dropout}" factor="{factor}" use_cp="{use_cp}" use_scalar="{use_scalar}" decompose_both="{decompose_both}" rank_dropout_scale="{rank_dropout_scale}" algo="lokr" train_norm="{train_norm}"'
network_args = f' preset={LyCORIS_preset} conv_dim={conv_dim} conv_alpha={conv_alpha} rank_dropout={rank_dropout} bypass_mode={bypass_mode} dora_wd={dora_wd} module_dropout={module_dropout} factor={factor} use_cp={use_cp} use_scalar={use_scalar} decompose_both={decompose_both} rank_dropout_scale={rank_dropout_scale} algo=lokr train_norm={train_norm}'
if LoRA_type == "LyCORIS/Native Fine-Tuning":
network_module = "lycoris.kohya"
network_args = f' preset="{LyCORIS_preset}" rank_dropout="{rank_dropout}" module_dropout="{module_dropout}" use_tucker="{use_tucker}" use_scalar="{use_scalar}" rank_dropout_scale="{rank_dropout_scale}" algo="full" train_norm="{train_norm}"'
network_args = f' preset={LyCORIS_preset} rank_dropout={rank_dropout} module_dropout={module_dropout} use_tucker={use_tucker} use_scalar={use_scalar} rank_dropout_scale={rank_dropout_scale} algo=full train_norm={train_norm}'
if LoRA_type in ["Kohya LoCon", "Standard"]:
kohya_lora_var_list = [
@ -931,6 +959,150 @@ def train_model(
# Define a dictionary of parameters
run_cmd_params = {
# "adaptive_noise_scale": adaptive_noise_scale,
# "bucket_no_upscale": bucket_no_upscale,
# "bucket_reso_steps": bucket_reso_steps,
# "cache_latents": cache_latents,
# "cache_latents_to_disk": cache_latents_to_disk,
# "cache_text_encoder_outputs": (
# True if sdxl and sdxl_cache_text_encoder_outputs else None
# ),
# "caption_dropout_every_n_epochs": caption_dropout_every_n_epochs,
# "caption_dropout_rate": caption_dropout_rate,
# "caption_extension": caption_extension,
# "clip_skip": clip_skip,
# "color_aug": color_aug,
# "dataset_config": dataset_config,
# "debiased_estimation_loss": debiased_estimation_loss,
# "dim_from_weights": dim_from_weights,
# "enable_bucket": enable_bucket,
# "epoch": epoch,
# "flip_aug": flip_aug,
# "masked_loss": masked_loss,
# "fp8_base": fp8_base,
# "full_bf16": full_bf16,
# "full_fp16": full_fp16,
# "gradient_accumulation_steps": gradient_accumulation_steps,
# "gradient_checkpointing": gradient_checkpointing,
# "ip_noise_gamma": ip_noise_gamma,
# "ip_noise_gamma_random_strength": ip_noise_gamma_random_strength,
# "keep_tokens": keep_tokens,
# "learning_rate": learning_rate,
# "logging_dir": logging_dir,
# "log_tracker_name": log_tracker_name,
# "log_tracker_config": log_tracker_config,
# "lora_network_weights": lora_network_weights,
# "lr_scheduler": lr_scheduler,
# "lr_scheduler_args": lr_scheduler_args,
# "lr_scheduler_num_cycles": lr_scheduler_num_cycles,
# "lr_scheduler_power": lr_scheduler_power,
# "lr_warmup_steps": lr_warmup_steps,
# "max_bucket_reso": max_bucket_reso,
"max_data_loader_n_workers": max_data_loader_n_workers,
# "max_grad_norm": max_grad_norm,
# "max_resolution": max_resolution,
# "max_timestep": max_timestep,
# "max_token_length": max_token_length,
# "max_train_epochs": max_train_epochs,
# "max_train_steps": max_train_steps,
# "mem_eff_attn": mem_eff_attn,
# "min_bucket_reso": min_bucket_reso,
# "min_snr_gamma": min_snr_gamma,
# "min_timestep": min_timestep,
# "mixed_precision": mixed_precision,
# "multires_noise_discount": multires_noise_discount,
# "multires_noise_iterations": multires_noise_iterations,
# "network_alpha": network_alpha,
# "network_args": network_args,
# "network_dim": network_dim,
# "network_dropout": network_dropout,
# "network_module": network_module,
# "network_train_unet_only": network_train_unet_only,
# "network_train_text_encoder_only": network_train_text_encoder_only,
# "no_half_vae": True if sdxl and sdxl_no_half_vae else None,
# "noise_offset": noise_offset,
# "noise_offset_random_strength": noise_offset_random_strength,
# "noise_offset_type": noise_offset_type,
# "optimizer": optimizer,
# "optimizer_args": optimizer_args,
# "output_dir": output_dir,
# "output_name": output_name,
# "persistent_data_loader_workers": persistent_data_loader_workers,
# "pretrained_model_name_or_path": pretrained_model_name_or_path,
# "prior_loss_weight": prior_loss_weight,
# "random_crop": random_crop,
# "reg_data_dir": reg_data_dir,
# "resume": resume,
# "save_every_n_epochs": save_every_n_epochs,
# "save_every_n_steps": save_every_n_steps,
# "save_last_n_steps": save_last_n_steps,
# "save_last_n_steps_state": save_last_n_steps_state,
# "save_model_as": save_model_as,
# "save_precision": save_precision,
# "save_state": save_state,
# "save_state_on_train_end": save_state_on_train_end,
# "scale_v_pred_loss_like_noise_pred": scale_v_pred_loss_like_noise_pred,
# "scale_weight_norms": scale_weight_norms,
# "seed": seed,
# "shuffle_caption": shuffle_caption,
# "stop_text_encoder_training": stop_text_encoder_training,
# "text_encoder_lr": text_encoder_lr,
# "train_batch_size": train_batch_size,
# "train_data_dir": train_data_dir,
# "training_comment": training_comment,
# "unet_lr": unet_lr,
# "use_wandb": use_wandb,
# "v2": v2,
# "v_parameterization": v_parameterization,
# "v_pred_like_loss": v_pred_like_loss,
# "vae": vae,
# "vae_batch_size": vae_batch_size,
# "wandb_api_key": wandb_api_key,
# "wandb_run_name": wandb_run_name,
# "weighted_captions": weighted_captions,
# "xformers": xformers,
"additional_parameters": additional_parameters,
# "loss_type": loss_type,
# "huber_schedule": huber_schedule,
# "huber_c": huber_c,
}
# Use the ** syntax to unpack the dictionary when calling the function
run_cmd = run_cmd_advanced_training(run_cmd=run_cmd, **run_cmd_params)
# run_cmd = run_cmd_sample(
# run_cmd,
# sample_every_n_steps,
# sample_every_n_epochs,
# sample_sampler,
# sample_prompts,
# output_dir,
# )
if max_data_loader_n_workers == "" or None:
max_data_loader_n_workers = 0
else:
max_data_loader_n_workers = int(max_data_loader_n_workers)
if max_train_steps == "" or None:
max_train_steps = 0
else:
max_train_steps = int(max_train_steps)
if seed == "":
seed = 0
else:
seed = int(seed)
config_toml_data = {
"huggingface_repo_id": huggingface_repo_id,
"huggingface_token": huggingface_token,
"huggingface_repo_type": huggingface_repo_type,
"huggingface_repo_visibility": huggingface_repo_visibility,
"huggingface_path_in_repo": huggingface_path_in_repo,
"save_state_to_huggingface": save_state_to_huggingface,
"resume_from_huggingface": resume_from_huggingface,
"async_upload": async_upload,
"adaptive_noise_scale": adaptive_noise_scale,
"bucket_no_upscale": bucket_no_upscale,
"bucket_reso_steps": bucket_reso_steps,
@ -942,13 +1114,13 @@ def train_model(
"caption_dropout_every_n_epochs": caption_dropout_every_n_epochs,
"caption_dropout_rate": caption_dropout_rate,
"caption_extension": caption_extension,
"clip_skip": clip_skip,
"clip_skip": int(clip_skip),
"color_aug": color_aug,
"dataset_config": dataset_config,
"debiased_estimation_loss": debiased_estimation_loss,
"dim_from_weights": dim_from_weights,
"enable_bucket": enable_bucket,
"epoch": epoch,
"epoch": int(epoch),
"flip_aug": flip_aug,
"masked_loss": masked_loss,
"fp8_base": fp8_base,
@ -958,34 +1130,33 @@ def train_model(
"gradient_checkpointing": gradient_checkpointing,
"ip_noise_gamma": ip_noise_gamma,
"ip_noise_gamma_random_strength": ip_noise_gamma_random_strength,
"keep_tokens": keep_tokens,
"keep_tokens": int(keep_tokens),
"learning_rate": learning_rate,
"logging_dir": logging_dir,
"log_tracker_name": log_tracker_name,
"log_tracker_config": log_tracker_config,
"lora_network_weights": lora_network_weights,
"lr_scheduler": lr_scheduler,
"lr_scheduler_args": lr_scheduler_args,
"lr_scheduler_args": str(lr_scheduler_args).replace('"', '').split(),
"lr_scheduler_num_cycles": lr_scheduler_num_cycles,
"lr_scheduler_power": lr_scheduler_power,
"lr_warmup_steps": lr_warmup_steps,
"max_bucket_reso": max_bucket_reso,
"max_data_loader_n_workers": max_data_loader_n_workers,
"max_grad_norm": max_grad_norm,
"max_resolution": max_resolution,
"max_timestep": max_timestep,
"max_token_length": max_token_length,
"max_token_length": int(max_token_length),
"max_train_epochs": max_train_epochs,
"max_train_steps": max_train_steps,
"max_train_steps": int(max_train_steps),
"mem_eff_attn": mem_eff_attn,
"min_bucket_reso": min_bucket_reso,
"min_bucket_reso": int(min_bucket_reso),
"min_snr_gamma": min_snr_gamma,
"min_timestep": min_timestep,
"min_timestep": int(min_timestep),
"mixed_precision": mixed_precision,
"multires_noise_discount": multires_noise_discount,
"multires_noise_iterations": multires_noise_iterations,
"network_alpha": network_alpha,
"network_args": network_args,
"network_args": str(network_args).replace('"', '').split(),
"network_dim": network_dim,
"network_dropout": network_dropout,
"network_module": network_module,
@ -995,8 +1166,8 @@ def train_model(
"noise_offset": noise_offset,
"noise_offset_random_strength": noise_offset_random_strength,
"noise_offset_type": noise_offset_type,
"optimizer": optimizer,
"optimizer_args": optimizer_args,
"optimizer_type": optimizer,
"optimizer_args": str(optimizer_args).replace('"', '').split(),
"output_dir": output_dir,
"output_name": output_name,
"persistent_data_loader_workers": persistent_data_loader_workers,
@ -1004,7 +1175,12 @@ def train_model(
"prior_loss_weight": prior_loss_weight,
"random_crop": random_crop,
"reg_data_dir": reg_data_dir,
"resolution": max_resolution,
"resume": resume,
"sample_every_n_epochs": sample_every_n_epochs,
"sample_every_n_steps": sample_every_n_steps,
"sample_prompts": create_prompt_file(output_dir, output_dir),
"sample_sampler": sample_sampler,
"save_every_n_epochs": save_every_n_epochs,
"save_every_n_steps": save_every_n_steps,
"save_last_n_steps": save_last_n_steps,
@ -1015,7 +1191,7 @@ def train_model(
"save_state_on_train_end": save_state_on_train_end,
"scale_v_pred_loss_like_noise_pred": scale_v_pred_loss_like_noise_pred,
"scale_weight_norms": scale_weight_norms,
"seed": seed,
"seed": int(seed),
"shuffle_caption": shuffle_caption,
"stop_text_encoder_training": stop_text_encoder_training,
"text_encoder_lr": text_encoder_lr,
@ -1039,17 +1215,30 @@ def train_model(
"huber_c": huber_c,
}
# Use the ** syntax to unpack the dictionary when calling the function
run_cmd = run_cmd_advanced_training(run_cmd=run_cmd, **run_cmd_params)
# Given dictionary `config_toml_data`
# Remove all values = ""
config_toml_data = {
key: value
for key, value in config_toml_data.items()
if value != "" and value != False
}
run_cmd = run_cmd_sample(
run_cmd,
sample_every_n_steps,
sample_every_n_epochs,
sample_sampler,
sample_prompts,
output_dir,
)
tmpfilename = "./outputs/tmpfilelora.toml"
# Save the updated TOML data back to the file
with open(tmpfilename, "w") as toml_file:
toml.dump(config_toml_data, toml_file)
if not os.path.exists(toml_file.name):
log.error(f"Failed to write TOML file: {toml_file.name}")
toml_file_path = (
os.path.abspath(os.path.normpath(toml_file.name)).replace("\\", "/")
if os.name == "nt"
else toml_file.name
)
run_cmd.append(f"--config_file")
run_cmd.append(rf"{toml_file_path}")
if print_only:
log.warning(
@ -1898,6 +2087,10 @@ def lora_tab(
with gr.Accordion("Samples", open=False, elem_id="samples_tab"):
sample = SampleImages(config=config)
global huggingface
with gr.Accordion("HuggingFace", open=False):
huggingface = HuggingFace(config=config)
LoRA_type.change(
update_LoRA_settings,
inputs=[
@ -2094,6 +2287,14 @@ def lora_tab(
LyCORIS_preset,
advanced_training.debiased_estimation_loss,
accelerate_launch.extra_accelerate_launch_args,
huggingface.huggingface_repo_id,
huggingface.huggingface_token,
huggingface.huggingface_repo_type,
huggingface.huggingface_repo_visibility,
huggingface.huggingface_path_in_repo,
huggingface.save_state_to_huggingface,
huggingface.resume_from_huggingface,
huggingface.async_upload,
]
configuration.button_open_config.click(

316
kohya_gui/textual_inversion_gui.py
View File

@ -2,7 +2,7 @@ import gradio as gr
import json
import math
import os
import shlex
import toml
import time
from datetime import datetime
from .common_gui import (
@ -29,12 +29,13 @@ from .class_advanced_training import AdvancedTraining
from .class_folders import Folders
from .class_sdxl_parameters import SDXLParameters
from .class_command_executor import CommandExecutor
from .class_huggingface import HuggingFace
from .class_tensorboard import TensorboardManager
from .dreambooth_folder_creation_gui import (
gradio_dreambooth_folder_creation_tab,
)
from .dataset_balancing_gui import gradio_dataset_balancing_tab
from .class_sample_images import SampleImages, run_cmd_sample
from .class_sample_images import SampleImages, run_cmd_sample, create_prompt_file
from .custom_logging import setup_logging
@ -44,6 +45,9 @@ log = setup_logging()
# Setup command executor
executor = CommandExecutor()
# Setup huggingface
huggingface = None
TRAIN_BUTTON_VISIBLE = [gr.Button(visible=True), gr.Button(visible=False)]
@ -154,6 +158,14 @@ def save_configuration(
max_timestep,
sdxl_no_half_vae,
extra_accelerate_launch_args,
huggingface_repo_id,
huggingface_token,
huggingface_repo_type,
huggingface_repo_visibility,
huggingface_path_in_repo,
save_state_to_huggingface,
resume_from_huggingface,
async_upload,
):
# Get list of function parameters and values
parameters = list(locals().items())
@ -296,6 +308,14 @@ def open_configuration(
max_timestep,
sdxl_no_half_vae,
extra_accelerate_launch_args,
huggingface_repo_id,
huggingface_token,
huggingface_repo_type,
huggingface_repo_visibility,
huggingface_path_in_repo,
save_state_to_huggingface,
resume_from_huggingface,
async_upload,
):
# Get list of function parameters and values
parameters = list(locals().items())
@ -431,6 +451,14 @@ def train_model(
max_timestep,
sdxl_no_half_vae,
extra_accelerate_launch_args,
huggingface_repo_id,
huggingface_token,
huggingface_repo_type,
huggingface_repo_visibility,
huggingface_path_in_repo,
save_state_to_huggingface,
resume_from_huggingface,
async_upload,
):
# Get list of function parameters and values
parameters = list(locals().items())
@ -565,111 +593,175 @@ def train_model(
else:
run_cmd.append(f"{scriptdir}/sd-scripts/train_textual_inversion.py")
run_cmd = run_cmd_advanced_training(
run_cmd=run_cmd,
adaptive_noise_scale=adaptive_noise_scale,
bucket_no_upscale=bucket_no_upscale,
bucket_reso_steps=bucket_reso_steps,
cache_latents=cache_latents,
cache_latents_to_disk=cache_latents_to_disk,
caption_dropout_every_n_epochs=caption_dropout_every_n_epochs,
caption_extension=caption_extension,
clip_skip=clip_skip,
color_aug=color_aug,
dataset_config=dataset_config,
enable_bucket=enable_bucket,
epoch=epoch,
flip_aug=flip_aug,
full_fp16=full_fp16,
gradient_accumulation_steps=gradient_accumulation_steps,
gradient_checkpointing=gradient_checkpointing,
ip_noise_gamma=ip_noise_gamma,
ip_noise_gamma_random_strength=ip_noise_gamma_random_strength,
keep_tokens=keep_tokens,
learning_rate=learning_rate,
logging_dir=logging_dir,
log_tracker_name=log_tracker_name,
log_tracker_config=log_tracker_config,
lr_scheduler=lr_scheduler,
lr_scheduler_args=lr_scheduler_args,
lr_scheduler_num_cycles=lr_scheduler_num_cycles,
lr_scheduler_power=lr_scheduler_power,
lr_warmup_steps=lr_warmup_steps,
max_bucket_reso=max_bucket_reso,
max_data_loader_n_workers=max_data_loader_n_workers,
max_resolution=max_resolution,
max_timestep=max_timestep,
max_token_length=max_token_length,
max_train_epochs=max_train_epochs,
max_train_steps=max_train_steps,
mem_eff_attn=mem_eff_attn,
min_bucket_reso=min_bucket_reso,
min_snr_gamma=min_snr_gamma,
min_timestep=min_timestep,
mixed_precision=mixed_precision,
multires_noise_discount=multires_noise_discount,
multires_noise_iterations=multires_noise_iterations,
no_half_vae=True if sdxl and sdxl_no_half_vae else None,
no_token_padding=no_token_padding,
noise_offset=noise_offset,
noise_offset_random_strength=noise_offset_random_strength,
noise_offset_type=noise_offset_type,
optimizer=optimizer,
optimizer_args=optimizer_args,
output_dir=output_dir,
output_name=output_name,
persistent_data_loader_workers=persistent_data_loader_workers,
pretrained_model_name_or_path=pretrained_model_name_or_path,
prior_loss_weight=prior_loss_weight,
random_crop=random_crop,
reg_data_dir=reg_data_dir,
resume=resume,
save_every_n_epochs=save_every_n_epochs,
save_every_n_steps=save_every_n_steps,
save_last_n_steps=save_last_n_steps,
save_last_n_steps_state=save_last_n_steps_state,
save_model_as=save_model_as,
save_precision=save_precision,
save_state=save_state,
save_state_on_train_end=save_state_on_train_end,
scale_v_pred_loss_like_noise_pred=scale_v_pred_loss_like_noise_pred,
seed=seed,
shuffle_caption=shuffle_caption,
stop_text_encoder_training=stop_text_encoder_training,
train_batch_size=train_batch_size,
train_data_dir=train_data_dir,
use_wandb=use_wandb,
v2=v2,
v_parameterization=v_parameterization,
v_pred_like_loss=v_pred_like_loss,
vae=vae,
vae_batch_size=vae_batch_size,
wandb_api_key=wandb_api_key,
wandb_run_name=wandb_run_name,
xformers=xformers,
additional_parameters=additional_parameters,
loss_type=loss_type,
huber_schedule=huber_schedule,
huber_c=huber_c,
)
run_cmd.append(f'--token_string="{shlex.quote(token_string)}"')
run_cmd.append(f'--init_word="{shlex.quote(init_word)}"')
run_cmd.append(f"--num_vectors_per_token={int(num_vectors_per_token)}")
if not weights == "":
run_cmd.append(f'--weights="{shlex.quote(weights)}"')
if template == "object template":
run_cmd.append("--use_object_template")
elif template == "style template":
run_cmd.append("--use_style_template")
# Initialize a dictionary with always-included keyword arguments
kwargs_for_training = {
"max_data_loader_n_workers": max_data_loader_n_workers,
"additional_parameters": additional_parameters,
}
# run_cmd.append(f'--token_string="{shlex.quote(token_string)}"')
# run_cmd.append(f'--init_word="{shlex.quote(init_word)}"')
# run_cmd.append(f"--num_vectors_per_token={int(num_vectors_per_token)}")
# if not weights == "":
# run_cmd.append(f'--weights="{shlex.quote(weights)}"')
# if template == "object template":
# run_cmd.append("--use_object_template")
# elif template == "style template":
# run_cmd.append("--use_style_template")
run_cmd = run_cmd_sample(
run_cmd,
sample_every_n_steps,
sample_every_n_epochs,
sample_sampler,
sample_prompts,
output_dir,
)
# Pass the dynamically constructed keyword arguments to the function
run_cmd = run_cmd_advanced_training(run_cmd=run_cmd, **kwargs_for_training)
# run_cmd = run_cmd_sample(
# run_cmd,
# sample_every_n_steps,
# sample_every_n_epochs,
# sample_sampler,
# sample_prompts,
# output_dir,
# )
if max_data_loader_n_workers == "" or None:
max_data_loader_n_workers = 0
else:
max_data_loader_n_workers = int(max_data_loader_n_workers)
if max_train_steps == "" or None:
max_train_steps = 0
else:
max_train_steps = int(max_train_steps)
# def save_huggingface_to_toml(self, toml_file_path: str):
config_toml_data = {
# Update the values in the TOML data
"huggingface_repo_id": huggingface_repo_id,
"huggingface_token": huggingface_token,
"huggingface_repo_type": huggingface_repo_type,
"huggingface_repo_visibility": huggingface_repo_visibility,
"huggingface_path_in_repo": huggingface_path_in_repo,
"save_state_to_huggingface": save_state_to_huggingface,
"resume_from_huggingface": resume_from_huggingface,
"async_upload": async_upload,
"adaptive_noise_scale": adaptive_noise_scale,
"bucket_no_upscale": bucket_no_upscale,
"bucket_reso_steps": bucket_reso_steps,
"cache_latents": cache_latents,
"cache_latents_to_disk": cache_latents_to_disk,
"caption_dropout_every_n_epochs": caption_dropout_every_n_epochs,
"caption_extension": caption_extension,
"clip_skip": int(clip_skip),
"color_aug": color_aug,
"dataset_config": dataset_config,
"enable_bucket": enable_bucket,
"epoch": int(epoch),
"flip_aug": flip_aug,
"full_fp16": full_fp16,
"gradient_accumulation_steps": gradient_accumulation_steps,
"gradient_checkpointing": gradient_checkpointing,
"init_word": init_word,
"ip_noise_gamma": ip_noise_gamma,
"ip_noise_gamma_random_strength": ip_noise_gamma_random_strength,
"keep_tokens": int(keep_tokens),
"learning_rate": learning_rate,
"logging_dir": logging_dir,
"log_tracker_name": log_tracker_name,
"log_tracker_config": log_tracker_config,
"lr_scheduler": lr_scheduler,
"lr_scheduler_args": str(lr_scheduler_args).replace('"', '').split(),
"lr_scheduler_num_cycles": lr_scheduler_num_cycles,
"lr_scheduler_power": lr_scheduler_power,
"lr_warmup_steps": lr_warmup_steps,
"max_bucket_reso": max_bucket_reso,
"max_data_loader_n_workers": max_data_loader_n_workers,
"max_timestep": max_timestep,
"max_token_length": int(max_token_length),
"max_train_epochs": max_train_epochs,
"max_train_steps": int(max_train_steps),
"mem_eff_attn": mem_eff_attn,
"min_bucket_reso": int(min_bucket_reso),
"min_snr_gamma": min_snr_gamma,
"min_timestep": int(min_timestep),
"mixed_precision": mixed_precision,
"multires_noise_discount": multires_noise_discount,
"multires_noise_iterations": multires_noise_iterations,
"no_half_vae": sdxl_no_half_vae,
"no_token_padding": no_token_padding,
"noise_offset": noise_offset,
"noise_offset_random_strength": noise_offset_random_strength,
"noise_offset_type": noise_offset_type,
"num_vectors_per_token": int(num_vectors_per_token),
"optimizer_type": optimizer,
"optimizer_args": str(optimizer_args).replace('"', '').split(),
"output_dir": output_dir,
"output_name": output_name,
"persistent_data_loader_workers": persistent_data_loader_workers,
"pretrained_model_name_or_path": pretrained_model_name_or_path,
"prior_loss_weight": prior_loss_weight,
"random_crop": random_crop,
"reg_data_dir": reg_data_dir,
"resolution": max_resolution,
"resume": resume,
"sample_every_n_epochs": sample_every_n_epochs,
"sample_every_n_steps": sample_every_n_steps,
"sample_prompts": create_prompt_file(output_dir, output_dir),
"sample_sampler": sample_sampler,
"save_every_n_epochs": save_every_n_epochs,
"save_every_n_steps": save_every_n_steps,
"save_last_n_steps": save_last_n_steps,
"save_last_n_steps_state": save_last_n_steps_state,
"save_model_as": save_model_as,
"save_precision": save_precision,
"save_state": save_state,
"save_state_on_train_end": save_state_on_train_end,
"scale_v_pred_loss_like_noise_pred": scale_v_pred_loss_like_noise_pred,
"seed": int(seed),
"shuffle_caption": shuffle_caption,
"stop_text_encoder_training": stop_text_encoder_training,
"token_string": token_string,
"train_batch_size": train_batch_size,
"train_data_dir": train_data_dir,
"use_wandb": use_wandb,
"v2": v2,
"v_parameterization": v_parameterization,
"v_pred_like_loss": v_pred_like_loss,
"vae": vae,
"vae_batch_size": vae_batch_size,
"wandb_api_key": wandb_api_key,
"wandb_run_name": wandb_run_name,
"weigts": weights,
"use_object_template": True if template == "object template" else None,
"use_style_template": True if template == "style template" else None,
"xformers": xformers,
"additional_parameters": additional_parameters,
"loss_type": loss_type,
"huber_schedule": huber_schedule,
"huber_c": huber_c,
}
# Given dictionary `config_toml_data`
# Remove all values = ""
config_toml_data = {
key: value
for key, value in config_toml_data.items()
if value != "" and value != False
}
tmpfilename = "./outputs/tmpfileti.toml"
# Save the updated TOML data back to the file
with open(tmpfilename, "w") as toml_file:
toml.dump(config_toml_data, toml_file)
if not os.path.exists(toml_file.name):
log.error(f"Failed to write TOML file: {toml_file.name}")
toml_file_path = (
os.path.abspath(os.path.normpath(toml_file.name)).replace("\\", "/")
if os.name == "nt"
else toml_file.name
)
run_cmd.append(f"--config_file")
run_cmd.append(rf"{toml_file_path}")
if print_only:
log.warning(
@ -870,6 +962,10 @@ def ti_tab(headless=False, default_output_dir=None, config: dict = {}):
with gr.Accordion("Samples", open=False, elem_id="samples_tab"):
sample = SampleImages(config=config)
global huggingface
with gr.Accordion("HuggingFace", open=False):
huggingface = HuggingFace(config=config)
with gr.Column(), gr.Group():
with gr.Row():
button_run = gr.Button("Start training", variant="primary")
@ -990,6 +1086,14 @@ def ti_tab(headless=False, default_output_dir=None, config: dict = {}):
advanced_training.max_timestep,
sdxl_params.sdxl_no_half_vae,
accelerate_launch.extra_accelerate_launch_args,
huggingface.huggingface_repo_id,
huggingface.huggingface_token,
huggingface.huggingface_repo_type,
huggingface.huggingface_repo_visibility,
huggingface.huggingface_path_in_repo,
huggingface.save_state_to_huggingface,
huggingface.resume_from_huggingface,
huggingface.async_upload,
]
configuration.button_open_config.click(

21
test/config/dreambooth-AdamW8bit.json
View File

@ -1,11 +1,12 @@
{
"adaptive_noise_scale": 0,
"additional_parameters": "",
"additional_parameters": "--config_file ./outputs/tmpfile.toml",
"async_upload": false,
"bucket_no_upscale": true,
"bucket_reso_steps": 64,
"cache_latents": true,
"cache_latents_to_disk": false,
"caption_dropout_every_n_epochs": 0.0,
"caption_dropout_every_n_epochs": 0,
"caption_dropout_rate": 0.05,
"caption_extension": "",
"clip_skip": 2,
@ -23,6 +24,11 @@
"gradient_checkpointing": false,
"huber_c": 0.1,
"huber_schedule": "snr",
"huggingface_path_in_repo": "",
"huggingface_repo_id": "",
"huggingface_repo_type": "",
"huggingface_repo_visibility": "",
"huggingface_token": "",
"ip_noise_gamma": 0.1,
"ip_noise_gamma_random_strength": true,
"keep_tokens": "0",
@ -52,7 +58,7 @@
"min_bucket_reso": 256,
"min_snr_gamma": 0,
"min_timestep": 0,
"mixed_precision": "bf16",
"mixed_precision": "fp16",
"model_list": "runwayml/stable-diffusion-v1-5",
"multi_gpu": false,
"multires_noise_discount": 0,
@ -63,21 +69,23 @@
"noise_offset_type": "Original",
"num_cpu_threads_per_process": 2,
"num_machines": 1,
"num_processes": 1,
"num_processes": 8,
"optimizer": "AdamW8bit",
"optimizer_args": "",
"output_dir": "./test/output",
"output_name": "db-AdamW8bit",
"persistent_data_loader_workers": false,
"pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
"prior_loss_weight": 1.0,
"prior_loss_weight": 1,
"random_crop": false,
"reg_data_dir": "",
"resume": "",
"resume_from_huggingface": "",
"sample_every_n_epochs": 0,
"sample_every_n_steps": 20,
"sample_prompts": "a painting of a gas mask , by darius kawasaki",
"sample_sampler": "euler_a",
"save_as_bool": false,
"save_every_n_epochs": 1,
"save_every_n_steps": 0,
"save_last_n_steps": 0,
@ -86,6 +94,7 @@
"save_precision": "fp16",
"save_state": false,
"save_state_on_train_end": false,
"save_state_to_huggingface": false,
"scale_v_pred_loss_like_noise_pred": false,
"sdxl": false,
"seed": "1234",
@ -97,7 +106,7 @@
"v2": false,
"v_parameterization": false,
"v_pred_like_loss": 0,
"vae": "stabilityai/vae",
"vae": "",
"vae_batch_size": 0,
"wandb_api_key": "",
"wandb_run_name": "",

56
test/config/dreambooth-Prodigy.json
View File

@ -1,54 +1,91 @@
{
"adaptive_noise_scale": 0,
"additional_parameters": "",
"async_upload": false,
"bucket_no_upscale": true,
"bucket_reso_steps": 1,
"cache_latents": true,
"cache_latents_to_disk": false,
"caption_dropout_every_n_epochs": 0.0,
"caption_dropout_every_n_epochs": 0,
"caption_dropout_rate": 0,
"caption_extension": "",
"clip_skip": 2,
"clip_skip": 1,
"color_aug": false,
"dataset_config": "",
"debiased_estimation_loss": true,
"enable_bucket": true,
"epoch": 1,
"extra_accelerate_launch_args": "",
"flip_aug": false,
"full_bf16": false,
"full_fp16": false,
"gradient_accumulation_steps": 1.0,
"gpu_ids": "",
"gradient_accumulation_steps": 1,
"gradient_checkpointing": false,
"huber_c": 0.1,
"huber_schedule": "snr",
"huggingface_path_in_repo": "",
"huggingface_repo_id": "",
"huggingface_repo_type": "",
"huggingface_repo_visibility": "",
"huggingface_token": "",
"ip_noise_gamma": 0.1,
"ip_noise_gamma_random_strength": true,
"keep_tokens": "0",
"learning_rate": 1.0,
"learning_rate": 1,
"learning_rate_te": 1e-05,
"learning_rate_te1": 1e-05,
"learning_rate_te2": 0,
"log_tracker_config": "",
"log_tracker_name": "",
"logging_dir": "./test/logs",
"loss_type": "l2",
"lr_scheduler": "cosine",
"lr_scheduler_args": "",
"lr_scheduler_num_cycles": "",
"lr_scheduler_power": "",
"lr_warmup": 0,
"main_process_port": 0,
"masked_loss": false,
"max_bucket_reso": 2048,
"max_data_loader_n_workers": "0",
"max_resolution": "512,512",
"max_timestep": 1000,
"max_token_length": "75",
"max_train_epochs": "",
"max_train_steps": "30",
"mem_eff_attn": false,
"min_bucket_reso": 256,
"min_snr_gamma": 0,
"min_timestep": 0,
"mixed_precision": "bf16",
"model_list": "runwayml/stable-diffusion-v1-5",
"multi_gpu": false,
"multires_noise_discount": 0.2,
"multires_noise_iterations": 8,
"no_token_padding": false,
"noise_offset": "0.05",
"noise_offset": 0.05,
"noise_offset_random_strength": true,
"noise_offset_type": "Multires",
"num_cpu_threads_per_process": 2,
"num_machines": 1,
"num_processes": 1,
"optimizer": "Prodigy",
"optimizer_args": "decouple=True weight_decay=0.6 betas=0.9,0.99 use_bias_correction=True",
"output_dir": "./test/output",
"output_name": "db-Prodigy",
"persistent_data_loader_workers": false,
"pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
"prior_loss_weight": 1.0,
"prior_loss_weight": 1,
"random_crop": false,
"reg_data_dir": "",
"resume": "",
"resume_from_huggingface": "",
"sample_every_n_epochs": 0,
"sample_every_n_steps": 25,
"sample_prompts": "a painting of a gas mask , by darius kawasaki",
"sample_sampler": "euler_a",
"save_as_bool": false,
"save_every_n_epochs": 1,
"save_every_n_steps": 0,
"save_last_n_steps": 0,
@ -56,7 +93,10 @@
"save_model_as": "safetensors",
"save_precision": "fp16",
"save_state": false,
"save_state_on_train_end": false,
"save_state_to_huggingface": false,
"scale_v_pred_loss_like_noise_pred": false,
"sdxl": false,
"seed": "1234",
"shuffle_caption": false,
"stop_text_encoder_training": 0,
@ -65,9 +105,11 @@
"use_wandb": false,
"v2": false,
"v_parameterization": false,
"v_pred_like_loss": 0,
"vae": "",
"vae_batch_size": 0,
"wandb_api_key": "",
"wandb_run_name": "",
"weighted_captions": false,
"xformers": true
"xformers": "xformers"
}