1580158
/
kohya_ss
mirror of https://github.com/bmaltais/kohya_ss
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2137 from bmaltais/dev 

v23.0.15
pull/2149/head v23.0.15
bmaltais 2024-03-21 13:39:18 -04:00 committed by GitHub
parent 55386bbb29 aaf0396d62
commit 5bbb4fcf78
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
25 changed files with 1721 additions and 651 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -50,4 +50,5 @@ dataset/**
!dataset/**/.gitkeep
models
data
config.toml
config.toml
sd-scripts

2
.release
View File

@ -1 +1 @@
v23.0.14
v23.0.15

19
README.md
View File

@ -38,8 +38,9 @@ The GUI allows you to set the training parameters and generate and run the requi
- [No module called tkinter](#no-module-called-tkinter)
- [SDXL training](#sdxl-training)
- [Change History](#change-history)
- [2024/03/13 (v23.0.14)](#20240313-v23014)
- [2024/03/13 (v23.0.13)](#20240313-v23013)
- [2024/03/20 (v23.0.15)](#20240320-v23015)
- [2024/03/19 (v23.0.14)](#20240319-v23014)
- [2024/03/19 (v23.0.13)](#20240319-v23013)
- [2024/03/16 (v23.0.12)](#20240316-v23012)
- [New Features \& Improvements](#new-features--improvements)
- [Software Updates](#software-updates)
@ -381,11 +382,19 @@ The documentation in this section will be moved to a separate document later.
## Change History
### 2024/03/13 (v23.0.14)
### 2024/03/21 (v23.0.15)
- Add support for toml dataset configuration fole to all trainers
- Add new setup menu option to install Triton 2.1.0 for Windows
- Add support for LyCORIS BOFT and DoRA and QLyCORIS options for LoHA, LoKr and LoCon
- Fix issue with vae path validation
- Other fixes
### 2024/03/19 (v23.0.14)
- Fix blip caption issue
-
### 2024/03/13 (v23.0.13)
### 2024/03/19 (v23.0.13)
- Fix issue with image samples.

0
examples/stable_cascade/test.toml Normal file
View File

10
kohya_gui/class_advanced_training.py
View File

@ -7,6 +7,7 @@ from .common_gui import (
list_files,
list_dirs,
create_refresh_button,
document_symbol
)
@ -38,8 +39,7 @@ class AdvancedTraining:
headless (bool): Run in headless mode without GUI.
finetuning (bool): Enable model fine-tuning.
training_type (str): The type of training to be performed.
default_vae_dir (str): Default directory for VAE models.
default_output_dir (str): Default directory for output files.
config (dict): Configuration options for the training process.
"""
self.headless = headless
self.finetuning = finetuning
@ -368,10 +368,6 @@ class AdvancedTraining:
outputs=self.resume,
show_progress=False,
)
# self.max_train_epochs = gr.Textbox(
# label='Max train epoch',
# placeholder='(Optional) Override number of epoch',
# )
self.max_data_loader_n_workers = gr.Textbox(
label="Max num workers for DataLoader",
placeholder="(Optional) Override number of epoch. Default: 8",
@ -437,7 +433,7 @@ class AdvancedTraining:
"open_folder_small",
)
self.log_tracker_config_button = gr.Button(
"📂", elem_id="open_folder_small", visible=(not headless)
document_symbol, elem_id="open_folder_small", visible=(not headless)
)
self.log_tracker_config_button.click(
get_any_file_path,

2
kohya_gui/class_folders.py
View File

@ -1,6 +1,6 @@
import gradio as gr
import os
from .common_gui import get_folder_path, scriptdir, list_dirs, create_refresh_button
from .common_gui import get_folder_path, scriptdir, list_dirs, list_files, create_refresh_button
class Folders:
"""

67
kohya_gui/class_source_model.py
View File

@ -2,7 +2,7 @@ import gradio as gr
import os
from .common_gui import (
get_any_file_path,
get_file_path,
get_folder_path,
set_pretrained_model_name_or_path_input,
scriptdir,
@ -61,6 +61,8 @@ class SourceModel:
self.current_train_data_dir = self.config.get(
"train_data_dir", os.path.join(scriptdir, "data")
)
self.current_dataset_config_dir = self.config.get('dataset_config_dir', os.path.join(scriptdir, "dataset_config"))
model_checkpoints = list(
list_files(
@ -79,6 +81,21 @@ class SourceModel:
def list_train_data_dirs(path):
self.current_train_data_dir = path if not path == "" else "."
return list(list_dirs(path))
def list_dataset_config_dirs(path: str) -> list:
"""
List directories and toml files in the dataset_config directory.
Parameters:
- path (str): The path to list directories and files from.
Returns:
- list: A list of directories and files.
"""
current_dataset_config_dir = path if not path == "" else "."
# Lists all .json files in the current configuration directory, used for populating dropdown choices.
return list(list_files(current_dataset_config_dir, exts=[".toml"], all=True))
with gr.Column(), gr.Group():
# Define the input elements
@ -107,7 +124,7 @@ class SourceModel:
visible=(not headless),
)
self.pretrained_model_name_or_path_file.click(
get_any_file_path,
get_file_path,
inputs=self.pretrained_model_name_or_path,
outputs=self.pretrained_model_name_or_path,
show_progress=False,
@ -124,7 +141,15 @@ class SourceModel:
outputs=self.pretrained_model_name_or_path,
show_progress=False,
)
with gr.Column(), gr.Row():
self.output_name = gr.Textbox(
label="Trained Model output name",
placeholder="(Name of the model to output)",
value="last",
interactive=True,
)
with gr.Row():
with gr.Column(), gr.Row():
self.train_data_dir = gr.Dropdown(
label=(
@ -158,6 +183,36 @@ class SourceModel:
outputs=self.train_data_dir,
show_progress=False,
)
with gr.Column(), gr.Row():
# Toml directory dropdown
self.dataset_config = gr.Dropdown(
label='Dataset config file (Optional. Select the toml configuration file to use for the dataset)',
choices=[""] + list_dataset_config_dirs(self.current_dataset_config_dir),
value="",
interactive=True,
allow_custom_value=True,
)
# Refresh button for dataset_config directory
create_refresh_button(self.dataset_config, lambda: None, lambda: {"choices": [""] + list_dataset_config_dirs(self.current_dataset_config_dir)}, "open_folder_small")
# Toml directory button
self.dataset_config_folder = gr.Button(
document_symbol, elem_id='open_folder_small', elem_classes=["tool"], visible=(not self.headless)
)
# Toml directory button click event
self.dataset_config_folder.click(
get_file_path,
inputs=[self.dataset_config, gr.Textbox(value='*.toml', visible=False), gr.Textbox(value='Dataset config types', visible=False)],
outputs=self.dataset_config,
show_progress=False,
)
# Change event for dataset_config directory dropdown
self.dataset_config.change(
fn=lambda path: gr.Dropdown(choices=[""] + list_dataset_config_dirs(path)),
inputs=self.dataset_config,
outputs=self.dataset_config,
show_progress=False,
)
with gr.Row():
with gr.Column():
@ -181,12 +236,6 @@ class SourceModel:
gr.Box(visible=False)
with gr.Row():
self.output_name = gr.Textbox(
label="Trained Model output name",
placeholder="(Name of the model to output)",
value="last",
interactive=True,
)
self.training_comment = gr.Textbox(
label="Training comment",
placeholder="(Optional) Add training comment to be included in metadata",

29
kohya_gui/common_gui.py
View File

@ -54,6 +54,23 @@ ALL_PRESET_MODELS = V2_BASE_MODELS + V_PARAMETERIZATION_MODELS + V1_MODELS + SDX
ENV_EXCLUSION = ["COLAB_GPU", "RUNPOD_POD_ID"]
def calculate_max_train_steps(
total_steps: int,
train_batch_size: int,
gradient_accumulation_steps: int,
epoch: int,
reg_factor: int,
):
return int(
math.ceil(
float(total_steps)
/ int(train_batch_size)
/ int(gradient_accumulation_steps)
* int(epoch)
* int(reg_factor)
)
)
def check_if_model_exist(
output_name: str, output_dir: str, save_model_as: str, headless: bool = False
) -> bool:
@ -1077,6 +1094,11 @@ def run_cmd_advanced_training(**kwargs):
if color_aug:
run_cmd += " --color_aug"
dataset_config = kwargs.get("dataset_config")
if dataset_config:
dataset_config = os.path.abspath(os.path.normpath(dataset_config))
run_cmd += f' --dataset_config="{dataset_config}"'
dataset_repeats = kwargs.get("dataset_repeats")
if dataset_repeats:
run_cmd += f' --dataset_repeats="{dataset_repeats}"'
@ -1753,6 +1775,13 @@ def validate_paths(headless: bool = False, **kwargs: Optional[str]) -> bool:
if key in ["output_dir", "logging_dir"]:
if not validate_path(value, key, create_if_missing=True):
return False
elif key in ["vae"]:
# Check if it matches the Hugging Face model pattern
if re.match(r"^[\w-]+\/[\w-]+$", value):
log.info("Checking vae... huggingface.co model, skipping validation")
else:
if not validate_path(value, key):
return False
else:
if key not in ["pretrained_model_name_or_path"]:
if not validate_path(value, key):

177
kohya_gui/dreambooth_gui.py
View File

@ -58,6 +58,7 @@ def save_configuration(
train_data_dir,
reg_data_dir,
output_dir,
dataset_config,
max_resolution,
learning_rate,
learning_rate_te,
@ -189,6 +190,7 @@ def open_configuration(
train_data_dir,
reg_data_dir,
output_dir,
dataset_config,
max_resolution,
learning_rate,
learning_rate_te,
@ -315,6 +317,7 @@ def train_model(
train_data_dir,
reg_data_dir,
output_dir,
dataset_config,
max_resolution,
learning_rate,
learning_rate_te,
@ -421,6 +424,7 @@ def train_model(
log_tracker_config=log_tracker_config,
resume=resume,
vae=vae,
dataset_config=dataset_config,
):
return
@ -429,102 +433,90 @@ def train_model(
):
return
# if sdxl:
# output_message(
# msg='Dreambooth training is not compatible with SDXL models yet..',
# headless=headless_bool,
# )
# return
# if optimizer == 'Adafactor' and lr_warmup != '0':
# output_message(
# msg="Warning: lr_scheduler is set to 'Adafactor', so 'LR warmup (% of steps)' will be considered 0.",
# title='Warning',
# headless=headless_bool,
# )
# lr_warmup = '0'
# Get a list of all subfolders in train_data_dir, excluding hidden folders
subfolders = [
f
for f in os.listdir(train_data_dir)
if os.path.isdir(os.path.join(train_data_dir, f)) and not f.startswith(".")
]
# Check if subfolders are present. If not let the user know and return
if not subfolders:
log.info(f"No {subfolders} were found in train_data_dir can't train...")
return
total_steps = 0
# Loop through each subfolder and extract the number of repeats
for folder in subfolders:
# Extract the number of repeats from the folder name
try:
repeats = int(folder.split("_")[0])
except ValueError:
log.info(
f"Subfolder {folder} does not have a proper repeat value, please correct the name or remove it... can't train..."
)
continue
# Count the number of images in the folder
num_images = len(
[
f
for f, lower_f in (
(file, file.lower())
for file in os.listdir(os.path.join(train_data_dir, folder))
)
if lower_f.endswith((".jpg", ".jpeg", ".png", ".webp"))
]
)
if num_images == 0:
log.info(f"{folder} folder contain no images, skipping...")
else:
# Calculate the total number of steps for this folder
steps = repeats * num_images
total_steps += steps
# Print the result
log.info(f"Folder {folder} : steps {steps}")
if total_steps == 0:
log.info(f"No images were found in folder {train_data_dir}... please rectify!")
return
# Print the result
# log.info(f"{total_steps} total steps")
if reg_data_dir == "":
reg_factor = 1
if dataset_config:
log.info("Dataset config toml file used, skipping total_steps, train_batch_size, gradient_accumulation_steps, epoch, reg_factor, max_train_steps calculations...")
else:
log.info(
f"Regularisation images are used... Will double the number of steps required..."
)
reg_factor = 2
# Get a list of all subfolders in train_data_dir, excluding hidden folders
subfolders = [
f
for f in os.listdir(train_data_dir)
if os.path.isdir(os.path.join(train_data_dir, f)) and not f.startswith(".")
]
if max_train_steps == "" or max_train_steps == "0":
# calculate max_train_steps
max_train_steps = int(
math.ceil(
float(total_steps)
/ int(train_batch_size)
/ int(gradient_accumulation_steps)
* int(epoch)
* int(reg_factor)
# Check if subfolders are present. If not let the user know and return
if not subfolders:
log.info(f"No {subfolders} were found in train_data_dir can't train...")
return
total_steps = 0
# Loop through each subfolder and extract the number of repeats
for folder in subfolders:
# Extract the number of repeats from the folder name
try:
repeats = int(folder.split("_")[0])
except ValueError:
log.info(
f"Subfolder {folder} does not have a proper repeat value, please correct the name or remove it... can't train..."
)
continue
# Count the number of images in the folder
num_images = len(
[
f
for f, lower_f in (
(file, file.lower())
for file in os.listdir(os.path.join(train_data_dir, folder))
)
if lower_f.endswith((".jpg", ".jpeg", ".png", ".webp"))
]
)
if num_images == 0:
log.info(f"{folder} folder contain no images, skipping...")
else:
# Calculate the total number of steps for this folder
steps = repeats * num_images
total_steps += steps
# Print the result
log.info(f"Folder {folder} : steps {steps}")
if total_steps == 0:
log.info(f"No images were found in folder {train_data_dir}... please rectify!")
return
# Print the result
# log.info(f"{total_steps} total steps")
if reg_data_dir == "":
reg_factor = 1
else:
log.info(
f"Regularisation images are used... Will double the number of steps required..."
)
reg_factor = 2
if max_train_steps == "" or max_train_steps == "0":
# calculate max_train_steps
max_train_steps = int(
math.ceil(
float(total_steps)
/ int(train_batch_size)
/ int(gradient_accumulation_steps)
* int(epoch)
* int(reg_factor)
)
)
log.info(
f"max_train_steps ({total_steps} / {train_batch_size} / {gradient_accumulation_steps} * {epoch} * {reg_factor}) = {max_train_steps}"
)
)
log.info(
f"max_train_steps ({total_steps} / {train_batch_size} / {gradient_accumulation_steps} * {epoch} * {reg_factor}) = {max_train_steps}"
)
# calculate stop encoder training
if int(stop_text_encoder_training_pct) == -1:
stop_text_encoder_training = -1
elif stop_text_encoder_training_pct == None:
elif stop_text_encoder_training_pct == None or (not max_train_steps == "" or not max_train_steps == "0"):
stop_text_encoder_training = 0
else:
stop_text_encoder_training = math.ceil(
@ -532,7 +524,10 @@ def train_model(
)
log.info(f"stop_text_encoder_training = {stop_text_encoder_training}")
lr_warmup_steps = round(float(int(lr_warmup) * int(max_train_steps) / 100))
if not max_train_steps == "":
lr_warmup_steps = round(float(int(lr_warmup) * int(max_train_steps) / 100))
else:
lr_warmup_steps = 0
log.info(f"lr_warmup_steps = {lr_warmup_steps}")
# run_cmd = f'accelerate launch --num_cpu_threads_per_process={num_cpu_threads_per_process} "train_db.py"'
@ -564,6 +559,7 @@ def train_model(
"caption_extension": caption_extension,
"clip_skip": clip_skip,
"color_aug": color_aug,
"dataset_config": dataset_config,
"enable_bucket": enable_bucket,
"epoch": epoch,
"flip_aug": flip_aug,
@ -788,6 +784,7 @@ def dreambooth_tab(
source_model.train_data_dir,
folders.reg_data_dir,
folders.output_dir,
source_model.dataset_config,
basic_training.max_resolution,
basic_training.learning_rate,
basic_training.learning_rate_te,

153
kohya_gui/finetune_gui.py
View File

@ -62,6 +62,7 @@ def save_configuration(
train_dir,
image_folder,
output_dir,
dataset_config,
logging_dir,
max_resolution,
min_bucket_reso,
@ -108,6 +109,7 @@ def save_configuration(
output_name,
max_token_length,
max_train_epochs,
max_train_steps,
max_data_loader_n_workers,
full_fp16,
color_aug,
@ -200,6 +202,7 @@ def open_configuration(
train_dir,
image_folder,
output_dir,
dataset_config,
logging_dir,
max_resolution,
min_bucket_reso,
@ -246,6 +249,7 @@ def open_configuration(
output_name,
max_token_length,
max_train_epochs,
max_train_steps,
max_data_loader_n_workers,
full_fp16,
color_aug,
@ -345,6 +349,7 @@ def train_model(
train_dir,
image_folder,
output_dir,
dataset_config,
logging_dir,
max_resolution,
min_bucket_reso,
@ -391,6 +396,7 @@ def train_model(
output_name,
max_token_length,
max_train_epochs,
max_train_steps,
max_data_loader_n_workers,
full_fp16,
color_aug,
@ -455,93 +461,99 @@ def train_model(
logging_dir=logging_dir,
log_tracker_config=log_tracker_config,
resume=resume,
dataset_config=dataset_config
):
return
if not print_only_bool and check_if_model_exist(output_name, output_dir, save_model_as, headless_bool):
return
# create caption json file
if generate_caption_database:
run_cmd = fr'"{PYTHON}" "{scriptdir}/sd-scripts/finetune/merge_captions_to_metadata.py"'
if caption_extension == "":
run_cmd += f' --caption_extension=".caption"'
else:
run_cmd += f" --caption_extension={caption_extension}"
run_cmd += fr' "{image_folder}"'
run_cmd += fr' "{train_dir}/{caption_metadata_filename}"'
if full_path:
run_cmd += f" --full_path"
if dataset_config:
log.info("Dataset config toml file used, skipping caption json file, image buckets, total_steps, train_batch_size, gradient_accumulation_steps, epoch, reg_factor, max_train_steps creation...")
else:
# create caption json file
if generate_caption_database:
run_cmd = fr'"{PYTHON}" "{scriptdir}/sd-scripts/finetune/merge_captions_to_metadata.py"'
if caption_extension == "":
run_cmd += f' --caption_extension=".caption"'
else:
run_cmd += f" --caption_extension={caption_extension}"
run_cmd += fr' "{image_folder}"'
run_cmd += fr' "{train_dir}/{caption_metadata_filename}"'
if full_path:
run_cmd += f" --full_path"
log.info(run_cmd)
log.info(run_cmd)
env = os.environ.copy()
env['PYTHONPATH'] = fr"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}"
env = os.environ.copy()
env['PYTHONPATH'] = fr"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}"
if not print_only_bool:
# Run the command
subprocess.run(run_cmd, shell=True, env=env)
if not print_only_bool:
# Run the command
subprocess.run(run_cmd, shell=True, env=env)
# create images buckets
if generate_image_buckets:
run_cmd = fr'"{PYTHON}" "{scriptdir}/sd-scripts/finetune/prepare_buckets_latents.py"'
run_cmd += fr' "{image_folder}"'
run_cmd += fr' "{train_dir}/{caption_metadata_filename}"'
run_cmd += fr' "{train_dir}/{latent_metadata_filename}"'
run_cmd += fr' "{pretrained_model_name_or_path}"'
run_cmd += f" --batch_size={batch_size}"
run_cmd += f" --max_resolution={max_resolution}"
run_cmd += f" --min_bucket_reso={min_bucket_reso}"
run_cmd += f" --max_bucket_reso={max_bucket_reso}"
run_cmd += f" --mixed_precision={mixed_precision}"
# if flip_aug:
# run_cmd += f' --flip_aug'
if full_path:
run_cmd += f" --full_path"
if sdxl_checkbox and sdxl_no_half_vae:
log.info("Using mixed_precision = no because no half vae is selected...")
run_cmd += f' --mixed_precision="no"'
# create images buckets
if generate_image_buckets:
run_cmd = fr'"{PYTHON}" "{scriptdir}/sd-scripts/finetune/prepare_buckets_latents.py"'
run_cmd += fr' "{image_folder}"'
run_cmd += fr' "{train_dir}/{caption_metadata_filename}"'
run_cmd += fr' "{train_dir}/{latent_metadata_filename}"'
run_cmd += fr' "{pretrained_model_name_or_path}"'
run_cmd += f" --batch_size={batch_size}"
run_cmd += f" --max_resolution={max_resolution}"
run_cmd += f" --min_bucket_reso={min_bucket_reso}"
run_cmd += f" --max_bucket_reso={max_bucket_reso}"
run_cmd += f" --mixed_precision={mixed_precision}"
# if flip_aug:
# run_cmd += f' --flip_aug'
if full_path:
run_cmd += f" --full_path"
if sdxl_checkbox and sdxl_no_half_vae:
log.info("Using mixed_precision = no because no half vae is selected...")
run_cmd += f' --mixed_precision="no"'
log.info(run_cmd)
log.info(run_cmd)
env = os.environ.copy()
env['PYTHONPATH'] = fr"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}"
env = os.environ.copy()
env['PYTHONPATH'] = fr"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}"
if not print_only_bool:
# Run the command
subprocess.run(run_cmd, shell=True, env=env)
if not print_only_bool:
# Run the command
subprocess.run(run_cmd, shell=True, env=env)
image_num = len(
[
f
for f, lower_f in (
(file, file.lower()) for file in os.listdir(image_folder)
)
if lower_f.endswith((".jpg", ".jpeg", ".png", ".webp"))
]
)
log.info(f"image_num = {image_num}")
repeats = int(image_num) * int(dataset_repeats)
log.info(f"repeats = {str(repeats)}")
# calculate max_train_steps
max_train_steps = int(
math.ceil(
float(repeats)
/ int(train_batch_size)
/ int(gradient_accumulation_steps)
* int(epoch)
image_num = len(
[
f
for f, lower_f in (
(file, file.lower()) for file in os.listdir(image_folder)
)
if lower_f.endswith((".jpg", ".jpeg", ".png", ".webp"))
]
)
)
log.info(f"image_num = {image_num}")
# Divide by two because flip augmentation create two copied of the source images
if flip_aug:
max_train_steps = int(math.ceil(float(max_train_steps) / 2))
repeats = int(image_num) * int(dataset_repeats)
log.info(f"repeats = {str(repeats)}")
log.info(f"max_train_steps = {max_train_steps}")
# calculate max_train_steps
max_train_steps = int(
math.ceil(
float(repeats)
/ int(train_batch_size)
/ int(gradient_accumulation_steps)
* int(epoch)
)
)
lr_warmup_steps = round(float(int(lr_warmup) * int(max_train_steps) / 100))
# Divide by two because flip augmentation create two copied of the source images
if flip_aug and max_train_steps:
max_train_steps = int(math.ceil(float(max_train_steps) / 2))
if max_train_steps != "":
log.info(f"max_train_steps = {max_train_steps}")
lr_warmup_steps = round(float(int(lr_warmup) * int(max_train_steps) / 100))
else:
lr_warmup_steps = 0
log.info(f"lr_warmup_steps = {lr_warmup_steps}")
run_cmd = "accelerate launch"
@ -581,6 +593,7 @@ def train_model(
"caption_extension": caption_extension,
"clip_skip": clip_skip,
"color_aug": color_aug,
"dataset_config": dataset_config,
"dataset_repeats": dataset_repeats,
"enable_bucket": True,
"flip_aug": flip_aug,
@ -861,6 +874,7 @@ def finetune_tab(headless=False, config: dict = {}):
train_dir,
image_folder,
output_dir,
source_model.dataset_config,
logging_dir,
max_resolution,
min_bucket_reso,
@ -906,6 +920,7 @@ def finetune_tab(headless=False, config: dict = {}):
output_name,
advanced_training.max_token_length,
basic_training.max_train_epochs,
basic_training.max_train_steps,
advanced_training.max_data_loader_n_workers,
advanced_training.full_fp16,
advanced_training.color_aug,

788
kohya_gui/lora_gui.py
View File

File diff suppressed because it is too large Load Diff

116
kohya_gui/textual_inversion_gui.py
View File

@ -58,6 +58,7 @@ def save_configuration(
train_data_dir,
reg_data_dir,
output_dir,
dataset_config,
max_resolution,
learning_rate,
lr_scheduler,
@ -192,6 +193,7 @@ def open_configuration(
train_data_dir,
reg_data_dir,
output_dir,
dataset_config,
max_resolution,
learning_rate,
lr_scheduler,
@ -319,6 +321,7 @@ def train_model(
train_data_dir,
reg_data_dir,
output_dir,
dataset_config,
max_resolution,
learning_rate,
lr_scheduler,
@ -424,6 +427,7 @@ def train_model(
log_tracker_config=log_tracker_config,
resume=resume,
vae=vae,
dataset_config=dataset_config,
):
return
@ -440,68 +444,71 @@ def train_model(
):
return
# Get a list of all subfolders in train_data_dir
subfolders = [
f
for f in os.listdir(train_data_dir)
if os.path.isdir(os.path.join(train_data_dir, f))
]
if dataset_config:
log.info("Dataset config toml file used, skipping total_steps, train_batch_size, gradient_accumulation_steps, epoch, reg_factor, max_train_steps calculations...")
else:
# Get a list of all subfolders in train_data_dir
subfolders = [
f
for f in os.listdir(train_data_dir)
if os.path.isdir(os.path.join(train_data_dir, f))
]
total_steps = 0
total_steps = 0
# Loop through each subfolder and extract the number of repeats
for folder in subfolders:
# Extract the number of repeats from the folder name
repeats = int(folder.split("_")[0])
# Loop through each subfolder and extract the number of repeats
for folder in subfolders:
# Extract the number of repeats from the folder name
repeats = int(folder.split("_")[0])
# Count the number of images in the folder
num_images = len(
[
f
for f, lower_f in (
(file, file.lower())
for file in os.listdir(os.path.join(train_data_dir, folder))
)
if lower_f.endswith((".jpg", ".jpeg", ".png", ".webp"))
]
)
# Count the number of images in the folder
num_images = len(
[
f
for f, lower_f in (
(file, file.lower())
for file in os.listdir(os.path.join(train_data_dir, folder))
)
if lower_f.endswith((".jpg", ".jpeg", ".png", ".webp"))
]
)
# Calculate the total number of steps for this folder
steps = repeats * num_images
total_steps += steps
# Calculate the total number of steps for this folder
steps = repeats * num_images
total_steps += steps
# Print the result
log.info(f"Folder {folder}: {steps} steps")
# Print the result
log.info(f"Folder {folder}: {steps} steps")
# log.info(f"{total_steps} total steps")
# Print the result
# log.info(f"{total_steps} total steps")
if reg_data_dir == "":
reg_factor = 1
else:
log.info(
"Regularisation images are used... Will double the number of steps required..."
)
reg_factor = 2
# calculate max_train_steps
if max_train_steps == "" or max_train_steps == "0":
max_train_steps = int(
math.ceil(
float(total_steps)
/ int(train_batch_size)
/ int(gradient_accumulation_steps)
* int(epoch)
* int(reg_factor)
if reg_data_dir == "":
reg_factor = 1
else:
log.info(
"Regularisation images are used... Will double the number of steps required..."
)
)
else:
max_train_steps = int(max_train_steps)
reg_factor = 2
log.info(f"max_train_steps = {max_train_steps}")
# calculate max_train_steps
if max_train_steps == "" or max_train_steps == "0":
max_train_steps = int(
math.ceil(
float(total_steps)
/ int(train_batch_size)
/ int(gradient_accumulation_steps)
* int(epoch)
* int(reg_factor)
)
)
else:
max_train_steps = int(max_train_steps)
log.info(f"max_train_steps = {max_train_steps}")
# calculate stop encoder training
if stop_text_encoder_training_pct == None:
if stop_text_encoder_training_pct == None or (not max_train_steps == "" or not max_train_steps == "0"):
stop_text_encoder_training = 0
else:
stop_text_encoder_training = math.ceil(
@ -509,7 +516,10 @@ def train_model(
)
log.info(f"stop_text_encoder_training = {stop_text_encoder_training}")
lr_warmup_steps = round(float(int(lr_warmup) * int(max_train_steps) / 100))
if not max_train_steps == "":
lr_warmup_steps = round(float(int(lr_warmup) * int(max_train_steps) / 100))
else:
lr_warmup_steps = 0
log.info(f"lr_warmup_steps = {lr_warmup_steps}")
run_cmd = "accelerate launch"
@ -538,6 +548,7 @@ def train_model(
caption_extension=caption_extension,
clip_skip=clip_skip,
color_aug=color_aug,
dataset_config=dataset_config,
enable_bucket=enable_bucket,
epoch=epoch,
flip_aug=flip_aug,
@ -843,6 +854,7 @@ def ti_tab(headless=False, default_output_dir=None, config: dict = {}):
source_model.train_data_dir,
folders.reg_data_dir,
folders.output_dir,
source_model.dataset_config,
basic_training.max_resolution,
basic_training.learning_rate,
basic_training.lr_scheduler,

84
setup/setup_windows.py
View File

@ -22,9 +22,12 @@ def cudnn_install():
"nvidia-cudnn-cu11 8.9.5.29",
reinstall=True,
)
# Original path with "..\\venv"
original_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..\\venv\\Lib\\site-packages\\nvidia\\cudnn\\bin")
original_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
"..\\venv\\Lib\\site-packages\\nvidia\\cudnn\\bin",
)
# Normalize the path to resolve "..\\venv"
cudnn_src = os.path.abspath(original_path)
cudnn_dest = os.path.join(sysconfig.get_paths()["purelib"], "torch", "lib")
@ -35,7 +38,7 @@ def cudnn_install():
# check for different files
filecmp.clear_cache()
for file in os.listdir(cudnn_src):
if file.lower().endswith('.dll'): # Check if the file is a .dll file
if file.lower().endswith(".dll"): # Check if the file is a .dll file
src_file = os.path.join(cudnn_src, file)
dest_file = os.path.join(cudnn_dest, file)
# if dest file exists, check if it's different
@ -110,16 +113,18 @@ def install_kohya_ss_torch2(headless: bool = False):
setup_common.check_repo_version()
if not setup_common.check_python_version():
exit(1)
setup_common.update_submodule()
setup_common.install("pip")
setup_common.install_requirements(
"requirements_windows_torch2.txt", check_no_verify_flag=False
)
setup_common.configure_accelerate(run_accelerate=not headless) # False if headless is True and vice versa
setup_common.configure_accelerate(
run_accelerate=not headless
) # False if headless is True and vice versa
def install_bitsandbytes_0_35_0():
@ -147,6 +152,7 @@ def install_bitsandbytes_0_41_1():
reinstall=True,
)
def install_bitsandbytes_0_41_2():
log.info("Installing bitsandbytes 0.41.2...")
setup_common.install(
@ -155,21 +161,34 @@ def install_bitsandbytes_0_41_2():
reinstall=True,
)
def install_triton_2_1_0():
log.info("Installing triton 2.1.0...")
setup_common.install(
"--upgrade https://huggingface.co/Rodeszones/CogVLM-grounding-generalist-hf-quant4/resolve/main/triton-2.1.0-cp310-cp310-win_amd64.whl?download=true",
"triton 2.1.0",
reinstall=True,
)
def main_menu(headless: bool = False):
if headless:
install_kohya_ss_torch2(headless=headless)
else:
setup_common.clear_screen()
while True:
print("\nKohya_ss GUI setup menu:\n")
print("1. Install kohya_ss gui")
print("2. (Optional) Install cudnn files (if you want to use latest supported cudnn version)")
print("3. (Optional) Install specific bitsandbytes versions")
print("4. (Optional) Manually configure accelerate")
print("5. (Optional) Start Kohya_ss GUI in browser")
print("6. Quit")
print("\nKohya_ss setup menu:\n")
print("1. Install kohya_ss GUI")
print(
"2. (Optional) Install CuDNN files (to use the latest supported CuDNN version)"
)
print("3. (Optional) Install Triton 2.1.0 for Windows")
print("4. (Optional) Install specific version of bitsandbytes")
print("5. (Optional) Manually configure Accelerate")
print("6. (Optional) Launch Kohya_ss GUI in browser")
print("7. Exit Setup")
choice = input("\nEnter your choice: ")
choice = input("\nSelect an option: ")
print("")
if choice == "1":
@ -177,22 +196,25 @@ def main_menu(headless: bool = False):
elif choice == "2":
cudnn_install()
elif choice == "3":
install_triton_2_1_0()
elif choice == "4":
while True:
print("1. (Optional) Force installation of bitsandbytes 0.35.0")
print("\nBitsandBytes Installation Menu:")
print("1. Force install Bitsandbytes 0.35.0")
print(
"2. (Optional) Force installation of bitsandbytes 0.40.1 for new optimizer options support and pre-bugfix results"
"2. Force install Bitsandbytes 0.40.1 (supports new optimizer options, pre-bugfix results)"
)
print(
"3. (Optional) Force installation of bitsandbytes 0.41.1 for new optimizer options support"
"3. Force installation Bitsandbytes 0.41.1 (supports new optimizer options)"
)
print(
"4. (Recommended) Force installation of bitsandbytes 0.41.2 for new optimizer options support"
"4. (Recommended) Force install Bitsandbytes 0.41.2 (supports new optimizer options)"
)
print(
"5. (Danger) Install bitsandbytes-windows (this package has been reported to cause issues for most... avoid...)"
"5. (Warning) Install bitsandbytes-windows (may cause issues, use with caution)"
)
print("6. Exit")
choice_torch = input("\nEnter your choice: ")
print("6. Return to Previous Menu:")
choice_torch = input("\nSelect an option: ")
print("")
if choice_torch == "1":
@ -215,29 +237,29 @@ def main_menu(headless: bool = False):
elif choice_torch == "6":
break
else:
print("Invalid choice. Please enter a number between 1-3.")
elif choice == "4":
setup_common.run_cmd("accelerate config")
print("Invalid choice. Please chose an option between 1-6.")
elif choice == "5":
setup_common.run_cmd("accelerate config")
elif choice == "6":
subprocess.Popen(
"start cmd /k .\gui.bat --inbrowser", shell=True
) # /k keep the terminal open on quit. /c would close the terminal instead
elif choice == "6":
print("Quitting the program.")
elif choice == "7":
print("Exiting setup.")
break
else:
print("Invalid choice. Please enter a number between 1-5.")
print("Invalid selection. Please choose an option between 1-7.")
if __name__ == "__main__":
setup_common.ensure_base_requirements()
setup_common.setup_logging()
# Setup argument parser
parser = argparse.ArgumentParser(description="Your Script Description")
parser.add_argument('--headless', action='store_true', help='Run in headless mode')
parser.add_argument("--headless", action="store_true", help="Run in headless mode")
# Parse arguments
args = parser.parse_args()
main_menu(headless=args.headless)

131
test/config/Diag-OFT-AdamW8bit-toml.json Normal file
View File

@ -0,0 +1,131 @@
{
"LoRA_type": "LyCORIS/Diag-OFT",
"LyCORIS_preset": "full",
"adaptive_noise_scale": 0,
"additional_parameters": "",
"block_alphas": "",
"block_dims": "",
"block_lr_zero_threshold": "",
"bucket_no_upscale": true,
"bucket_reso_steps": 64,
"bypass_mode": true,
"cache_latents": true,
"cache_latents_to_disk": false,
"caption_dropout_every_n_epochs": 0.0,
"caption_dropout_rate": 0.05,
"caption_extension": "",
"clip_skip": 2,
"color_aug": false,
"constrain": 0.0,
"conv_alpha": 4,
"conv_block_alphas": "",
"conv_block_dims": "",
"conv_dim": 8,
"dataset_config": "D:/kohya_ss/test/config/dataset.toml",
"debiased_estimation_loss": false,
"decompose_both": false,
"dim_from_weights": false,
"down_lr_weight": "",
"enable_bucket": true,
"epoch": 1,
"factor": -1,
"flip_aug": false,
"fp8_base": false,
"full_bf16": false,
"full_fp16": false,
"gpu_ids": "",
"gradient_accumulation_steps": 1,
"gradient_checkpointing": false,
"keep_tokens": "0",
"learning_rate": 0.0005,
"log_tracker_config": "",
"log_tracker_name": "",
"logging_dir": "./test/logs",
"lora_network_weights": "",
"lr_scheduler": "constant",
"lr_scheduler_args": "",
"lr_scheduler_num_cycles": "",
"lr_scheduler_power": "",
"lr_warmup": 0,
"max_bucket_reso": 2048,
"max_data_loader_n_workers": "0",
"max_grad_norm": 1,
"max_resolution": "512,512",
"max_timestep": 1000,
"max_token_length": "75",
"max_train_epochs": "",
"max_train_steps": "",
"mem_eff_attn": false,
"mid_lr_weight": "",
"min_bucket_reso": 256,
"min_snr_gamma": 0,
"min_timestep": 0,
"mixed_precision": "bf16",
"model_list": "runwayml/stable-diffusion-v1-5",
"module_dropout": 0,
"multi_gpu": false,
"multires_noise_discount": 0,
"multires_noise_iterations": 0,
"network_alpha": 16,
"network_dim": 32,
"network_dropout": 0,
"noise_offset": 0.05,
"noise_offset_type": "Original",
"num_cpu_threads_per_process": 2,
"num_machines": 1,
"num_processes": 1,
"optimizer": "AdamW8bit",
"optimizer_args": "",
"output_dir": "./test/output",
"output_name": "Diag-OFT-AdamW8bit-toml",
"persistent_data_loader_workers": false,
"pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
"prior_loss_weight": 1.0,
"random_crop": false,
"rank_dropout": 0,
"rank_dropout_scale": false,
"reg_data_dir": "",
"rescaled": false,
"resume": "",
"sample_every_n_epochs": 0,
"sample_every_n_steps": 25,
"sample_prompts": "a painting of a gas mask , by darius kawasaki",
"sample_sampler": "euler_a",
"save_every_n_epochs": 1,
"save_every_n_steps": 0,
"save_last_n_steps": 0,
"save_last_n_steps_state": 0,
"save_model_as": "safetensors",
"save_precision": "fp16",
"save_state": false,
"scale_v_pred_loss_like_noise_pred": false,
"scale_weight_norms": 0,
"sdxl": false,
"sdxl_cache_text_encoder_outputs": false,
"sdxl_no_half_vae": true,
"seed": "1234",
"shuffle_caption": false,
"stop_text_encoder_training": 0,
"text_encoder_lr": 0.0,
"train_batch_size": 4,
"train_data_dir": "",
"train_norm": false,
"train_on_input": true,
"training_comment": "",
"unet_lr": 0.0,
"unit": 1,
"up_lr_weight": "",
"use_cp": false,
"use_scalar": false,
"use_tucker": false,
"use_wandb": false,
"v2": false,
"v_parameterization": false,
"v_pred_like_loss": 0,
"vae": "",
"vae_batch_size": 0,
"wandb_api_key": "",
"wandb_run_name": "",
"weighted_captions": false,
"xformers": "xformers"
}

132
test/config/LoKR-AdamW8bit-toml.json Normal file
View File

@ -0,0 +1,132 @@
{
"LoRA_type": "LyCORIS/LoKr",
"LyCORIS_preset": "full",
"adaptive_noise_scale": 0,
"additional_parameters": "--lr_scheduler_type \"CosineAnnealingLR\" --lr_scheduler_args \"T_max=1000\" \"eta_min=0e-0\"",
"block_alphas": "",
"block_dims": "",
"block_lr_zero_threshold": "",
"bucket_no_upscale": true,
"bucket_reso_steps": 1,
"bypass_mode": false,
"cache_latents": true,
"cache_latents_to_disk": true,
"caption_dropout_every_n_epochs": 0.0,
"caption_dropout_rate": 0.1,
"caption_extension": ".txt",
"clip_skip": "1",
"color_aug": false,
"constrain": 0.0,
"conv_alpha": 1,
"conv_block_alphas": "",
"conv_block_dims": "",
"conv_dim": 100000,
"dataset_config": "D:/kohya_ss/test/config/dataset.toml",
"debiased_estimation_loss": false,
"decompose_both": false,
"dim_from_weights": false,
"dora_wd": false,
"down_lr_weight": "",
"enable_bucket": true,
"epoch": 150,
"factor": 6,
"flip_aug": false,
"fp8_base": false,
"full_bf16": false,
"full_fp16": false,
"gpu_ids": "",
"gradient_accumulation_steps": 1,
"gradient_checkpointing": false,
"keep_tokens": 1,
"learning_rate": 1.0,
"log_tracker_config": "",
"log_tracker_name": "",
"logging_dir": "./test/logs",
"lora_network_weights": "",
"lr_scheduler": "cosine",
"lr_scheduler_args": "",
"lr_scheduler_num_cycles": "",
"lr_scheduler_power": "",
"lr_warmup": 0,
"max_bucket_reso": 2048,
"max_data_loader_n_workers": "0",
"max_grad_norm": 1,
"max_resolution": "512,512",
"max_timestep": 1000,
"max_token_length": "75",
"max_train_epochs": "",
"max_train_steps": "",
"mem_eff_attn": false,
"mid_lr_weight": "",
"min_bucket_reso": 256,
"min_snr_gamma": 5,
"min_timestep": 0,
"mixed_precision": "bf16",
"model_list": "custom",
"module_dropout": 0,
"multi_gpu": false,
"multires_noise_discount": 0.1,
"multires_noise_iterations": 6,
"network_alpha": 1,
"network_dim": 100000,
"network_dropout": 0,
"noise_offset": 0,
"noise_offset_type": "Multires",
"num_cpu_threads_per_process": 2,
"num_machines": 1,
"num_processes": 1,
"optimizer": "Prodigy",
"optimizer_args": "\"d0=1e-5\" \"d_coef=1.0\" \"weight_decay=0.4\" \"decouple=True\" \"safeguard_warmup=True\" \"use_bias_correction=True\"",
"output_dir": "./test/output",
"output_name": "LoKR-AdamW8bit-toml",
"persistent_data_loader_workers": false,
"pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
"prior_loss_weight": 1.0,
"random_crop": false,
"rank_dropout": 0,
"rank_dropout_scale": false,
"reg_data_dir": "",
"rescaled": false,
"resume": "",
"sample_every_n_epochs": 0,
"sample_every_n_steps": 25,
"sample_prompts": "a painting of a gas mask , by darius kawasaki",
"sample_sampler": "euler_a",
"save_every_n_epochs": 15,
"save_every_n_steps": 0,
"save_last_n_steps": 0,
"save_last_n_steps_state": 0,
"save_model_as": "safetensors",
"save_precision": "bf16",
"save_state": false,
"scale_v_pred_loss_like_noise_pred": false,
"scale_weight_norms": 0,
"sdxl": false,
"sdxl_cache_text_encoder_outputs": false,
"sdxl_no_half_vae": true,
"seed": "",
"shuffle_caption": true,
"stop_text_encoder_training": 0,
"text_encoder_lr": 1.0,
"train_batch_size": 2,
"train_data_dir": "",
"train_norm": false,
"train_on_input": false,
"training_comment": "KoopaTroopa",
"unet_lr": 1.0,
"unit": 1,
"up_lr_weight": "",
"use_cp": false,
"use_scalar": false,
"use_tucker": false,
"use_wandb": false,
"v2": false,
"v_parameterization": false,
"v_pred_like_loss": 0,
"vae": "",
"vae_batch_size": 0,
"wandb_api_key": "",
"wandb_run_name": "",
"weighted_captions": false,
"xformers": "xformers"
}

96
test/config/TI-AdamW8bit-toml.json Normal file
View File

@ -0,0 +1,96 @@
{
"adaptive_noise_scale": 0,
"additional_parameters": "",
"bucket_no_upscale": true,
"bucket_reso_steps": 1,
"cache_latents": true,
"cache_latents_to_disk": false,
"caption_dropout_every_n_epochs": 0.0,
"caption_dropout_rate": 0.05,
"caption_extension": "",
"clip_skip": 2,
"color_aug": false,
"dataset_config": "D:/kohya_ss/test/config/dataset.toml",
"enable_bucket": true,
"epoch": 4,
"flip_aug": false,
"full_fp16": false,
"gpu_ids": "",
"gradient_accumulation_steps": 1,
"gradient_checkpointing": false,
"init_word": "*",
"keep_tokens": "0",
"learning_rate": 0.0001,
"log_tracker_config": "",
"log_tracker_name": "",
"logging_dir": "./test/logs",
"lr_scheduler": "cosine",
"lr_scheduler_args": "",
"lr_scheduler_num_cycles": "",
"lr_scheduler_power": "",
"lr_warmup": 0,
"max_bucket_reso": 2048,
"max_data_loader_n_workers": "0",
"max_resolution": "512,512",
"max_timestep": 1000,
"max_token_length": "75",
"max_train_epochs": "",
"max_train_steps": "80",
"mem_eff_attn": false,
"min_bucket_reso": 256,
"min_snr_gamma": 10,
"min_timestep": 0,
"mixed_precision": "bf16",
"model_list": "runwayml/stable-diffusion-v1-5",
"multi_gpu": false,
"multires_noise_discount": 0.2,
"multires_noise_iterations": 8,
"no_token_padding": false,
"noise_offset": 0.05,
"noise_offset_type": "Multires",
"num_cpu_threads_per_process": 2,
"num_machines": 1,
"num_processes": 1,
"num_vectors_per_token": 8,
"optimizer": "AdamW8bit",
"optimizer_args": "",
"output_dir": "./test/output",
"output_name": "TI-Adamw8bit-toml",
"persistent_data_loader_workers": false,
"pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
"prior_loss_weight": 1.0,
"random_crop": false,
"reg_data_dir": "",
"resume": "",
"sample_every_n_epochs": 0,
"sample_every_n_steps": 20,
"sample_prompts": "a painting of man wearing a gas mask , by darius kawasaki",
"sample_sampler": "euler_a",
"save_every_n_epochs": 1,
"save_every_n_steps": 0,
"save_last_n_steps": 0,
"save_last_n_steps_state": 0,
"save_model_as": "safetensors",
"save_precision": "fp16",
"save_state": false,
"scale_v_pred_loss_like_noise_pred": false,
"sdxl": false,
"sdxl_no_half_vae": false,
"seed": "1234",
"shuffle_caption": false,
"stop_text_encoder_training": 0,
"template": "style template",
"token_string": "zxc",
"train_batch_size": 4,
"train_data_dir": "",
"use_wandb": false,
"v2": false,
"v_parameterization": false,
"v_pred_like_loss": 0,
"vae": "",
"vae_batch_size": 0,
"wandb_api_key": "",
"wandb_run_name": "",
"weights": "",
"xformers": "xformers"
}

96
test/config/TI-AdamW8bit.json Normal file
View File

@ -0,0 +1,96 @@
{
"adaptive_noise_scale": 0,
"additional_parameters": "",
"bucket_no_upscale": true,
"bucket_reso_steps": 1,
"cache_latents": true,
"cache_latents_to_disk": false,
"caption_dropout_every_n_epochs": 0.0,
"caption_dropout_rate": 0.05,
"caption_extension": "",
"clip_skip": 2,
"color_aug": false,
"dataset_config": "",
"enable_bucket": true,
"epoch": 4,
"flip_aug": false,
"full_fp16": false,
"gpu_ids": "",
"gradient_accumulation_steps": 1,
"gradient_checkpointing": false,
"init_word": "*",
"keep_tokens": "0",
"learning_rate": 0.0001,
"log_tracker_config": "",
"log_tracker_name": "",
"logging_dir": "./test/logs",
"lr_scheduler": "cosine",
"lr_scheduler_args": "",
"lr_scheduler_num_cycles": "",
"lr_scheduler_power": "",
"lr_warmup": 0,
"max_bucket_reso": 2048,
"max_data_loader_n_workers": "0",
"max_resolution": "512,512",
"max_timestep": 1000,
"max_token_length": "75",
"max_train_epochs": "",
"max_train_steps": "",
"mem_eff_attn": false,
"min_bucket_reso": 256,
"min_snr_gamma": 10,
"min_timestep": 0,
"mixed_precision": "bf16",
"model_list": "runwayml/stable-diffusion-v1-5",
"multi_gpu": false,
"multires_noise_discount": 0.2,
"multires_noise_iterations": 8,
"no_token_padding": false,
"noise_offset": 0.05,
"noise_offset_type": "Multires",
"num_cpu_threads_per_process": 2,
"num_machines": 1,
"num_processes": 1,
"num_vectors_per_token": 8,
"optimizer": "AdamW8bit",
"optimizer_args": "",
"output_dir": "./test/output",
"output_name": "TI-Adamw8bit",
"persistent_data_loader_workers": false,
"pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
"prior_loss_weight": 1.0,
"random_crop": false,
"reg_data_dir": "",
"resume": "",
"sample_every_n_epochs": 0,
"sample_every_n_steps": 20,
"sample_prompts": "a painting of man wearing a gas mask , by darius kawasaki",
"sample_sampler": "euler_a",
"save_every_n_epochs": 1,
"save_every_n_steps": 0,
"save_last_n_steps": 0,
"save_last_n_steps_state": 0,
"save_model_as": "safetensors",
"save_precision": "fp16",
"save_state": false,
"scale_v_pred_loss_like_noise_pred": false,
"sdxl": false,
"sdxl_no_half_vae": false,
"seed": "1234",
"shuffle_caption": false,
"stop_text_encoder_training": 0,
"template": "style template",
"token_string": "zxc",
"train_batch_size": 4,
"train_data_dir": "./test/img",
"use_wandb": false,
"v2": false,
"v_parameterization": false,
"v_pred_like_loss": 0,
"vae": "",
"vae_batch_size": 0,
"wandb_api_key": "",
"wandb_run_name": "",
"weights": "",
"xformers": "xformers"
}

14
test/config/dataset-finetune.toml Normal file
View File

@ -0,0 +1,14 @@
[[datasets]]
resolution = 512
batch_size = 4
keep_tokens = 1
enable_bucket = true
min_bucket_reso = 64
max_bucket_reso = 1024
bucket_reso_steps = 32
bucket_no_upscale = false
[[datasets.subsets]]
image_dir = '.\test\img\10_darius kawasaki person'
num_repeats = 10
metadata_file = '.\test\config\meta-1_lat.json'

15
test/config/dataset.toml Normal file
View File

@ -0,0 +1,15 @@
[[datasets]]
resolution = 512
batch_size = 4
keep_tokens = 1
enable_bucket = true
min_bucket_reso = 64
max_bucket_reso = 1024
bucket_reso_steps = 32
bucket_no_upscale = true
[[datasets.subsets]]
image_dir = '.\test\img\10_darius kawasaki person'
num_repeats = 10
class_tokens = 'darius kawasaki person'
caption_extension = '.txt'

95
test/config/dreambooth-AdamW8bit-toml.json Normal file
View File

@ -0,0 +1,95 @@
{
"adaptive_noise_scale": 0,
"additional_parameters": "",
"bucket_no_upscale": true,
"bucket_reso_steps": 64,
"cache_latents": true,
"cache_latents_to_disk": false,
"caption_dropout_every_n_epochs": 0.0,
"caption_dropout_rate": 0.05,
"caption_extension": "",
"clip_skip": 2,
"color_aug": false,
"dataset_config": "D:/kohya_ss/test/config/dataset.toml",
"enable_bucket": true,
"epoch": 1,
"flip_aug": false,
"full_bf16": false,
"full_fp16": false,
"gpu_ids": "",
"gradient_accumulation_steps": 1,
"gradient_checkpointing": false,
"keep_tokens": "0",
"learning_rate": 5e-05,
"learning_rate_te": 1e-05,
"learning_rate_te1": 1e-05,
"learning_rate_te2": 1e-05,
"log_tracker_config": "",
"log_tracker_name": "",
"logging_dir": "./test/logs",
"lr_scheduler": "constant",
"lr_scheduler_args": "",
"lr_scheduler_num_cycles": "",
"lr_scheduler_power": "",
"lr_warmup": 0,
"max_bucket_reso": 2048,
"max_data_loader_n_workers": "0",
"max_resolution": "512,512",
"max_timestep": 1000,
"max_token_length": "75",
"max_train_epochs": "",
"max_train_steps": "",
"mem_eff_attn": false,
"min_bucket_reso": 256,
"min_snr_gamma": 0,
"min_timestep": 0,
"mixed_precision": "bf16",
"model_list": "runwayml/stable-diffusion-v1-5",
"multi_gpu": false,
"multires_noise_discount": 0,
"multires_noise_iterations": 0,
"no_token_padding": false,
"noise_offset": 0.05,
"noise_offset_type": "Original",
"num_cpu_threads_per_process": 2,
"num_machines": 1,
"num_processes": 1,
"optimizer": "AdamW8bit",
"optimizer_args": "",
"output_dir": "./test/output",
"output_name": "db-AdamW8bit-toml",
"persistent_data_loader_workers": false,
"pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
"prior_loss_weight": 1.0,
"random_crop": false,
"reg_data_dir": "",
"resume": "",
"sample_every_n_epochs": 0,
"sample_every_n_steps": 25,
"sample_prompts": "a painting of a gas mask , by darius kawasaki",
"sample_sampler": "euler_a",
"save_every_n_epochs": 1,
"save_every_n_steps": 0,
"save_last_n_steps": 0,
"save_last_n_steps_state": 0,
"save_model_as": "safetensors",
"save_precision": "fp16",
"save_state": false,
"scale_v_pred_loss_like_noise_pred": false,
"sdxl": false,
"seed": "1234",
"shuffle_caption": false,
"stop_text_encoder_training": 0,
"train_batch_size": 4,
"train_data_dir": "",
"use_wandb": false,
"v2": false,
"v_parameterization": false,
"v_pred_like_loss": 0,
"vae": "",
"vae_batch_size": 0,
"wandb_api_key": "",
"wandb_run_name": "",
"weighted_captions": false,
"xformers": "xformers"
}

46
test/config/dreambooth-AdamW8bit.json
View File

@ -1,11 +1,6 @@
{
"LoRA_type": "Standard",
"LyCORIS_preset": "full",
"adaptive_noise_scale": 0,
"additional_parameters": "",
"block_alphas": "",
"block_dims": "",
"block_lr_zero_threshold": "",
"bucket_no_upscale": true,
"bucket_reso_steps": 64,
"cache_latents": true,
@ -15,20 +10,10 @@
"caption_extension": "",
"clip_skip": 2,
"color_aug": false,
"constrain": 0.0,
"conv_alpha": 1,
"conv_block_alphas": "",
"conv_block_dims": "",
"conv_dim": 1,
"debiased_estimation_loss": false,
"decompose_both": false,
"dim_from_weights": false,
"down_lr_weight": "",
"dataset_config": "",
"enable_bucket": true,
"epoch": 1,
"factor": -1,
"flip_aug": false,
"fp8_base": false,
"full_bf16": false,
"full_fp16": false,
"gpu_ids": "",
@ -36,10 +21,12 @@
"gradient_checkpointing": false,
"keep_tokens": "0",
"learning_rate": 5e-05,
"learning_rate_te": 1e-05,
"learning_rate_te1": 1e-05,
"learning_rate_te2": 1e-05,
"log_tracker_config": "",
"log_tracker_name": "",
"logging_dir": "./test/logs",
"lora_network_weights": "",
"lr_scheduler": "constant",
"lr_scheduler_args": "",
"lr_scheduler_num_cycles": "",
@ -47,26 +34,21 @@
"lr_warmup": 0,
"max_bucket_reso": 2048,
"max_data_loader_n_workers": "0",
"max_grad_norm": 1,
"max_resolution": "512,512",
"max_timestep": 1000,
"max_token_length": "75",
"max_train_epochs": "",
"max_train_steps": "",
"mem_eff_attn": false,
"mid_lr_weight": "",
"min_bucket_reso": 256,
"min_snr_gamma": 0,
"min_timestep": 0,
"mixed_precision": "bf16",
"model_list": "runwayml/stable-diffusion-v1-5",
"module_dropout": 0,
"multi_gpu": false,
"multires_noise_discount": 0,
"multires_noise_iterations": 0,
"network_alpha": 1,
"network_dim": 8,
"network_dropout": 0,
"no_token_padding": false,
"noise_offset": 0.05,
"noise_offset_type": "Original",
"num_cpu_threads_per_process": 2,
@ -80,10 +62,7 @@
"pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
"prior_loss_weight": 1.0,
"random_crop": false,
"rank_dropout": 0,
"rank_dropout_scale": false,
"reg_data_dir": "",
"rescaled": false,
"resume": "",
"sample_every_n_epochs": 0,
"sample_every_n_steps": 25,
@ -97,30 +76,17 @@
"save_precision": "fp16",
"save_state": false,
"scale_v_pred_loss_like_noise_pred": false,
"scale_weight_norms": 0,
"sdxl": false,
"sdxl_cache_text_encoder_outputs": false,
"sdxl_no_half_vae": true,
"seed": "1234",
"shuffle_caption": false,
"stop_text_encoder_training": 0,
"text_encoder_lr": 0.0,
"train_batch_size": 4,
"train_data_dir": "./test/img",
"train_norm": false,
"train_on_input": true,
"training_comment": "",
"unet_lr": 0.0,
"unit": 1,
"up_lr_weight": "",
"use_cp": false,
"use_scalar": false,
"use_tucker": false,
"use_wandb": false,
"v2": false,
"v_parameterization": false,
"v_pred_like_loss": 0,
"vae": "",
"vae": "stabilityai/vae",
"vae_batch_size": 0,
"wandb_api_key": "",
"wandb_run_name": "",

100
test/config/finetune-AdamW-toml.json Normal file
View File

@ -0,0 +1,100 @@
{
"adaptive_noise_scale": 0,
"additional_parameters": "",
"batch_size": "8",
"block_lr": "",
"bucket_no_upscale": false,
"bucket_reso_steps": 1,
"cache_latents": true,
"cache_latents_to_disk": false,
"caption_dropout_every_n_epochs": 0.0,
"caption_dropout_rate": 0,
"caption_extension": ".txt",
"caption_metadata_filename": "meta-1_cap.json",
"clip_skip": 1,
"color_aug": false,
"create_buckets": false,
"create_caption": true,
"dataset_config": "D:/kohya_ss/test/config/dataset-finetune.toml",
"dataset_repeats": "50",
"epoch": 2,
"flip_aug": false,
"full_bf16": false,
"full_fp16": false,
"full_path": true,
"gpu_ids": "",
"gradient_accumulation_steps": 1.0,
"gradient_checkpointing": false,
"image_folder": "",
"keep_tokens": 0,
"latent_metadata_filename": "meta-1_lat.json",
"learning_rate": 1e-05,
"learning_rate_te": 5e-06,
"learning_rate_te1": 5e-06,
"learning_rate_te2": 0.0,
"log_tracker_config": "",
"log_tracker_name": "",
"logging_dir": "./test/ft",
"lr_scheduler": "cosine_with_restarts",
"lr_scheduler_args": "",
"lr_warmup": 10,
"max_bucket_reso": "1024",
"max_data_loader_n_workers": "0",
"max_resolution": "512,512",
"max_timestep": 1000,
"max_token_length": "75",
"max_train_epochs": "",
"max_train_steps": "20",
"mem_eff_attn": false,
"min_bucket_reso": "256",
"min_snr_gamma": 0,
"min_timestep": 0,
"mixed_precision": "bf16",
"model_list": "runwayml/stable-diffusion-v1-5",
"multi_gpu": false,
"multires_noise_discount": 0,
"multires_noise_iterations": 0,
"noise_offset": 0,
"noise_offset_type": "Original",
"num_cpu_threads_per_process": 2,
"num_machines": 1,
"num_processes": 1,
"optimizer": "AdamW",
"optimizer_args": "",
"output_dir": "./test/output",
"output_name": "test_ft-toml",
"persistent_data_loader_workers": false,
"pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
"random_crop": false,
"resume": "",
"sample_every_n_epochs": 0,
"sample_every_n_steps": 0,
"sample_prompts": "",
"sample_sampler": "euler_a",
"save_every_n_epochs": 1,
"save_every_n_steps": 0,
"save_last_n_steps": 0,
"save_last_n_steps_state": 0,
"save_model_as": "safetensors",
"save_precision": "bf16",
"save_state": false,
"scale_v_pred_loss_like_noise_pred": false,
"sdxl_cache_text_encoder_outputs": false,
"sdxl_checkbox": false,
"sdxl_no_half_vae": false,
"seed": "1234",
"shuffle_caption": false,
"train_batch_size": 4,
"train_dir": "./test",
"train_text_encoder": true,
"use_latent_files": "No",
"use_wandb": false,
"v2": false,
"v_parameterization": false,
"v_pred_like_loss": 0,
"vae_batch_size": 0,
"wandb_api_key": "",
"wandb_run_name": "",
"weighted_captions": false,
"xformers": "xformers"
}

9
test/config/finetune-AdamW.json
View File

@ -15,12 +15,14 @@
"color_aug": false,
"create_buckets": false,
"create_caption": true,
"dataset_config": "",
"dataset_repeats": "50",
"epoch": 2,
"flip_aug": false,
"full_bf16": false,
"full_fp16": false,
"full_path": true,
"gpu_ids": "",
"gradient_accumulation_steps": 1.0,
"gradient_checkpointing": false,
"image_folder": ".\\test\\img\\10_darius kawasaki person",
@ -30,6 +32,8 @@
"learning_rate_te": 5e-06,
"learning_rate_te1": 5e-06,
"learning_rate_te2": 0.0,
"log_tracker_config": "",
"log_tracker_name": "",
"logging_dir": "./test/ft",
"lr_scheduler": "cosine_with_restarts",
"lr_scheduler_args": "",
@ -40,17 +44,21 @@
"max_timestep": 1000,
"max_token_length": "75",
"max_train_epochs": "",
"max_train_steps": "",
"mem_eff_attn": false,
"min_bucket_reso": "256",
"min_snr_gamma": 0,
"min_timestep": 0,
"mixed_precision": "bf16",
"model_list": "runwayml/stable-diffusion-v1-5",
"multi_gpu": false,
"multires_noise_discount": 0,
"multires_noise_iterations": 0,
"noise_offset": 0,
"noise_offset_type": "Original",
"num_cpu_threads_per_process": 2,
"num_machines": 1,
"num_processes": 1,
"optimizer": "AdamW",
"optimizer_args": "",
"output_dir": "./test/output",
@ -86,6 +94,7 @@
"v_pred_like_loss": 0,
"vae_batch_size": 0,
"wandb_api_key": "",
"wandb_run_name": "",
"weighted_captions": false,
"xformers": "xformers"
}

130
test/config/locon-AdamW8bit-toml.json Normal file
View File

@ -0,0 +1,130 @@
{
"LoRA_type": "Standard",
"LyCORIS_preset": "full",
"adaptive_noise_scale": 0,
"additional_parameters": "",
"block_alphas": "",
"block_dims": "",
"block_lr_zero_threshold": "",
"bucket_no_upscale": true,
"bucket_reso_steps": 64,
"cache_latents": true,
"cache_latents_to_disk": false,
"caption_dropout_every_n_epochs": 0.0,
"caption_dropout_rate": 0.05,
"caption_extension": "",
"clip_skip": 2,
"color_aug": false,
"constrain": 0.0,
"conv_alpha": 1,
"conv_block_alphas": "",
"conv_block_dims": "",
"conv_dim": 1,
"dataset_config": "D:/kohya_ss/test/config/dataset.toml",
"debiased_estimation_loss": false,
"decompose_both": false,
"dim_from_weights": false,
"down_lr_weight": "",
"enable_bucket": true,
"epoch": 1,
"factor": -1,
"flip_aug": false,
"fp8_base": false,
"full_bf16": false,
"full_fp16": false,
"gpu_ids": "",
"gradient_accumulation_steps": 1,
"gradient_checkpointing": false,
"keep_tokens": "0",
"learning_rate": 0.0005,
"log_tracker_config": "",
"log_tracker_name": "",
"logging_dir": "./test/logs",
"lora_network_weights": "",
"lr_scheduler": "constant",
"lr_scheduler_args": "",
"lr_scheduler_num_cycles": "",
"lr_scheduler_power": "",
"lr_warmup": 0,
"max_bucket_reso": 2048,
"max_data_loader_n_workers": "0",
"max_grad_norm": 1,
"max_resolution": "512,512",
"max_timestep": 1000,
"max_token_length": "75",
"max_train_epochs": "",
"max_train_steps": "",
"mem_eff_attn": false,
"mid_lr_weight": "",
"min_bucket_reso": 256,
"min_snr_gamma": 0,
"min_timestep": 0,
"mixed_precision": "bf16",
"model_list": "runwayml/stable-diffusion-v1-5",
"module_dropout": 0,
"multi_gpu": false,
"multires_noise_discount": 0,
"multires_noise_iterations": 0,
"network_alpha": 1,
"network_dim": 8,
"network_dropout": 0,
"noise_offset": 0.05,
"noise_offset_type": "Original",
"num_cpu_threads_per_process": 2,
"num_machines": 1,
"num_processes": 1,
"optimizer": "AdamW8bit",
"optimizer_args": "",
"output_dir": "./test/output",
"output_name": "locon-AdamW8bit-toml",
"persistent_data_loader_workers": false,
"pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
"prior_loss_weight": 1.0,
"random_crop": false,
"rank_dropout": 0,
"rank_dropout_scale": false,
"reg_data_dir": "",
"rescaled": false,
"resume": "",
"sample_every_n_epochs": 0,
"sample_every_n_steps": 25,
"sample_prompts": "a painting of a gas mask , by darius kawasaki",
"sample_sampler": "euler_a",
"save_every_n_epochs": 1,
"save_every_n_steps": 0,
"save_last_n_steps": 0,
"save_last_n_steps_state": 0,
"save_model_as": "safetensors",
"save_precision": "fp16",
"save_state": false,
"scale_v_pred_loss_like_noise_pred": false,
"scale_weight_norms": 0,
"sdxl": false,
"sdxl_cache_text_encoder_outputs": false,
"sdxl_no_half_vae": true,
"seed": "1234",
"shuffle_caption": false,
"stop_text_encoder_training": 0,
"text_encoder_lr": 0.0,
"train_batch_size": 4,
"train_data_dir": "",
"train_norm": false,
"train_on_input": true,
"training_comment": "",
"unet_lr": 0.0,
"unit": 1,
"up_lr_weight": "",
"use_cp": false,
"use_scalar": false,
"use_tucker": false,
"use_wandb": false,
"v2": false,
"v_parameterization": false,
"v_pred_like_loss": 0,
"vae": "",
"vae_batch_size": 0,
"wandb_api_key": "",
"wandb_run_name": "",
"weighted_captions": false,
"xformers": "xformers"
}

58
test/config/meta-1_lat.json Normal file
View File

@ -0,0 +1,58 @@
{
"test\\img\\10_darius kawasaki person\\Dariusz_Zawadzki.jpg": {
"caption": "a painting of a steam punk skull with a gas mask , by darius kawasaki",
"train_resolution": [
1024,
1024
]
},
"test\\img\\10_darius kawasaki person\\Dariusz_Zawadzki_2.jpg": {
"caption": "a painting of a man with a skull on his head , by darius kawasaki",
"train_resolution": [
1024,
1024
]
},
"test\\img\\10_darius kawasaki person\\Dariusz_Zawadzki_3.jpg": {
"caption": "a painting of a woman with a helmet on her head , by darius kawasaki",
"train_resolution": [
1024,
1024
]
},
"test\\img\\10_darius kawasaki person\\Dariusz_Zawadzki_4.jpg": {
"caption": "a painting of a horned man with a goat head , by darius kawasaki",
"train_resolution": [
1024,
1024
]
},
"test\\img\\10_darius kawasaki person\\Dariusz_Zawadzki_5.jpg": {
"caption": "a painting of a man playing a piano , by darius kawasaki",
"train_resolution": [
1024,
1024
]
},
"test\\img\\10_darius kawasaki person\\Dariusz_Zawadzki_6.jpg": {
"caption": "a painting of a robot sitting on a rock , by darius kawasaki",
"train_resolution": [
1024,
1024
]
},
"test\\img\\10_darius kawasaki person\\Dariusz_Zawadzki_7.jpg": {
"caption": "a painting of a soldier with a helmet on , by darius kawasaki",
"train_resolution": [
1024,
1024
]
},
"test\\img\\10_darius kawasaki person\\Dariusz_Zawadzki_8.jpg": {
"caption": "a painting of a giant crab with a large body , by darius kawasaki",
"train_resolution": [
1024,
1024
]
}
}