diff --git a/=13.7.1 b/=13.7.1 new file mode 100644 index 0000000..e624aa0 --- /dev/null +++ b/=13.7.1 @@ -0,0 +1,12 @@ +Requirement already satisfied: rich in ./venv/lib/python3.10/site-packages (13.7.0) +Collecting rich + Using cached rich-13.7.1-py3-none-any.whl (240 kB) +Requirement already satisfied: markdown-it-py>=2.2.0 in ./venv/lib/python3.10/site-packages (from rich) (3.0.0) +Requirement already satisfied: pygments<3.0.0,>=2.13.0 in ./venv/lib/python3.10/site-packages (from rich) (2.17.2) +Requirement already satisfied: mdurl~=0.1 in ./venv/lib/python3.10/site-packages (from markdown-it-py>=2.2.0->rich) (0.1.2) +Installing collected packages: rich + Attempting uninstall: rich + Found existing installation: rich 13.7.0 + Uninstalling rich-13.7.0: + Successfully uninstalled rich-13.7.0 +Successfully installed rich-13.7.1 diff --git a/assets/style.css b/assets/style.css index d4d77f6..38c22d4 100644 --- a/assets/style.css +++ b/assets/style.css @@ -35,13 +35,13 @@ } #myTensorButton { - background: radial-gradient(ellipse, #007bff, #00b0ff); + background: radial-gradient(ellipse, #3a99ff, #52c8ff); color: white; border: none; } #myTensorButtonStop { - background: radial-gradient(ellipse, #00b0ff, #007bff); + background: radial-gradient(ellipse, #52c8ff, #3a99ff); color: black; border: none; } \ No newline at end of file diff --git a/kohya_gui/basic_caption_gui.py b/kohya_gui/basic_caption_gui.py index c47442a..079519c 100644 --- a/kohya_gui/basic_caption_gui.py +++ b/kohya_gui/basic_caption_gui.py @@ -62,25 +62,31 @@ def caption_images( log.info(f"Captioning files in {images_dir} with {caption_text}...") # Build the command to run caption.py - run_cmd = rf'"{PYTHON}" "{scriptdir}/tools/caption.py"' - run_cmd += f' --caption_text="{caption_text}"' + run_cmd = [PYTHON, f"{scriptdir}/tools/caption.py"] + + # Add required arguments + run_cmd.append('--caption_text') + run_cmd.append(caption_text) # Add optional flags to the command if overwrite: - run_cmd += f" --overwrite" + run_cmd.append("--overwrite") if caption_ext: - run_cmd += f' --caption_file_ext="{caption_ext}"' + run_cmd.append('--caption_file_ext') + run_cmd.append(caption_ext) - run_cmd += f' "{images_dir}"' + # Add the directory containing the images + run_cmd.append(images_dir) # Log the command - log.info(run_cmd) + log.info(' '.join(run_cmd)) # Set the environment variable for the Python path env = os.environ.copy() env["PYTHONPATH"] = ( - rf"{scriptdir}{os.pathsep}{scriptdir}/tools{os.pathsep}{env.get('PYTHONPATH', '')}" + rf"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}" ) + env["TF_ENABLE_ONEDNN_OPTS"] = "0" # Run the command based on the operating system subprocess.run(run_cmd, env=env) diff --git a/kohya_gui/blip_caption_gui.py b/kohya_gui/blip_caption_gui.py index fc4ca16..29db10e 100644 --- a/kohya_gui/blip_caption_gui.py +++ b/kohya_gui/blip_caption_gui.py @@ -56,31 +56,49 @@ def caption_images( log.info(f"Captioning files in {train_data_dir}...") - # Construct the command to run - run_cmd = rf'"{PYTHON}" "{scriptdir}/sd-scripts/finetune/make_captions.py"' - run_cmd += f' --batch_size="{int(batch_size)}"' - run_cmd += f' --num_beams="{int(num_beams)}"' - run_cmd += f' --top_p="{top_p}"' - run_cmd += f' --max_length="{int(max_length)}"' - run_cmd += f' --min_length="{int(min_length)}"' - if beam_search: - run_cmd += f" --beam_search" - if caption_file_ext: - run_cmd += f' --caption_extension="{caption_file_ext}"' - run_cmd += f' "{train_data_dir}"' - run_cmd += f' --caption_weights="https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_large_caption.pth"' + # Construct the command to run make_captions.py + run_cmd = [PYTHON, f"{scriptdir}/sd-scripts/finetune/make_captions.py"] - log.info(run_cmd) + # Add required arguments + run_cmd.append('--batch_size') + run_cmd.append(str(batch_size)) + run_cmd.append('--num_beams') + run_cmd.append(str(num_beams)) + run_cmd.append('--top_p') + run_cmd.append(str(top_p)) + run_cmd.append('--max_length') + run_cmd.append(str(max_length)) + run_cmd.append('--min_length') + run_cmd.append(str(min_length)) + + # Add optional flags to the command + if beam_search: + run_cmd.append("--beam_search") + if caption_file_ext: + run_cmd.append('--caption_extension') + run_cmd.append(caption_file_ext) + + # Add the directory containing the training data + run_cmd.append(train_data_dir) + + # Add URL for caption model weights + run_cmd.append('--caption_weights') + run_cmd.append("https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_large_caption.pth") + + # Log the command + log.info(' '.join(run_cmd)) # Set up the environment env = os.environ.copy() env["PYTHONPATH"] = ( f"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}" ) + env["TF_ENABLE_ONEDNN_OPTS"] = "0" # Run the command in the sd-scripts folder context subprocess.run(run_cmd, env=env, cwd=f"{scriptdir}/sd-scripts") + # Add prefix and postfix add_pre_postfix( folder=train_data_dir, diff --git a/kohya_gui/class_accelerate_launch.py b/kohya_gui/class_accelerate_launch.py index 9f4bd73..39ffc45 100644 --- a/kohya_gui/class_accelerate_launch.py +++ b/kohya_gui/class_accelerate_launch.py @@ -1,5 +1,7 @@ import gradio as gr import os +import shlex + from .class_gui_config import KohyaSSGUIConfig @@ -75,46 +77,41 @@ class AccelerateLaunch: info="List of extra parameters to pass to accelerate launch", ) - def run_cmd(**kwargs): - run_cmd = "" + def run_cmd(run_cmd: list, **kwargs): + if ( + "extra_accelerate_launch_args" in kwargs + and kwargs.get("extra_accelerate_launch_args") != "" + ): + run_cmd.append(kwargs["extra_accelerate_launch_args"]) - if "extra_accelerate_launch_args" in kwargs: - extra_accelerate_launch_args = kwargs.get("extra_accelerate_launch_args") - if extra_accelerate_launch_args != "": - run_cmd += rf" {extra_accelerate_launch_args}" + if "gpu_ids" in kwargs and kwargs.get("gpu_ids") != "": + run_cmd.append("--gpu_ids") + run_cmd.append(shlex.quote(kwargs["gpu_ids"])) - if "gpu_ids" in kwargs: - gpu_ids = kwargs.get("gpu_ids") - if not gpu_ids == "": - run_cmd += f' --gpu_ids="{gpu_ids}"' + if "main_process_port" in kwargs and kwargs.get("main_process_port", 0) > 0: + run_cmd.append("--main_process_port") + run_cmd.append(str(int(kwargs["main_process_port"]))) - if "main_process_port" in kwargs: - main_process_port = kwargs.get("main_process_port") - if main_process_port > 0: - run_cmd += f' --main_process_port="{main_process_port}"' + if "mixed_precision" in kwargs and kwargs.get("mixed_precision"): + run_cmd.append("--mixed_precision") + run_cmd.append(shlex.quote(kwargs["mixed_precision"])) - if "mixed_precision" in kwargs: - run_cmd += rf' --mixed_precision="{kwargs.get("mixed_precision")}"' + if "multi_gpu" in kwargs and kwargs.get("multi_gpu"): + run_cmd.append("--multi_gpu") - if "multi_gpu" in kwargs: - if kwargs.get("multi_gpu"): - run_cmd += " --multi_gpu" + if "num_processes" in kwargs and int(kwargs.get("num_processes", 0)) > 0: + run_cmd.append("--num_processes") + run_cmd.append(str(int(kwargs["num_processes"]))) - if "num_processes" in kwargs: - num_processes = kwargs.get("num_processes") - if int(num_processes) > 0: - run_cmd += f" --num_processes={int(num_processes)}" + if "num_machines" in kwargs and int(kwargs.get("num_machines", 0)) > 0: + run_cmd.append("--num_machines") + run_cmd.append(str(int(kwargs["num_machines"]))) - if "num_machines" in kwargs: - num_machines = kwargs.get("num_machines") - if int(num_machines) > 0: - run_cmd += f" --num_machines={int(num_machines)}" - - if "num_cpu_threads_per_process" in kwargs: - num_cpu_threads_per_process = kwargs.get("num_cpu_threads_per_process") - if int(num_cpu_threads_per_process) > 0: - run_cmd += ( - f" --num_cpu_threads_per_process={int(num_cpu_threads_per_process)}" - ) + if ( + "num_cpu_threads_per_process" in kwargs + and int(kwargs.get("num_cpu_threads_per_process", 0)) > 0 + ): + run_cmd.append("--num_cpu_threads_per_process") + run_cmd.append(str(int(kwargs["num_cpu_threads_per_process"]))) return run_cmd diff --git a/kohya_gui/class_basic_training.py b/kohya_gui/class_basic_training.py index 57c8e73..c5a2f72 100644 --- a/kohya_gui/class_basic_training.py +++ b/kohya_gui/class_basic_training.py @@ -179,7 +179,7 @@ class BasicTraining: self.lr_scheduler_args = gr.Textbox( label="LR scheduler extra arguments", lines=2, - placeholder='(Optional) eg: "milestones=[1,10,30,50]" "gamma=0.1"', + placeholder='(Optional) eg: milestones=[1,10,30,50] gamma=0.1', value=self.config.get("basic.lr_scheduler_args", ""), ) # Initialize the optimizer extra arguments textbox diff --git a/kohya_gui/class_command_executor.py b/kohya_gui/class_command_executor.py index 355c673..56e6d8b 100644 --- a/kohya_gui/class_command_executor.py +++ b/kohya_gui/class_command_executor.py @@ -1,6 +1,8 @@ import subprocess import psutil +import os import gradio as gr +import shlex from .custom_logging import setup_logging # Set up logging @@ -29,6 +31,11 @@ class CommandExecutor: if self.process and self.process.poll() is None: log.info("The command is already running. Please wait for it to finish.") else: + # Reconstruct the safe command string for display + command_to_run = ' '.join(run_cmd) + log.info(f"Executings command: {command_to_run}") + + # Execute the command securely self.process = subprocess.Popen(run_cmd, **kwargs) def kill_command(self): diff --git a/kohya_gui/class_sample_images.py b/kohya_gui/class_sample_images.py index 7a1fa0c..ff98041 100644 --- a/kohya_gui/class_sample_images.py +++ b/kohya_gui/class_sample_images.py @@ -1,5 +1,6 @@ import os import gradio as gr +import shlex from .custom_logging import setup_logging from .class_gui_config import KohyaSSGUIConfig @@ -19,6 +20,7 @@ document_symbol = "\U0001F4C4" # 📄 def run_cmd_sample( + run_cmd: list, sample_every_n_steps, sample_every_n_epochs, sample_sampler, @@ -41,8 +43,6 @@ def run_cmd_sample( output_dir = os.path.join(output_dir, "sample") os.makedirs(output_dir, exist_ok=True) - run_cmd = "" - if sample_every_n_epochs is None: sample_every_n_epochs = 0 @@ -58,18 +58,33 @@ def run_cmd_sample( with open(sample_prompts_path, "w") as f: f.write(sample_prompts) - run_cmd += f" --sample_sampler={sample_sampler}" - run_cmd += f' --sample_prompts="{sample_prompts_path}"' + # Append the sampler with proper quoting for safety against special characters + run_cmd.append("--sample_sampler") + run_cmd.append(shlex.quote(sample_sampler)) + # Normalize and fix the path for the sample prompts, handle cross-platform path differences + sample_prompts_path = os.path.abspath(os.path.normpath(sample_prompts_path)) + if os.name == "nt": # Normalize path for Windows + sample_prompts_path = sample_prompts_path.replace("\\", "/") + + # Append the sample prompts path + run_cmd.append('--sample_prompts') + run_cmd.append(sample_prompts_path) + + # Append the sampling frequency for epochs, only if non-zero if sample_every_n_epochs != 0: - run_cmd += f" --sample_every_n_epochs={sample_every_n_epochs}" + run_cmd.append("--sample_every_n_epochs") + run_cmd.append(str(sample_every_n_epochs)) + # Append the sampling frequency for steps, only if non-zero if sample_every_n_steps != 0: - run_cmd += f" --sample_every_n_steps={sample_every_n_steps}" + run_cmd.append("--sample_every_n_steps") + run_cmd.append(str(sample_every_n_steps)) return run_cmd + class SampleImages: """ A class for managing the Gradio interface for sampling images during training. diff --git a/kohya_gui/common_gui.py b/kohya_gui/common_gui.py index 1b3e0fc..1fa946e 100644 --- a/kohya_gui/common_gui.py +++ b/kohya_gui/common_gui.py @@ -7,8 +7,10 @@ import os import re import gradio as gr import sys +import shlex import json import math +import shutil # Set up logging log = setup_logging() @@ -19,6 +21,8 @@ save_style_symbol = "\U0001f4be" # 💾 document_symbol = "\U0001F4C4" # 📄 scriptdir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) +if os.name == "nt": + scriptdir = scriptdir.replace("\\", "/") # insert sd-scripts path into PYTHONPATH sys.path.insert(0, os.path.join(scriptdir, "sd-scripts")) @@ -55,6 +59,29 @@ ALL_PRESET_MODELS = V2_BASE_MODELS + V_PARAMETERIZATION_MODELS + V1_MODELS + SDX ENV_EXCLUSION = ["COLAB_GPU", "RUNPOD_POD_ID"] +def get_executable_path(executable_name: str = None) -> str: + """ + Retrieve and sanitize the path to an executable in the system's PATH. + + Args: + executable_name (str): The name of the executable to find. + + Returns: + str: The full, sanitized path to the executable if found, otherwise an empty string. + """ + if executable_name: + executable_path = shutil.which(executable_name) + if executable_path: + # Replace backslashes with forward slashes on Windows + if os.name == "nt": + executable_path = executable_path.replace("\\", "/") + return executable_path + else: + return "" # Return empty string if the executable is not found + else: + return "" # Return empty string if no executable name is provided + + def calculate_max_train_steps( total_steps: int, train_batch_size: int, @@ -719,6 +746,7 @@ def has_ext_files(folder_path: str, file_extension: str) -> bool: # If no file with the specified extension is found, return False return False + def find_replace( folder_path: str = "", caption_file_ext: str = ".caption", @@ -750,7 +778,7 @@ def find_replace( ) # Exit the function early return - + # Check if the caption file extension is one of the supported extensions if caption_file_ext not in [".caption", ".txt", ".txt2", ".cap"]: log.error( @@ -758,7 +786,7 @@ def find_replace( ) # Exit the function early return - + # Check if the folder path exists if not os.path.exists(folder_path): log.error(f"The provided path '{folder_path}' is not a valid folder.") @@ -766,7 +794,9 @@ def find_replace( # List all caption files in the folder try: - caption_files = [f for f in os.listdir(folder_path) if f.endswith(caption_file_ext)] + caption_files = [ + f for f in os.listdir(folder_path) if f.endswith(caption_file_ext) + ] except Exception as e: log.error(f"Error accessing folder {folder_path}: {e}") return @@ -999,7 +1029,7 @@ def get_str_or_default(kwargs, key, default_value=""): return str(value) -def run_cmd_advanced_training(**kwargs): +def run_cmd_advanced_training(run_cmd: list = [], **kwargs): """ This function, run_cmd_advanced_training, dynamically constructs a command line string for advanced training configurations based on provided keyword arguments (kwargs). Each argument represents a different training parameter @@ -1020,69 +1050,79 @@ def run_cmd_advanced_training(**kwargs): str: A command line string constructed based on the provided keyword arguments. This string includes the base command and additional parameters and flags tailored to the user's specifications for the training process """ - run_cmd = "" - - if "additional_parameters" in kwargs: - run_cmd += f' {kwargs["additional_parameters"]}' + if "additional_parameters" in kwargs and kwargs["additional_parameters"] != "": + additional_parameters = kwargs["additional_parameters"].replace('"', "") + for arg in additional_parameters.split(): + run_cmd.append(shlex.quote(arg)) if "block_lr" in kwargs and kwargs["block_lr"] != "": - run_cmd += f' --block_lr="{kwargs["block_lr"]}"' + run_cmd.append("--block_lr") + run_cmd.append(shlex.quote(kwargs["block_lr"])) if kwargs.get("bucket_no_upscale"): - run_cmd += " --bucket_no_upscale" + run_cmd.append("--bucket_no_upscale") if "bucket_reso_steps" in kwargs: - run_cmd += f' --bucket_reso_steps={int(kwargs["bucket_reso_steps"])}' + run_cmd.append("--bucket_reso_steps") + run_cmd.append(str(int(kwargs["bucket_reso_steps"]))) if kwargs.get("cache_latents"): - run_cmd += " --cache_latents" + run_cmd.append("--cache_latents") if kwargs.get("cache_latents_to_disk"): - run_cmd += " --cache_latents_to_disk" + run_cmd.append("--cache_latents_to_disk") if kwargs.get("cache_text_encoder_outputs"): - run_cmd += " --cache_text_encoder_outputs" + run_cmd.append("--cache_text_encoder_outputs") if ( "caption_dropout_every_n_epochs" in kwargs and int(kwargs["caption_dropout_every_n_epochs"]) > 0 ): - run_cmd += f' --caption_dropout_every_n_epochs="{int(kwargs["caption_dropout_every_n_epochs"])}"' + run_cmd.append("--caption_dropout_every_n_epochs") + run_cmd.append(str(int(kwargs["caption_dropout_every_n_epochs"]))) - caption_dropout_rate = kwargs.get("caption_dropout_rate") - if caption_dropout_rate and float(caption_dropout_rate) > 0: - run_cmd += f' --caption_dropout_rate="{caption_dropout_rate}"' + caption_dropout_rate = kwargs.get("caption_dropout_rate") + if caption_dropout_rate and float(caption_dropout_rate) > 0: + run_cmd.append("--caption_dropout_rate") + run_cmd.append(str(caption_dropout_rate)) - caption_extension = kwargs.get("caption_extension") - if caption_extension: - run_cmd += f' --caption_extension="{caption_extension}"' + caption_extension = kwargs.get("caption_extension") + if caption_extension: + run_cmd.append("--caption_extension") + run_cmd.append(shlex.quote(caption_extension)) - clip_skip = kwargs.get("clip_skip") - if clip_skip and int(clip_skip) > 1: - run_cmd += f" --clip_skip={int(clip_skip)}" + clip_skip = kwargs.get("clip_skip") + if clip_skip and int(clip_skip) > 1: + run_cmd.append("--clip_skip") + run_cmd.append(str(clip_skip)) - color_aug = kwargs.get("color_aug") - if color_aug: - run_cmd += " --color_aug" + color_aug = kwargs.get("color_aug") + if color_aug: + run_cmd.append("--color_aug") - dataset_config = kwargs.get("dataset_config") - if dataset_config: - dataset_config = os.path.abspath(os.path.normpath(dataset_config)) - run_cmd += f' --dataset_config="{dataset_config}"' + dataset_config = kwargs.get("dataset_config") + if dataset_config: + dataset_config = os.path.abspath(os.path.normpath(dataset_config)) + if os.name == "nt": + dataset_config = dataset_config.replace("\\", "/") + run_cmd.append("--dataset_config") + run_cmd.append(dataset_config) dataset_repeats = kwargs.get("dataset_repeats") if dataset_repeats: - run_cmd += f' --dataset_repeats="{dataset_repeats}"' + run_cmd.append("--dataset_repeats") + run_cmd.append(shlex.quote(dataset_repeats)) debiased_estimation_loss = kwargs.get("debiased_estimation_loss") if debiased_estimation_loss: - run_cmd += " --debiased_estimation_loss" + run_cmd.append("--debiased_estimation_loss") dim_from_weights = kwargs.get("dim_from_weights") if dim_from_weights and kwargs.get( "lora_network_weights" ): # Only if lora_network_weights is true - run_cmd += f" --dim_from_weights" + run_cmd.append("--dim_from_weights") # Check if enable_bucket is true and both min_bucket_reso and max_bucket_reso are provided as part of the kwargs if ( @@ -1090,435 +1130,502 @@ def run_cmd_advanced_training(**kwargs): and "min_bucket_reso" in kwargs and "max_bucket_reso" in kwargs ): - # Append the enable_bucket flag and min/max bucket resolution values to the run_cmd string - run_cmd += f' --enable_bucket --min_bucket_reso={kwargs["min_bucket_reso"]} --max_bucket_reso={kwargs["max_bucket_reso"]}' + run_cmd.append("--enable_bucket") + run_cmd.append("--min_bucket_reso") + run_cmd.append(str(kwargs["min_bucket_reso"])) # Ensure it is a string + run_cmd.append("--max_bucket_reso") + run_cmd.append(str(kwargs["max_bucket_reso"])) # Ensure it is a string in_json = kwargs.get("in_json") if in_json: - run_cmd += f' --in_json="{in_json}"' + run_cmd.append("--in_json") + run_cmd.append(shlex.quote(in_json)) flip_aug = kwargs.get("flip_aug") if flip_aug: - run_cmd += " --flip_aug" + run_cmd.append("--flip_aug") fp8_base = kwargs.get("fp8_base") if fp8_base: - run_cmd += " --fp8_base" + run_cmd.append("--fp8_base") full_bf16 = kwargs.get("full_bf16") if full_bf16: - run_cmd += " --full_bf16" + run_cmd.append("--full_bf16") full_fp16 = kwargs.get("full_fp16") if full_fp16: - run_cmd += " --full_fp16" + run_cmd.append("--full_fp16") if ( "gradient_accumulation_steps" in kwargs and int(kwargs["gradient_accumulation_steps"]) > 1 ): - run_cmd += f" --gradient_accumulation_steps={int(kwargs['gradient_accumulation_steps'])}" + run_cmd.append("--gradient_accumulation_steps") + run_cmd.append(str(int(kwargs["gradient_accumulation_steps"]))) if kwargs.get("gradient_checkpointing"): - run_cmd += " --gradient_checkpointing" - - if kwargs.get("huber_c"): - run_cmd += fr' --huber_c="{kwargs.get("huber_c")}"' - - if kwargs.get("huber_schedule"): - run_cmd += fr' --huber_schedule="{kwargs.get("huber_schedule")}"' + run_cmd.append("--gradient_checkpointing") + + if "huber_c" in kwargs: + run_cmd.append("--huber_c") + run_cmd.append(str(kwargs.get("huber_c"))) + + if "huber_schedule" in kwargs: + run_cmd.append("--huber_schedule") + run_cmd.append(shlex.quote(kwargs.get("huber_schedule"))) if kwargs.get("ip_noise_gamma"): if float(kwargs["ip_noise_gamma"]) > 0: - run_cmd += f' --ip_noise_gamma={kwargs["ip_noise_gamma"]}' + run_cmd.append("--ip_noise_gamma") + run_cmd.append(kwargs["ip_noise_gamma"]) if kwargs.get("ip_noise_gamma_random_strength"): if kwargs["ip_noise_gamma_random_strength"]: - run_cmd += f" --ip_noise_gamma_random_strength" + run_cmd.append("--ip_noise_gamma_random_strength") if "keep_tokens" in kwargs and int(kwargs["keep_tokens"]) > 0: - run_cmd += f' --keep_tokens="{int(kwargs["keep_tokens"])}"' + run_cmd.append("--keep_tokens") + run_cmd.append(str(int(kwargs["keep_tokens"]))) if "learning_rate" in kwargs: - run_cmd += f' --learning_rate="{kwargs["learning_rate"]}"' + run_cmd.append("--learning_rate") + run_cmd.append(str(float(kwargs["learning_rate"]))) if "learning_rate_te" in kwargs: if kwargs["learning_rate_te"] == 0: - run_cmd += f' --learning_rate_te="0"' + run_cmd.append("--learning_rate_te") + run_cmd.append("0") else: - run_cmd += f' --learning_rate_te="{kwargs["learning_rate_te"]}"' + run_cmd.append("--learning_rate_te") + run_cmd.append(str(float(kwargs["learning_rate_te"]))) if "learning_rate_te1" in kwargs: if kwargs["learning_rate_te1"] == 0: - run_cmd += f' --learning_rate_te1="0"' + run_cmd.append("--learning_rate_te1") + run_cmd.append("0") else: - run_cmd += f' --learning_rate_te1="{kwargs["learning_rate_te1"]}"' + run_cmd.append("--learning_rate_te1") + run_cmd.append(str(float(kwargs["learning_rate_te1"]))) if "learning_rate_te2" in kwargs: if kwargs["learning_rate_te2"] == 0: - run_cmd += f' --learning_rate_te2="0"' + run_cmd.append("--learning_rate_te2") + run_cmd.append("0") else: - run_cmd += f' --learning_rate_te2="{kwargs["learning_rate_te2"]}"' + run_cmd.append("--learning_rate_te2") + run_cmd.append(str(float(kwargs["learning_rate_te2"]))) - logging_dir = kwargs.get("logging_dir") - if logging_dir: + if logging_dir := kwargs.get("logging_dir"): if logging_dir.startswith('"') and logging_dir.endswith('"'): logging_dir = logging_dir[1:-1] if os.path.exists(logging_dir): - run_cmd += rf' --logging_dir="{logging_dir}"' + logging_dir = os.path.abspath(os.path.normpath(logging_dir)) + if os.name == "nt": + logging_dir = logging_dir.replace("\\", "/") + run_cmd.append("--logging_dir") + run_cmd.append(logging_dir) log_tracker_name = kwargs.get("log_tracker_name") if log_tracker_name: - run_cmd += rf' --log_tracker_name="{log_tracker_name}"' + run_cmd.append(rf"--log_tracker_name") + run_cmd.append(f"{shlex.quote(log_tracker_name)}") log_tracker_config = kwargs.get("log_tracker_config") if log_tracker_config: if log_tracker_config.startswith('"') and log_tracker_config.endswith('"'): log_tracker_config = log_tracker_config[1:-1] if os.path.exists(log_tracker_config): - run_cmd += rf' --log_tracker_config="{log_tracker_config}"' + log_tracker_config = os.path.abspath(os.path.normpath(log_tracker_config)) + if os.name == "nt": + log_tracker_config = log_tracker_config.replace("\\", "/") + run_cmd.append(f"--log_tracker_config") + run_cmd.append(f"{log_tracker_config}") lora_network_weights = kwargs.get("lora_network_weights") if lora_network_weights: - run_cmd += f' --network_weights="{lora_network_weights}"' # Yes, the parameter is now called network_weights instead of lora_network_weights - + run_cmd.append(f"--network_weights") + run_cmd.append( + f"{shlex.quote(lora_network_weights)}" + ) # Yes, the parameter is now called network_weights instead of lora_network_weights + if "loss_type" in kwargs: - run_cmd += fr' --loss_type="{kwargs.get("loss_type")}"' + run_cmd.append("--loss_type") + run_cmd.append(shlex.quote(kwargs["loss_type"])) lr_scheduler = kwargs.get("lr_scheduler") if lr_scheduler: - run_cmd += f' --lr_scheduler="{lr_scheduler}"' + run_cmd.append("--lr_scheduler") + run_cmd.append(shlex.quote(lr_scheduler)) lr_scheduler_args = kwargs.get("lr_scheduler_args") if lr_scheduler_args and lr_scheduler_args != "": - run_cmd += f" --lr_scheduler_args {lr_scheduler_args}" + lr_scheduler_args = lr_scheduler_args.replace('"', "") + args = lr_scheduler_args.split() + run_cmd.append("--lr_scheduler_args") + for arg in args: + run_cmd.append(shlex.quote(arg)) lr_scheduler_num_cycles = kwargs.get("lr_scheduler_num_cycles") - if lr_scheduler_num_cycles and not lr_scheduler_num_cycles == "": - run_cmd += f' --lr_scheduler_num_cycles="{lr_scheduler_num_cycles}"' + if lr_scheduler_num_cycles and lr_scheduler_num_cycles != "": + run_cmd.append("--lr_scheduler_num_cycles") + run_cmd.append(str(lr_scheduler_num_cycles)) else: epoch = kwargs.get("epoch") if epoch: - run_cmd += f' --lr_scheduler_num_cycles="{epoch}"' + run_cmd.append("--lr_scheduler_num_cycles") + run_cmd.append(str(epoch)) lr_scheduler_power = kwargs.get("lr_scheduler_power") - if lr_scheduler_power and not lr_scheduler_power == "": - run_cmd += f' --lr_scheduler_power="{lr_scheduler_power}"' + if lr_scheduler_power and lr_scheduler_power != "": + run_cmd.append("--lr_scheduler_power") + run_cmd.append(str(lr_scheduler_power)) lr_warmup_steps = kwargs.get("lr_warmup_steps") - if lr_warmup_steps: - if lr_scheduler == "constant": - log.info("Can't use LR warmup with LR Scheduler constant... ignoring...") - else: - run_cmd += f' --lr_warmup_steps="{lr_warmup_steps}"' + if lr_warmup_steps and lr_scheduler != "constant": + run_cmd.append("--lr_warmup_steps") + run_cmd.append(str(lr_warmup_steps)) + else: + log.info("Can't use LR warmup with LR Scheduler constant... ignoring...") - if "masked_loss" in kwargs: - if kwargs.get("masked_loss"): # Test if the value is true as it could be false - run_cmd += " --masked_loss" + if "masked_loss" in kwargs and kwargs.get( + "masked_loss" + ): # Test if the value is true as it could be false + run_cmd.append("--masked_loss") if "max_data_loader_n_workers" in kwargs: max_data_loader_n_workers = kwargs.get("max_data_loader_n_workers") - if not max_data_loader_n_workers == "": - run_cmd += f' --max_data_loader_n_workers="{max_data_loader_n_workers}"' + if max_data_loader_n_workers != "": + run_cmd.append("--max_data_loader_n_workers") + run_cmd.append(str(max_data_loader_n_workers)) - if "max_grad_norm" in kwargs: - max_grad_norm = kwargs.get("max_grad_norm") - if max_grad_norm != "": - run_cmd += f' --max_grad_norm="{max_grad_norm}"' + if "max_grad_norm" in kwargs and kwargs["max_grad_norm"] != "": + run_cmd.append("--max_grad_norm") + run_cmd.append(str(float(kwargs["max_grad_norm"]))) if "max_resolution" in kwargs: - run_cmd += rf' --resolution="{kwargs.get("max_resolution")}"' + run_cmd.append("--resolution") + run_cmd.append(shlex.quote(kwargs["max_resolution"])) - if "max_timestep" in kwargs: - max_timestep = kwargs.get("max_timestep") - if int(max_timestep) < 1000: - run_cmd += f" --max_timestep={int(max_timestep)}" + if "max_timestep" in kwargs and int(kwargs["max_timestep"]) < 1000: + run_cmd.append("--max_timestep") + run_cmd.append(str(int(kwargs["max_timestep"]))) - if "max_token_length" in kwargs: - max_token_length = kwargs.get("max_token_length") - if int(max_token_length) > 75: - run_cmd += f" --max_token_length={int(max_token_length)}" + if "max_token_length" in kwargs and int(kwargs["max_token_length"]) > 75: + run_cmd.append("--max_token_length") + run_cmd.append(str(int(kwargs["max_token_length"]))) - if "max_train_epochs" in kwargs: - max_train_epochs = kwargs.get("max_train_epochs") - if not max_train_epochs == "": - run_cmd += f" --max_train_epochs={max_train_epochs}" + if "max_train_epochs" in kwargs and kwargs["max_train_epochs"] != "": + run_cmd.append("--max_train_epochs") + run_cmd.append(str(int(kwargs["max_train_epochs"]))) - if "max_train_steps" in kwargs: - max_train_steps = kwargs.get("max_train_steps") - if not max_train_steps == "": - run_cmd += f' --max_train_steps="{max_train_steps}"' + if "max_train_steps" in kwargs and kwargs["max_train_steps"] != "": + run_cmd.append("--max_train_steps") + run_cmd.append(str(int(kwargs["max_train_steps"]))) - if "mem_eff_attn" in kwargs: - if kwargs.get("mem_eff_attn"): # Test if the value is true as it could be false - run_cmd += " --mem_eff_attn" + if "mem_eff_attn" in kwargs and kwargs.get("mem_eff_attn"): + run_cmd.append("--mem_eff_attn") - if "min_snr_gamma" in kwargs: - min_snr_gamma = kwargs.get("min_snr_gamma") - if int(min_snr_gamma) >= 1: - run_cmd += f" --min_snr_gamma={int(min_snr_gamma)}" + if "min_snr_gamma" in kwargs and int(kwargs["min_snr_gamma"]) >= 1: + run_cmd.append("--min_snr_gamma") + run_cmd.append(str(int(kwargs["min_snr_gamma"]))) - if "min_timestep" in kwargs: - min_timestep = kwargs.get("min_timestep") - if int(min_timestep) > -1: - run_cmd += f" --min_timestep={int(min_timestep)}" + if "min_timestep" in kwargs and int(kwargs["min_timestep"]) > -1: + run_cmd.append("--min_timestep") + run_cmd.append(str(int(kwargs["min_timestep"]))) if "mixed_precision" in kwargs: - run_cmd += rf' --mixed_precision="{kwargs.get("mixed_precision")}"' + run_cmd.append("--mixed_precision") + run_cmd.append(shlex.quote(kwargs["mixed_precision"])) if "network_alpha" in kwargs: - run_cmd += rf' --network_alpha="{kwargs.get("network_alpha")}"' + run_cmd.append("--network_alpha") + run_cmd.append(str(int(kwargs["network_alpha"]))) - if "network_args" in kwargs: - network_args = kwargs.get("network_args") - if network_args != "": - run_cmd += f" --network_args{network_args}" + if "network_args" in kwargs and kwargs["network_args"]: + network_args = kwargs["network_args"].replace('"', "") + args = network_args.split() + run_cmd.append("--network_args") + run_cmd.extend(args) # Adds all args to the command list directly if "network_dim" in kwargs: - run_cmd += rf' --network_dim={kwargs.get("network_dim")}' + run_cmd.append("--network_dim") + run_cmd.append(str(int(kwargs["network_dim"]))) - if "network_dropout" in kwargs: - network_dropout = kwargs.get("network_dropout") - if network_dropout > 0.0: - run_cmd += f" --network_dropout={network_dropout}" + if "network_dropout" in kwargs and float(kwargs["network_dropout"]) > 0.0: + run_cmd.append("--network_dropout") + run_cmd.append(str(kwargs["network_dropout"])) - if "network_module" in kwargs: - network_module = kwargs.get("network_module") - if network_module != "": - run_cmd += f" --network_module={network_module}" + if "network_module" in kwargs and kwargs["network_module"]: + run_cmd.append("--network_module") + run_cmd.append(shlex.quote(kwargs["network_module"])) - if "network_train_text_encoder_only" in kwargs: - if kwargs.get("network_train_text_encoder_only"): - run_cmd += " --network_train_text_encoder_only" + if ( + "network_train_text_encoder_only" in kwargs + and kwargs["network_train_text_encoder_only"] + ): + run_cmd.append("--network_train_text_encoder_only") - if "network_train_unet_only" in kwargs: - if kwargs.get("network_train_unet_only"): - run_cmd += " --network_train_unet_only" + if "network_train_unet_only" in kwargs and kwargs["network_train_unet_only"]: + run_cmd.append("--network_train_unet_only") - if "no_half_vae" in kwargs: - if kwargs.get("no_half_vae"): # Test if the value is true as it could be false - run_cmd += " --no_half_vae" + if "no_half_vae" in kwargs and kwargs["no_half_vae"]: + run_cmd.append("--no_half_vae") - if "no_token_padding" in kwargs: - if kwargs.get( - "no_token_padding" - ): # Test if the value is true as it could be false - run_cmd += " --no_token_padding" + if "no_token_padding" in kwargs and kwargs["no_token_padding"]: + run_cmd.append("--no_token_padding") if "noise_offset_type" in kwargs: noise_offset_type = kwargs["noise_offset_type"] if noise_offset_type == "Original": - if "noise_offset" in kwargs: - noise_offset = float(kwargs.get("noise_offset", 0)) - if noise_offset: - run_cmd += f" --noise_offset={float(noise_offset)}" + if "noise_offset" in kwargs and float(kwargs.get("noise_offset", 0)): + run_cmd.append("--noise_offset") + run_cmd.append(str(float(kwargs["noise_offset"]))) - if "adaptive_noise_scale" in kwargs: - adaptive_noise_scale = float(kwargs.get("adaptive_noise_scale", 0)) - if adaptive_noise_scale != 0 and noise_offset > 0: - run_cmd += f" --adaptive_noise_scale={adaptive_noise_scale}" + if ( + "adaptive_noise_scale" in kwargs + and float(kwargs.get("adaptive_noise_scale", 0)) != 0 + ): + adaptive_noise_scale = float(kwargs["adaptive_noise_scale"]) + if ( + adaptive_noise_scale > 0 + and float(kwargs.get("noise_offset", 0)) > 0 + ): + run_cmd.append("--adaptive_noise_scale") + run_cmd.append(str(adaptive_noise_scale)) + + if "noise_offset_random_strength" in kwargs and kwargs.get( + "noise_offset_random_strength" + ): + run_cmd.append("--noise_offset_random_strength") - if "noise_offset_random_strength" in kwargs: - if kwargs.get("noise_offset_random_strength"): - run_cmd += f" --noise_offset_random_strength" elif noise_offset_type == "Multires": - if "multires_noise_iterations" in kwargs: - multires_noise_iterations = int( - kwargs.get("multires_noise_iterations", 0) - ) - if multires_noise_iterations > 0: - run_cmd += ( - f' --multires_noise_iterations="{multires_noise_iterations}"' - ) + if ( + "multires_noise_iterations" in kwargs + and int(kwargs.get("multires_noise_iterations", 0)) > 0 + ): + run_cmd.append("--multires_noise_iterations") + run_cmd.append(str(int(kwargs["multires_noise_iterations"]))) - if "multires_noise_discount" in kwargs: - multires_noise_discount = float( - kwargs.get("multires_noise_discount", 0) - ) - if multires_noise_discount > 0: - run_cmd += f' --multires_noise_discount="{multires_noise_discount}"' + if ( + "multires_noise_discount" in kwargs + and float(kwargs.get("multires_noise_discount", 0)) > 0 + ): + run_cmd.append("--multires_noise_discount") + run_cmd.append(str(float(kwargs["multires_noise_discount"]))) - if "optimizer_args" in kwargs: - optimizer_args = kwargs.get("optimizer_args") - if optimizer_args != "": - run_cmd += f" --optimizer_args {optimizer_args}" + if "optimizer_args" in kwargs and kwargs.get("optimizer_args"): + run_cmd.append("--optimizer_args") + optimizer_args = kwargs["optimizer_args"].replace('"', "") + args = optimizer_args.split() + for arg in args: + run_cmd.append(shlex.quote(arg)) if "optimizer" in kwargs: - run_cmd += rf' --optimizer_type="{kwargs.get("optimizer")}"' + run_cmd.append("--optimizer_type") + run_cmd.append(shlex.quote(kwargs["optimizer"])) - if "output_dir" in kwargs: - output_dir = kwargs.get("output_dir") - if output_dir.startswith('"') and output_dir.endswith('"'): - output_dir = output_dir[1:-1] + if "output_dir" in kwargs and kwargs["output_dir"]: + output_dir = kwargs["output_dir"].strip( + '"' + ) # Remove surrounding quotes if present if os.path.exists(output_dir): - run_cmd += rf' --output_dir="{output_dir}"' + output_dir = os.path.abspath(os.path.normpath(output_dir)) + if os.name == "nt": + output_dir = output_dir.replace("\\", "/") + run_cmd.append("--output_dir") + run_cmd.append(output_dir) - if "output_name" in kwargs: - output_name = kwargs.get("output_name") - if not output_name == "": - run_cmd += f' --output_name="{output_name}"' + if "output_name" in kwargs and kwargs["output_name"]: + run_cmd.append("--output_name") + run_cmd.append(shlex.quote(kwargs["output_name"])) - if "persistent_data_loader_workers" in kwargs: - if kwargs.get("persistent_data_loader_workers"): - run_cmd += " --persistent_data_loader_workers" + if ( + "persistent_data_loader_workers" in kwargs + and kwargs["persistent_data_loader_workers"] + ): + run_cmd.append("--persistent_data_loader_workers") - if "pretrained_model_name_or_path" in kwargs: - run_cmd += rf' --pretrained_model_name_or_path="{kwargs.get("pretrained_model_name_or_path")}"' + if ( + "pretrained_model_name_or_path" in kwargs + and kwargs["pretrained_model_name_or_path"] + ): + path = kwargs["pretrained_model_name_or_path"] + if os.name == "nt": + path = path.replace("\\", "/") + run_cmd.append("--pretrained_model_name_or_path") + run_cmd.append(path) - if "prior_loss_weight" in kwargs: - prior_loss_weight = kwargs.get("prior_loss_weight") - if not float(prior_loss_weight) == 1.0: - run_cmd += f" --prior_loss_weight={prior_loss_weight}" + if "prior_loss_weight" in kwargs and float(kwargs["prior_loss_weight"]) != 1.0: + run_cmd.append("--prior_loss_weight") + run_cmd.append(shlex.quote(str(float(kwargs["prior_loss_weight"])))) - if "random_crop" in kwargs: - random_crop = kwargs.get("random_crop") - if random_crop: - run_cmd += " --random_crop" + if "random_crop" in kwargs and kwargs["random_crop"]: + run_cmd.append("--random_crop") - if "reg_data_dir" in kwargs: - reg_data_dir = kwargs.get("reg_data_dir") - if len(reg_data_dir): - if reg_data_dir.startswith('"') and reg_data_dir.endswith('"'): - reg_data_dir = reg_data_dir[1:-1] - if os.path.isdir(reg_data_dir): - run_cmd += rf' --reg_data_dir="{reg_data_dir}"' + if "reg_data_dir" in kwargs and kwargs["reg_data_dir"]: + reg_data_dir = kwargs["reg_data_dir"].strip( + '"' + ) # Remove surrounding quotes if present + if os.path.isdir(reg_data_dir): + reg_data_dir = os.path.abspath(os.path.normpath(reg_data_dir)) + if os.name == "nt": + reg_data_dir = reg_data_dir.replace("\\", "/") + run_cmd.append("--reg_data_dir") + run_cmd.append(reg_data_dir) - if "resume" in kwargs: - resume = kwargs.get("resume") - if len(resume): - run_cmd += f' --resume="{resume}"' + if "resume" in kwargs and kwargs["resume"]: + run_cmd.append("--resume") + run_cmd.append(shlex.quote(kwargs["resume"])) - if "save_every_n_epochs" in kwargs: - save_every_n_epochs = kwargs.get("save_every_n_epochs") - if int(save_every_n_epochs) > 0: - run_cmd += f' --save_every_n_epochs="{int(save_every_n_epochs)}"' + if "save_every_n_epochs" in kwargs and int(kwargs["save_every_n_epochs"]) > 0: + run_cmd.append("--save_every_n_epochs") + run_cmd.append(str(int(kwargs["save_every_n_epochs"]))) - if "save_every_n_steps" in kwargs: - save_every_n_steps = kwargs.get("save_every_n_steps") - if int(save_every_n_steps) > 0: - run_cmd += f' --save_every_n_steps="{int(save_every_n_steps)}"' + if "save_every_n_steps" in kwargs and int(kwargs["save_every_n_steps"]) > 0: + run_cmd.append("--save_every_n_steps") + run_cmd.append(str(int(kwargs["save_every_n_steps"]))) - if "save_last_n_steps" in kwargs: - save_last_n_steps = kwargs.get("save_last_n_steps") - if int(save_last_n_steps) > 0: - run_cmd += f' --save_last_n_steps="{int(save_last_n_steps)}"' + if "save_last_n_steps" in kwargs and int(kwargs["save_last_n_steps"]) > 0: + run_cmd.append("--save_last_n_steps") + run_cmd.append(str(int(kwargs["save_last_n_steps"]))) - if "save_last_n_steps_state" in kwargs: - save_last_n_steps_state = kwargs.get("save_last_n_steps_state") - if int(save_last_n_steps_state) > 0: - run_cmd += f' --save_last_n_steps_state="{int(save_last_n_steps_state)}"' + if ( + "save_last_n_steps_state" in kwargs + and int(kwargs["save_last_n_steps_state"]) > 0 + ): + run_cmd.append("--save_last_n_steps_state") + run_cmd.append(str(int(kwargs["save_last_n_steps_state"]))) - if "save_model_as" in kwargs: - save_model_as = kwargs.get("save_model_as") - if save_model_as != "same as source model": - run_cmd += f" --save_model_as={save_model_as}" + if "save_model_as" in kwargs and kwargs["save_model_as"] != "same as source model": + run_cmd.append("--save_model_as") + run_cmd.append(kwargs["save_model_as"]) if "save_precision" in kwargs: - run_cmd += rf' --save_precision="{kwargs.get("save_precision")}"' + run_cmd.append("--save_precision") + run_cmd.append(kwargs["save_precision"]) - if "save_state" in kwargs: - if kwargs.get("save_state"): - run_cmd += " --save_state" + if "save_state" in kwargs and kwargs["save_state"]: + run_cmd.append("--save_state") - if "save_state_on_train_end" in kwargs: - if kwargs.get("save_state_on_train_end"): - run_cmd += " --save_state_on_train_end" + if "save_state_on_train_end" in kwargs and kwargs["save_state_on_train_end"]: + run_cmd.append("--save_state_on_train_end") - if "scale_v_pred_loss_like_noise_pred" in kwargs: - if kwargs.get("scale_v_pred_loss_like_noise_pred"): - run_cmd += " --scale_v_pred_loss_like_noise_pred" + if ( + "scale_v_pred_loss_like_noise_pred" in kwargs + and kwargs["scale_v_pred_loss_like_noise_pred"] + ): + run_cmd.append("--scale_v_pred_loss_like_noise_pred") - if "scale_weight_norms" in kwargs: - scale_weight_norms = kwargs.get("scale_weight_norms") - if scale_weight_norms > 0.0: - run_cmd += f' --scale_weight_norms="{scale_weight_norms}"' + if "scale_weight_norms" in kwargs and kwargs["scale_weight_norms"] > 0.0: + run_cmd.append("--scale_weight_norms") + run_cmd.append(str(float(kwargs["scale_weight_norms"]))) - if "seed" in kwargs: - seed = kwargs.get("seed") - if seed != "": - run_cmd += f' --seed="{seed}"' + if "seed" in kwargs and kwargs["seed"] != "": + run_cmd.append("--seed") + run_cmd.append(str(int(kwargs["seed"]))) - if "shuffle_caption" in kwargs: - if kwargs.get("shuffle_caption"): - run_cmd += " --shuffle_caption" + if "shuffle_caption" in kwargs and kwargs["shuffle_caption"]: + run_cmd.append("--shuffle_caption") - if "stop_text_encoder_training" in kwargs: - stop_text_encoder_training = kwargs.get("stop_text_encoder_training") - if stop_text_encoder_training > 0: - run_cmd += f' --stop_text_encoder_training="{stop_text_encoder_training}"' + if ( + "stop_text_encoder_training" in kwargs + and kwargs["stop_text_encoder_training"] > 0 + ): + run_cmd.append("--stop_text_encoder_training") + run_cmd.append(str(int(kwargs["stop_text_encoder_training"]))) - if "text_encoder_lr" in kwargs: - text_encoder_lr = kwargs.get("text_encoder_lr") - if float(text_encoder_lr) > 0: - run_cmd += f" --text_encoder_lr={text_encoder_lr}" + # Text encoder learning rate + if "text_encoder_lr" in kwargs and float(kwargs["text_encoder_lr"]) > 0: + run_cmd.append("--text_encoder_lr") + run_cmd.append(str(kwargs["text_encoder_lr"])) + # Training batch size if "train_batch_size" in kwargs: - run_cmd += rf' --train_batch_size="{kwargs.get("train_batch_size")}"' + run_cmd.append("--train_batch_size") + run_cmd.append(str(int(kwargs["train_batch_size"]))) - training_comment = kwargs.get("training_comment") - if training_comment and len(training_comment): - run_cmd += f' --training_comment="{training_comment}"' + # Training comment + if training_comment := kwargs.get("training_comment"): + run_cmd.append("--training_comment") + run_cmd.append(shlex.quote(training_comment)) - train_data_dir = kwargs.get("train_data_dir") - if train_data_dir: - if train_data_dir.startswith('"') and train_data_dir.endswith('"'): - train_data_dir = train_data_dir[1:-1] + # Train data directory + if train_data_dir := kwargs.get("train_data_dir"): + train_data_dir = train_data_dir.strip('"') if os.path.exists(train_data_dir): - run_cmd += rf' --train_data_dir="{train_data_dir}"' + train_data_dir = os.path.abspath(os.path.normpath(train_data_dir)) + train_data_dir = ( + train_data_dir.replace("\\", "/") if os.name == "nt" else train_data_dir + ) + run_cmd.append("--train_data_dir") + run_cmd.append(train_data_dir) - train_text_encoder = kwargs.get("train_text_encoder") - if train_text_encoder: - run_cmd += " --train_text_encoder" + # Train text encoder + if kwargs.get("train_text_encoder"): + run_cmd.append("--train_text_encoder") - unet_lr = kwargs.get("unet_lr") - if unet_lr and (float(unet_lr) > 0): - run_cmd += f" --unet_lr={unet_lr}" + # Unet learning rate + if "unet_lr" in kwargs and float(kwargs["unet_lr"]) > 0: + run_cmd.append("--unet_lr") + run_cmd.append(str(kwargs["unet_lr"])) - use_wandb = kwargs.get("use_wandb") - if use_wandb: - run_cmd += " --log_with wandb" + # Use Weights and Biases logging + if kwargs.get("use_wandb"): + run_cmd.append("--log_with wandb") - v_parameterization = kwargs.get("v_parameterization") - if v_parameterization: - run_cmd += " --v_parameterization" + # V parameterization + if kwargs.get("v_parameterization"): + run_cmd.append("--v_parameterization") - v_pred_like_loss = kwargs.get("v_pred_like_loss") - if v_pred_like_loss and float(v_pred_like_loss) > 0: - run_cmd += f' --v_pred_like_loss="{float(v_pred_like_loss)}"' + # V predicted likelihood loss + if "v_pred_like_loss" in kwargs and float(kwargs["v_pred_like_loss"]) > 0: + run_cmd.append("--v_pred_like_loss") + run_cmd.append(str(float(kwargs["v_pred_like_loss"]))) - v2 = kwargs.get("v2") - if v2: - run_cmd += " --v2" + # Version 2 + if kwargs.get("v2"): + run_cmd.append("--v2") - vae = kwargs.get("vae") - if vae and not vae == "": - if not os.path.exists(vae): - vae = os.path.join("models", "VAE", vae).replace(os.sep, "/") + # VAE path + if vae := kwargs.get("vae"): + vae = ( + vae + if os.path.exists(vae) + else os.path.join("models", "VAE", vae).replace(os.sep, "/") + ) if os.path.exists(vae): - run_cmd += f' --vae="{vae}"' + vae = ( + os.path.abspath(os.path.normpath(vae)).replace("\\", "/") + if os.name == "nt" + else vae + ) + run_cmd.append("--vae") + run_cmd.append(vae) - vae_batch_size = kwargs.get("vae_batch_size") - if vae_batch_size and int(vae_batch_size) > 0: - run_cmd += f' --vae_batch_size="{int(vae_batch_size)}"' + # VAE batch size + if "vae_batch_size" in kwargs and int(kwargs["vae_batch_size"]) > 0: + run_cmd.append("--vae_batch_size") + run_cmd.append(str(int(kwargs["vae_batch_size"]))) - wandb_api_key = kwargs.get("wandb_api_key") - if wandb_api_key: - run_cmd += f' --wandb_api_key="{wandb_api_key}"' + # Weights and Biases API key + if wandb_api_key := kwargs.get("wandb_api_key"): + run_cmd.append("--wandb_api_key") + run_cmd.append(shlex.quote(wandb_api_key)) - wandb_run_name = kwargs.get("wandb_run_name") - if wandb_run_name: - run_cmd += f' --wandb_run_name="{wandb_run_name}"' + # Weights and Biases run name + if wandb_run_name := kwargs.get("wandb_run_name"): + run_cmd.append("--wandb_run_name") + run_cmd.append(shlex.quote(wandb_run_name)) - weighted_captions = kwargs.get("weighted_captions") - if weighted_captions: - run_cmd += " --weighted_captions" + # Weighted captions + if kwargs.get("weighted_captions"): + run_cmd.append("--weighted_captions") - xformers = kwargs.get("xformers") - if xformers and xformers == "xformers": - run_cmd += " --xformers" - elif xformers and xformers == "sdpa": - run_cmd += " --sdpa" + # Xformers + if xformers := kwargs.get("xformers"): + run_cmd.append("--xformers" if xformers == "xformers" else "--sdpa") return run_cmd @@ -1626,7 +1733,7 @@ def SaveConfigFile( # Check if the folder path for the file_path is valid # Extrach folder path folder_path = os.path.dirname(file_path) - + # Check if the folder exists if not os.path.exists(folder_path): # If not, create the folder diff --git a/kohya_gui/convert_lcm_gui.py b/kohya_gui/convert_lcm_gui.py index c087eec..5b8fb1e 100644 --- a/kohya_gui/convert_lcm_gui.py +++ b/kohya_gui/convert_lcm_gui.py @@ -41,22 +41,35 @@ def convert_lcm(name, model_path, lora_scale, model_type): path, ext = os.path.splitext(save_to) save_to = f"{path}_lcm{ext}" - # Construct the command to run the script - run_cmd += f" --lora-scale {lora_scale}" - run_cmd += f' --model "{model_path}"' - run_cmd += f' --name "{name}"' + # Initialize the command to run the script + run_cmd = [ + PYTHON, + f"{scriptdir}/path_to_script.py", + ] # Adjust the script path accordingly + # Add required arguments + run_cmd.append("--lora-scale") + run_cmd.append(str(lora_scale)) + run_cmd.append("--model") + run_cmd.append(model_path) + run_cmd.append("--name") + run_cmd.append(name) + + # Add conditional flags based on the model type if model_type == "SDXL": - run_cmd += f" --sdxl" + run_cmd.append("--sdxl") if model_type == "SSD-1B": - run_cmd += f" --ssd-1b" + run_cmd.append("--ssd-1b") - log.info(run_cmd) + # Log the command + log.info(' '.join(run_cmd)) + # Set up the environment env = os.environ.copy() env["PYTHONPATH"] = ( rf"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}" ) + env["TF_ENABLE_ONEDNN_OPTS"] = "0" # Run the command subprocess.run(run_cmd, env=env) diff --git a/kohya_gui/convert_model_gui.py b/kohya_gui/convert_model_gui.py index aedda3c..55639a9 100644 --- a/kohya_gui/convert_model_gui.py +++ b/kohya_gui/convert_model_gui.py @@ -48,65 +48,69 @@ def convert_model( msgbox("The provided target folder does not exist") return - run_cmd = ( - rf'"{PYTHON}" "{scriptdir}/sd-scripts/tools/convert_diffusers20_original_sd.py"' - ) + run_cmd = [ + PYTHON, f"{scriptdir}/sd-scripts/tools/convert_diffusers20_original_sd.py" + ] v1_models = [ "runwayml/stable-diffusion-v1-5", "CompVis/stable-diffusion-v1-4", ] - # check if v1 models + # Check if v1 models if str(source_model_type) in v1_models: log.info("SD v1 model specified. Setting --v1 parameter") - run_cmd += " --v1" + run_cmd.append("--v1") else: log.info("SD v2 model specified. Setting --v2 parameter") - run_cmd += " --v2" + run_cmd.append("--v2") if not target_save_precision_type == "unspecified": - run_cmd += f" --{target_save_precision_type}" + run_cmd.append(f"--{target_save_precision_type}") if target_model_type == "diffuser" or target_model_type == "diffuser_safetensors": - run_cmd += f' --reference_model="{source_model_type}"' + run_cmd.append('--reference_model') + run_cmd.append(source_model_type) if target_model_type == "diffuser_safetensors": - run_cmd += " --use_safetensors" - - # Fix for stabilityAI diffusers format. When saving v2 models in Diffusers format in training scripts and conversion scripts, - # it was found that the U-Net configuration is different from those of Hugging Face's stabilityai models (this repository is - # "use_linear_projection": false, stabilityai is true). Please note that the weight shapes are different, so please be careful - # when using the weight files directly. + run_cmd.append("--use_safetensors") + # Fix for stabilityAI diffusers format if unet_use_linear_projection: - run_cmd += " --unet_use_linear_projection" + run_cmd.append("--unet_use_linear_projection") - run_cmd += f' "{source_model_input}"' + # Add the source model input path + run_cmd.append(source_model_input) + # Determine the target model path if target_model_type == "diffuser" or target_model_type == "diffuser_safetensors": target_model_path = os.path.join( target_model_folder_input, target_model_name_input ) - run_cmd += f' "{target_model_path}"' else: target_model_path = os.path.join( target_model_folder_input, f"{target_model_name_input}.{target_model_type}", ) - run_cmd += f' "{target_model_path}"' - log.info(run_cmd) + # Add the target model path + run_cmd.append(target_model_path) + + # Log the command + log.info(' '.join(run_cmd)) env = os.environ.copy() env["PYTHONPATH"] = ( rf"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}" ) + # Adding an example of an environment variable that might be relevant + env["TF_ENABLE_ONEDNN_OPTS"] = "0" # Run the command subprocess.run(run_cmd, env=env) + ### # Gradio UI ### diff --git a/kohya_gui/dreambooth_gui.py b/kohya_gui/dreambooth_gui.py index a8381c5..b4be8f4 100644 --- a/kohya_gui/dreambooth_gui.py +++ b/kohya_gui/dreambooth_gui.py @@ -2,9 +2,11 @@ import gradio as gr import json import math import os +import shlex import sys from datetime import datetime from .common_gui import ( + get_executable_path, get_file_path, get_saveasfile_path, color_aug_changed, @@ -563,10 +565,10 @@ def train_model( lr_warmup_steps = 0 log.info(f"lr_warmup_steps = {lr_warmup_steps}") - # run_cmd = f'accelerate launch --num_cpu_threads_per_process={num_cpu_threads_per_process} "train_db.py"' - run_cmd = "accelerate launch" + run_cmd = ["accelerate", "launch"] - run_cmd += AccelerateLaunch.run_cmd( + run_cmd = AccelerateLaunch.run_cmd( + run_cmd=run_cmd, num_processes=num_processes, num_machines=num_machines, multi_gpu=multi_gpu, @@ -578,9 +580,9 @@ def train_model( ) if sdxl: - run_cmd += rf' "{scriptdir}/sd-scripts/sdxl_train.py"' + run_cmd.append(f'{scriptdir}/sd-scripts/sdxl_train.py') else: - run_cmd += rf' "{scriptdir}/sd-scripts/train_db.py"' + run_cmd.append(f'{scriptdir}/sd-scripts/train_db.py') # Initialize a dictionary with always-included keyword arguments kwargs_for_training = { @@ -682,9 +684,10 @@ def train_model( kwargs_for_training["learning_rate_te"] = learning_rate_te # Pass the dynamically constructed keyword arguments to the function - run_cmd += run_cmd_advanced_training(**kwargs_for_training) + run_cmd = run_cmd_advanced_training(run_cmd=run_cmd, **kwargs_for_training) - run_cmd += run_cmd_sample( + run_cmd = run_cmd_sample( + run_cmd, sample_every_n_steps, sample_every_n_epochs, sample_sampler, @@ -696,9 +699,12 @@ def train_model( log.warning( "Here is the trainer command as a reference. It will not be executed:\n" ) - print(run_cmd) + # Reconstruct the safe command string for display + command_to_run = ' '.join(run_cmd) + + print(command_to_run) - save_to_file(run_cmd) + save_to_file(command_to_run) else: # Saving config file for model current_datetime = datetime.now() @@ -714,7 +720,7 @@ def train_model( exclusion=["file_path", "save_as", "headless", "print_only"], ) - log.info(run_cmd) + # log.info(run_cmd) env = os.environ.copy() env["PYTHONPATH"] = ( diff --git a/kohya_gui/extract_lora_from_dylora_gui.py b/kohya_gui/extract_lora_from_dylora_gui.py index 9f55b66..e87c29d 100644 --- a/kohya_gui/extract_lora_from_dylora_gui.py +++ b/kohya_gui/extract_lora_from_dylora_gui.py @@ -49,19 +49,26 @@ def extract_dylora( path, ext = os.path.splitext(save_to) save_to = f"{path}_tmp{ext}" - run_cmd = ( - rf'"{PYTHON}" "{scriptdir}/sd-scripts/networks/extract_lora_from_dylora.py"' - ) - run_cmd += rf' --save_to "{save_to}"' - run_cmd += rf' --model "{model}"' - run_cmd += f" --unit {unit}" + run_cmd = [ + PYTHON, + f"{scriptdir}/sd-scripts/networks/extract_lora_from_dylora.py", + "--save_to", + save_to, + "--model", + model, + "--unit", + str(unit), + ] - log.info(run_cmd) + # Log the command + log.info(" ".join(run_cmd)) env = os.environ.copy() env["PYTHONPATH"] = ( rf"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}" ) + # Example environment variable adjustment for the Python environment + env["TF_ENABLE_ONEDNN_OPTS"] = "0" # Run the command subprocess.run(run_cmd, env=env) diff --git a/kohya_gui/extract_lora_gui.py b/kohya_gui/extract_lora_gui.py index 4ef0190..ea0cc53 100644 --- a/kohya_gui/extract_lora_gui.py +++ b/kohya_gui/extract_lora_gui.py @@ -72,34 +72,52 @@ def extract_lora( if not is_file_writable(save_to): return - run_cmd = ( - rf'"{PYTHON}" "{scriptdir}/sd-scripts/networks/extract_lora_from_models.py"' - ) - run_cmd += f" --load_precision {load_precision}" - run_cmd += f" --save_precision {save_precision}" - run_cmd += rf' --save_to "{save_to}"' - run_cmd += rf' --model_org "{model_org}"' - run_cmd += rf' --model_tuned "{model_tuned}"' - run_cmd += f" --dim {dim}" - run_cmd += f" --device {device}" - if conv_dim > 0: - run_cmd += f" --conv_dim {conv_dim}" - if v2: - run_cmd += f" --v2" - if sdxl: - run_cmd += f" --sdxl" - run_cmd += f" --clamp_quantile {clamp_quantile}" - run_cmd += f" --min_diff {min_diff}" - if sdxl: - run_cmd += f" --load_original_model_to {load_original_model_to}" - run_cmd += f" --load_tuned_model_to {load_tuned_model_to}" + run_cmd = [ + PYTHON, + f"{scriptdir}/sd-scripts/networks/extract_lora_from_models.py", + "--load_precision", + load_precision, + "--save_precision", + save_precision, + "--save_to", + save_to, + "--model_org", + model_org, + "--model_tuned", + model_tuned, + "--dim", + str(dim), + "--device", + device, + "--clamp_quantile", + str(clamp_quantile), + "--min_diff", + str(min_diff), + ] - log.info(run_cmd) + if conv_dim > 0: + run_cmd.append("--conv_dim") + run_cmd.append(str(conv_dim)) + + if v2: + run_cmd.append("--v2") + + if sdxl: + run_cmd.append("--sdxl") + run_cmd.append("--load_original_model_to") + run_cmd.append(load_original_model_to) + run_cmd.append("--load_tuned_model_to") + run_cmd.append(load_tuned_model_to) + + # Log the command + log.info(" ".join(run_cmd)) env = os.environ.copy() env["PYTHONPATH"] = ( rf"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}" ) + # Adding an example of another potentially relevant environment variable + env["TF_ENABLE_ONEDNN_OPTS"] = "0" # Run the command subprocess.run(run_cmd, env=env) diff --git a/kohya_gui/extract_lycoris_locon_gui.py b/kohya_gui/extract_lycoris_locon_gui.py index 1da7010..e22ef85 100644 --- a/kohya_gui/extract_lycoris_locon_gui.py +++ b/kohya_gui/extract_lycoris_locon_gui.py @@ -73,41 +73,66 @@ def extract_lycoris_locon( path, ext = os.path.splitext(output_name) output_name = f"{path}_tmp{ext}" - run_cmd = rf'"{PYTHON}" "{scriptdir}/tools/lycoris_locon_extract.py"' - if is_sdxl: - run_cmd += f" --is_sdxl" - if is_v2: - run_cmd += f" --is_v2" - run_cmd += f" --device {device}" - run_cmd += f" --mode {mode}" - run_cmd += f" --safetensors" - if mode == "fixed": - run_cmd += f" --linear_dim {linear_dim}" - run_cmd += f" --conv_dim {conv_dim}" - if mode == "threshold": - run_cmd += f" --linear_threshold {linear_threshold}" - run_cmd += f" --conv_threshold {conv_threshold}" - if mode == "ratio": - run_cmd += f" --linear_ratio {linear_ratio}" - run_cmd += f" --conv_ratio {conv_ratio}" - if mode == "quantile": - run_cmd += f" --linear_quantile {linear_quantile}" - run_cmd += f" --conv_quantile {conv_quantile}" - if use_sparse_bias: - run_cmd += f" --use_sparse_bias" - run_cmd += f" --sparsity {sparsity}" - if disable_cp: - run_cmd += f" --disable_cp" - run_cmd += rf' "{base_model}"' - run_cmd += rf' "{db_model}"' - run_cmd += rf' "{output_name}"' + run_cmd = [PYTHON, f"{scriptdir}/tools/lycoris_locon_extract.py"] - log.info(run_cmd) + if is_sdxl: + run_cmd.append("--is_sdxl") + if is_v2: + run_cmd.append("--is_v2") + + # Adding required parameters + run_cmd.append("--device") + run_cmd.append(device) + run_cmd.append("--mode") + run_cmd.append(mode) + run_cmd.append("--safetensors") + + # Handling conditional parameters based on mode + if mode == "fixed": + run_cmd.append("--linear_dim") + run_cmd.append(str(linear_dim)) + run_cmd.append("--conv_dim") + run_cmd.append(str(conv_dim)) + elif mode == "threshold": + run_cmd.append("--linear_threshold") + run_cmd.append(str(linear_threshold)) + run_cmd.append("--conv_threshold") + run_cmd.append(str(conv_threshold)) + elif mode == "ratio": + run_cmd.append("--linear_ratio") + run_cmd.append(str(linear_ratio)) + run_cmd.append("--conv_ratio") + run_cmd.append(str(conv_ratio)) + elif mode == "quantile": + run_cmd.append("--linear_quantile") + run_cmd.append(str(linear_quantile)) + run_cmd.append("--conv_quantile") + run_cmd.append(str(conv_quantile)) + + if use_sparse_bias: + run_cmd.append("--use_sparse_bias") + + # Adding additional options + run_cmd.append("--sparsity") + run_cmd.append(str(sparsity)) + + if disable_cp: + run_cmd.append("--disable_cp") + + # Add paths + run_cmd.append(base_model) + run_cmd.append(db_model) + run_cmd.append(output_name) + + # Log the command + log.info(" ".join(run_cmd)) env = os.environ.copy() env["PYTHONPATH"] = ( rf"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}" ) + # Adding an example of an environment variable that might be relevant + env["TF_ENABLE_ONEDNN_OPTS"] = "0" # Run the command subprocess.run(run_cmd, env=env) diff --git a/kohya_gui/finetune_gui.py b/kohya_gui/finetune_gui.py index d4906a6..94b2f40 100644 --- a/kohya_gui/finetune_gui.py +++ b/kohya_gui/finetune_gui.py @@ -3,9 +3,11 @@ import json import math import os import subprocess +import shlex import sys from datetime import datetime from .common_gui import ( + get_executable_path, get_file_path, get_saveasfile_path, run_cmd_advanced_training, @@ -497,62 +499,81 @@ def train_model( else: # create caption json file if generate_caption_database: - run_cmd = rf'"{PYTHON}" "{scriptdir}/sd-scripts/finetune/merge_captions_to_metadata.py"' + # Define the command components + run_cmd = [ + PYTHON, f"{scriptdir}/sd-scripts/finetune/merge_captions_to_metadata.py" + ] + + # Add the caption extension + run_cmd.append('--caption_extension') if caption_extension == "": - run_cmd += f' --caption_extension=".caption"' + run_cmd.append('.caption') # Default extension else: - run_cmd += f" --caption_extension={caption_extension}" - run_cmd += rf' "{image_folder}"' - run_cmd += rf' "{train_dir}/{caption_metadata_filename}"' + run_cmd.append(caption_extension) + + # Add paths for the image folder and the caption metadata file + run_cmd.append(image_folder) + run_cmd.append(os.path.join(train_dir, caption_metadata_filename)) + + # Include the full path flag if specified if full_path: - run_cmd += f" --full_path" + run_cmd.append("--full_path") - log.info(run_cmd) + # Log the built command + log.info(' '.join(run_cmd)) + # Prepare environment variables env = os.environ.copy() env["PYTHONPATH"] = ( - rf"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}" + f"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}" ) env["TF_ENABLE_ONEDNN_OPTS"] = "0" + # Execute the command if not in print-only mode if not print_only: - # Run the command subprocess.run(run_cmd, env=env) + # create images buckets if generate_image_buckets: - run_cmd = rf'"{PYTHON}" "{scriptdir}/sd-scripts/finetune/prepare_buckets_latents.py"' - run_cmd += rf' "{image_folder}"' - run_cmd += rf' "{train_dir}/{caption_metadata_filename}"' - run_cmd += rf' "{train_dir}/{latent_metadata_filename}"' - run_cmd += rf' "{pretrained_model_name_or_path}"' - run_cmd += f" --batch_size={batch_size}" - run_cmd += f" --max_resolution={max_resolution}" - run_cmd += f" --min_bucket_reso={min_bucket_reso}" - run_cmd += f" --max_bucket_reso={max_bucket_reso}" - run_cmd += f" --mixed_precision={mixed_precision}" - # if flip_aug: - # run_cmd += f' --flip_aug' + # Build the command to run the preparation script + run_cmd = [ + PYTHON, + f"{scriptdir}/sd-scripts/finetune/prepare_buckets_latents.py", + image_folder, + os.path.join(train_dir, caption_metadata_filename), + os.path.join(train_dir, latent_metadata_filename), + pretrained_model_name_or_path, + '--batch_size', str(batch_size), + '--max_resolution', str(max_resolution), + '--min_bucket_reso', str(min_bucket_reso), + '--max_bucket_reso', str(max_bucket_reso), + '--mixed_precision', str(mixed_precision) + ] + + # Conditional flags if full_path: - run_cmd += f" --full_path" + run_cmd.append('--full_path') if sdxl_checkbox and sdxl_no_half_vae: - log.info( - "Using mixed_precision = no because no half vae is selected..." - ) - run_cmd += f' --mixed_precision="no"' + log.info("Using mixed_precision = no because no half vae is selected...") + # Ensure 'no' is correctly handled without extra quotes that might be interpreted literally in command line + run_cmd.append('--mixed_precision=no') - log.info(run_cmd) + # Log the complete command as a string for clarity + log.info(' '.join(run_cmd)) + # Copy and modify environment variables env = os.environ.copy() env["PYTHONPATH"] = ( - rf"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}" + f"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}" ) env["TF_ENABLE_ONEDNN_OPTS"] = "0" + # Execute the command if not just for printing if not print_only: - # Run the command subprocess.run(run_cmd, env=env) + if image_folder == "": log.error("Image folder dir is empty") return TRAIN_BUTTON_VISIBLE @@ -592,9 +613,10 @@ def train_model( lr_warmup_steps = 0 log.info(f"lr_warmup_steps = {lr_warmup_steps}") - run_cmd = "accelerate launch" + run_cmd = ["accelerate", "launch"] - run_cmd += AccelerateLaunch.run_cmd( + run_cmd = AccelerateLaunch.run_cmd( + run_cmd=run_cmd, num_processes=num_processes, num_machines=num_machines, multi_gpu=multi_gpu, @@ -606,14 +628,14 @@ def train_model( ) if sdxl_checkbox: - run_cmd += rf' "{scriptdir}/sd-scripts/sdxl_train.py"' + run_cmd.append(f'{scriptdir}/sd-scripts/sdxl_train.py') else: - run_cmd += rf' "{scriptdir}/sd-scripts/fine_tune.py"' + run_cmd.append(f'{scriptdir}/sd-scripts/fine_tune.py') in_json = ( - rf"{train_dir}/{latent_metadata_filename}" + f"{train_dir}/{latent_metadata_filename}" if use_latent_files == "Yes" - else rf"{train_dir}/{caption_metadata_filename}" + else f"{train_dir}/{caption_metadata_filename}" ) cache_text_encoder_outputs = sdxl_checkbox and sdxl_cache_text_encoder_outputs no_half_vae = sdxl_checkbox and sdxl_no_half_vae @@ -715,9 +737,10 @@ def train_model( kwargs_for_training["learning_rate_te"] = learning_rate_te # Pass the dynamically constructed keyword arguments to the function - run_cmd += run_cmd_advanced_training(**kwargs_for_training) + run_cmd = run_cmd_advanced_training(run_cmd=run_cmd, **kwargs_for_training) - run_cmd += run_cmd_sample( + run_cmd = run_cmd_sample( + run_cmd, sample_every_n_steps, sample_every_n_epochs, sample_sampler, @@ -729,9 +752,12 @@ def train_model( log.warning( "Here is the trainer command as a reference. It will not be executed:\n" ) - print(run_cmd) + # Reconstruct the safe command string for display + command_to_run = ' '.join(run_cmd) + + print(command_to_run) - save_to_file(run_cmd) + save_to_file(command_to_run) else: # Saving config file for model current_datetime = datetime.now() @@ -747,7 +773,7 @@ def train_model( exclusion=["file_path", "save_as", "headless", "print_only"], ) - log.info(run_cmd) + # log.info(run_cmd) env = os.environ.copy() env["PYTHONPATH"] = ( diff --git a/kohya_gui/git_caption_gui.py b/kohya_gui/git_caption_gui.py index e2538b2..e413d24 100644 --- a/kohya_gui/git_caption_gui.py +++ b/kohya_gui/git_caption_gui.py @@ -33,22 +33,41 @@ def caption_images( return log.info(f"GIT captioning files in {train_data_dir}...") - run_cmd = rf'"{PYTHON}" "{scriptdir}/sd-scripts/finetune/make_captions_by_git.py"' - if not model_id == "": - run_cmd += f' --model_id="{model_id}"' - run_cmd += f' --batch_size="{int(batch_size)}"' - run_cmd += f' --max_data_loader_n_workers="{int(max_data_loader_n_workers)}"' - run_cmd += f' --max_length="{int(max_length)}"' - if caption_ext != "": - run_cmd += f' --caption_extension="{caption_ext}"' - run_cmd += f' "{train_data_dir}"' - log.info(run_cmd) + run_cmd = [PYTHON, f"{scriptdir}/sd-scripts/finetune/make_captions_by_git.py"] + + # Add --model_id if provided + if model_id != "": + run_cmd.append("--model_id") + run_cmd.append(model_id) + + # Add other arguments with their values + run_cmd.append("--batch_size") + run_cmd.append(str(batch_size)) + + run_cmd.append("--max_data_loader_n_workers") + run_cmd.append(str(max_data_loader_n_workers)) + + run_cmd.append("--max_length") + run_cmd.append(str(max_length)) + + # Add --caption_extension if provided + if caption_ext != "": + run_cmd.append("--caption_extension") + run_cmd.append(caption_ext) + + # Add the directory containing the training data + run_cmd.append(train_data_dir) + + # Log the command + log.info(" ".join(run_cmd)) env = os.environ.copy() env["PYTHONPATH"] = ( rf"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}" ) + # Adding an example of an environment variable that might be relevant + env["TF_ENABLE_ONEDNN_OPTS"] = "0" # Run the command subprocess.run(run_cmd, env=env) diff --git a/kohya_gui/group_images_gui.py b/kohya_gui/group_images_gui.py index 5219fb8..9536dd5 100644 --- a/kohya_gui/group_images_gui.py +++ b/kohya_gui/group_images_gui.py @@ -32,25 +32,35 @@ def group_images( log.info(f"Grouping images in {input_folder}...") - run_cmd = rf'"{PYTHON}" "{scriptdir}/tools/group_images.py"' - run_cmd += f' "{input_folder}"' - run_cmd += f' "{output_folder}"' - run_cmd += f" {(group_size)}" - if include_subfolders: - run_cmd += f" --include_subfolders" - if do_not_copy_other_files: - run_cmd += f" --do_not_copy_other_files" - if generate_captions: - run_cmd += f" --caption" - if caption_ext: - run_cmd += f" --caption_ext={caption_ext}" + run_cmd = [ + PYTHON, + f"{scriptdir}/tools/group_images.py", + input_folder, + output_folder, + str(group_size), + ] - log.info(run_cmd) + if include_subfolders: + run_cmd.append("--include_subfolders") + + if do_not_copy_other_files: + run_cmd.append("--do_not_copy_other_files") + + if generate_captions: + run_cmd.append("--caption") + if caption_ext: + run_cmd.append("--caption_ext") + run_cmd.append(caption_ext) + + # Log the command + log.info(" ".join(run_cmd)) env = os.environ.copy() env["PYTHONPATH"] = ( - rf"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}" + rf"{scriptdir}{os.pathsep}{scriptdir}/tools{os.pathsep}{env.get('PYTHONPATH', '')}" ) + # Adding a common environmental setting as an example if it's missing in the original context + env["TF_ENABLE_ONEDNN_OPTS"] = "0" # Run the command subprocess.run(run_cmd, env=env) diff --git a/kohya_gui/lora_gui.py b/kohya_gui/lora_gui.py index 3c310f3..9ea0184 100644 --- a/kohya_gui/lora_gui.py +++ b/kohya_gui/lora_gui.py @@ -2,8 +2,11 @@ import gradio as gr import json import math import os +import shlex + from datetime import datetime from .common_gui import ( + get_executable_path, get_file_path, get_any_file_path, get_saveasfile_path, @@ -440,6 +443,11 @@ def open_configuration( # Proceed if the file path is valid (not empty or None) if not file_path == "" and not file_path == None: + # Check if the file exists before opening it + if not os.path.isfile(file_path): + log.error(f"Config file {file_path} does not exist.") + return + # Load variables from JSON file with open(file_path, "r") as f: my_data = json.load(f) @@ -785,9 +793,10 @@ def train_model( lr_warmup_steps = 0 log.info(f"lr_warmup_steps = {lr_warmup_steps}") - run_cmd = "accelerate launch" + run_cmd = ["accelerate", "launch"] - run_cmd += AccelerateLaunch.run_cmd( + run_cmd = AccelerateLaunch.run_cmd( + run_cmd=run_cmd, num_processes=num_processes, num_machines=num_machines, multi_gpu=multi_gpu, @@ -799,9 +808,9 @@ def train_model( ) if sdxl: - run_cmd += rf' "{scriptdir}/sd-scripts/sdxl_train_network.py"' + run_cmd.append(f'{scriptdir}/sd-scripts/sdxl_train_network.py') else: - run_cmd += rf' "{scriptdir}/sd-scripts/train_network.py"' + run_cmd.append(f'{scriptdir}/sd-scripts/train_network.py') network_args = "" @@ -1031,9 +1040,10 @@ def train_model( } # Use the ** syntax to unpack the dictionary when calling the function - run_cmd += run_cmd_advanced_training(**run_cmd_params) + run_cmd = run_cmd_advanced_training(run_cmd=run_cmd, **run_cmd_params) - run_cmd += run_cmd_sample( + run_cmd = run_cmd_sample( + run_cmd, sample_every_n_steps, sample_every_n_epochs, sample_sampler, @@ -1045,9 +1055,12 @@ def train_model( log.warning( "Here is the trainer command as a reference. It will not be executed:\n" ) - print(run_cmd) + # Reconstruct the safe command string for display + command_to_run = ' '.join(run_cmd) + + print(command_to_run) - save_to_file(run_cmd) + save_to_file(command_to_run) else: # Saving config file for model current_datetime = datetime.now() @@ -1063,7 +1076,7 @@ def train_model( exclusion=["file_path", "save_as", "headless", "print_only"], ) - log.info(run_cmd) + # log.info(run_cmd) env = os.environ.copy() env["PYTHONPATH"] = ( rf"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}" diff --git a/kohya_gui/merge_lora_gui.py b/kohya_gui/merge_lora_gui.py index 75f638d..0196217 100644 --- a/kohya_gui/merge_lora_gui.py +++ b/kohya_gui/merge_lora_gui.py @@ -422,34 +422,40 @@ class GradioMergeLoRaTab: return if not sdxl_model: - run_cmd = rf'"{PYTHON}" "{scriptdir}/sd-scripts/networks/merge_lora.py"' + run_cmd = [PYTHON, f"{scriptdir}/sd-scripts/networks/merge_lora.py"] else: - run_cmd = ( - rf'"{PYTHON}" "{scriptdir}/sd-scripts/networks/sdxl_merge_lora.py"' - ) + run_cmd = [PYTHON, f"{scriptdir}/sd-scripts/networks/sdxl_merge_lora.py"] + if sd_model: - run_cmd += rf' --sd_model "{sd_model}"' - run_cmd += f" --save_precision {save_precision}" - run_cmd += f" --precision {precision}" - run_cmd += rf' --save_to "{save_to}"' + run_cmd.append("--sd_model") + run_cmd.append(sd_model) - # Create a space-separated string of non-empty models (from the second element onwards), enclosed in double quotes - models_cmd = " ".join([rf'"{model}"' for model in lora_models if model]) + run_cmd.extend(["--save_precision", save_precision]) + run_cmd.extend(["--precision", precision]) + run_cmd.append("--save_to") + run_cmd.append(save_to) - # Create a space-separated string of non-zero ratios corresponding to non-empty LoRa models + # Prepare model and ratios command as lists, including only non-empty models + valid_models = [model for model in lora_models if model] valid_ratios = [ratios[i] for i, model in enumerate(lora_models) if model] - ratios_cmd = " ".join([str(ratio) for ratio in valid_ratios]) - if models_cmd: - run_cmd += f" --models {models_cmd}" - run_cmd += f" --ratios {ratios_cmd}" + if valid_models: + run_cmd.append("--models") + run_cmd.extend(valid_models) # Each model is a separate argument + run_cmd.append("--ratios") + run_cmd.extend( + map(str, valid_ratios) + ) # Convert ratios to strings and include them as separate arguments - log.info(run_cmd) + # Log the command + log.info(" ".join(run_cmd)) env = os.environ.copy() env["PYTHONPATH"] = ( rf"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}" ) + # Example of adding an environment variable for TensorFlow, if necessary + env["TF_ENABLE_ONEDNN_OPTS"] = "0" # Run the command subprocess.run(run_cmd, env=env) diff --git a/kohya_gui/merge_lycoris_gui.py b/kohya_gui/merge_lycoris_gui.py index 3ddbc71..c958a7a 100644 --- a/kohya_gui/merge_lycoris_gui.py +++ b/kohya_gui/merge_lycoris_gui.py @@ -36,26 +36,37 @@ def merge_lycoris( ): log.info("Merge model...") - run_cmd = rf'"{PYTHON}" "{scriptdir}/tools/merge_lycoris.py"' - run_cmd += rf' "{base_model}"' - run_cmd += rf' "{lycoris_model}"' - run_cmd += rf' "{output_name}"' - run_cmd += f" --weight {weight}" - run_cmd += f" --device {device}" - run_cmd += f" --dtype {dtype}" + # Build the command to run merge_lycoris.py using list format + run_cmd = [ + PYTHON, + f"{scriptdir}/tools/merge_lycoris.py", + base_model, + lycoris_model, + output_name, + ] + + # Add additional required arguments with their values + run_cmd.extend(["--weight", str(weight)]) + run_cmd.extend(["--device", device]) + run_cmd.extend(["--dtype", dtype]) + + # Add optional flags based on conditions if is_sdxl: - run_cmd += f" --is_sdxl" + run_cmd.append("--is_sdxl") if is_v2: - run_cmd += f" --is_v2" + run_cmd.append("--is_v2") - log.info(run_cmd) + # Log the command + log.info(" ".join(run_cmd)) + # Copy and update the environment variables env = os.environ.copy() env["PYTHONPATH"] = ( rf"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}" ) + env["TF_ENABLE_ONEDNN_OPTS"] = "0" - # Run the command + # Execute the command with the modified environment subprocess.run(run_cmd, env=env) log.info("Done merging...") diff --git a/kohya_gui/resize_lora_gui.py b/kohya_gui/resize_lora_gui.py index 9253c0a..465acc9 100644 --- a/kohya_gui/resize_lora_gui.py +++ b/kohya_gui/resize_lora_gui.py @@ -63,25 +63,42 @@ def resize_lora( if device == "": device = "cuda" - run_cmd = rf'"{PYTHON}" "{scriptdir}/sd-scripts/networks/resize_lora.py"' - run_cmd += f" --save_precision {save_precision}" - run_cmd += rf' --save_to "{save_to}"' - run_cmd += rf' --model "{model}"' - run_cmd += f" --new_rank {new_rank}" - run_cmd += f" --device {device}" - if not dynamic_method == "None": - run_cmd += f" --dynamic_method {dynamic_method}" - run_cmd += f" --dynamic_param {dynamic_param}" - if verbose: - run_cmd += f" --verbose" + run_cmd = [ + PYTHON, + f"{scriptdir}/sd-scripts/networks/resize_lora.py", + "--save_precision", + save_precision, + "--save_to", + save_to, + "--model", + model, + "--new_rank", + str(new_rank), + "--device", + device, + ] - log.info(run_cmd) + # Conditional checks for dynamic parameters + if dynamic_method != "None": + run_cmd.extend( + ["--dynamic_method", dynamic_method, "--dynamic_param", str(dynamic_param)] + ) + + # Check for verbosity + if verbose: + run_cmd.append("--verbose") + + # Log the command + log.info(" ".join(run_cmd)) env = os.environ.copy() env["PYTHONPATH"] = ( rf"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}" ) + # Adding example environment variables if relevant + env["TF_ENABLE_ONEDNN_OPTS"] = "0" + # Run the command subprocess.run(run_cmd, env=env) diff --git a/kohya_gui/svd_merge_lora_gui.py b/kohya_gui/svd_merge_lora_gui.py index ebb4f08..dc1c78c 100644 --- a/kohya_gui/svd_merge_lora_gui.py +++ b/kohya_gui/svd_merge_lora_gui.py @@ -52,56 +52,60 @@ def svd_merge_lora( ratio_c /= total_ratio ratio_d /= total_ratio - run_cmd = rf'"{PYTHON}" "{scriptdir}/sd-scripts/networks/svd_merge_lora.py"' - run_cmd += f" --save_precision {save_precision}" - run_cmd += f" --precision {precision}" - run_cmd += rf' --save_to "{save_to}"' + run_cmd = [ + PYTHON, f"{scriptdir}/sd-scripts/networks/svd_merge_lora.py", + '--save_precision', save_precision, + '--precision', precision, + '--save_to', save_to + ] + + # Variables for model paths and their ratios + models = [] + ratios = [] - run_cmd_models = " --models" - run_cmd_ratios = " --ratios" # Add non-empty models and their ratios to the command - if lora_a_model: - if not os.path.isfile(lora_a_model): - msgbox("The provided model A is not a file") - return - run_cmd_models += rf' "{lora_a_model}"' - run_cmd_ratios += f" {ratio_a}" - if lora_b_model: - if not os.path.isfile(lora_b_model): - msgbox("The provided model B is not a file") - return - run_cmd_models += rf' "{lora_b_model}"' - run_cmd_ratios += f" {ratio_b}" - if lora_c_model: - if not os.path.isfile(lora_c_model): - msgbox("The provided model C is not a file") - return - run_cmd_models += rf' "{lora_c_model}"' - run_cmd_ratios += f" {ratio_c}" - if lora_d_model: - if not os.path.isfile(lora_d_model): - msgbox("The provided model D is not a file") - return - run_cmd_models += rf' "{lora_d_model}"' - run_cmd_ratios += f" {ratio_d}" + def add_model(model_path, ratio): + if not os.path.isfile(model_path): + msgbox(f"The provided model at {model_path} is not a file") + return False + models.append(model_path) + ratios.append(str(ratio)) + return True - run_cmd += run_cmd_models - run_cmd += run_cmd_ratios - run_cmd += f" --device {device}" - run_cmd += f' --new_rank "{new_rank}"' - run_cmd += f' --new_conv_rank "{new_conv_rank}"' + if lora_a_model and add_model(lora_a_model, ratio_a): + pass + if lora_b_model and add_model(lora_b_model, ratio_b): + pass + if lora_c_model and add_model(lora_c_model, ratio_c): + pass + if lora_d_model and add_model(lora_d_model, ratio_d): + pass - log.info(run_cmd) + if models and ratios: # Ensure we have valid models and ratios before appending + run_cmd.extend(['--models'] + models) + run_cmd.extend(['--ratios'] + ratios) + + run_cmd.extend([ + '--device', device, + '--new_rank', new_rank, + '--new_conv_rank', new_conv_rank + ]) + + # Log the command + log.info(' '.join(run_cmd)) env = os.environ.copy() env["PYTHONPATH"] = ( rf"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}" ) + # Example of setting additional environment variables if needed + env["TF_ENABLE_ONEDNN_OPTS"] = "0" # Run the command subprocess.run(run_cmd, env=env) + ### # Gradio UI ### diff --git a/kohya_gui/textual_inversion_gui.py b/kohya_gui/textual_inversion_gui.py index 7953fa7..003ba5a 100644 --- a/kohya_gui/textual_inversion_gui.py +++ b/kohya_gui/textual_inversion_gui.py @@ -2,8 +2,10 @@ import gradio as gr import json import math import os +import shlex from datetime import datetime from .common_gui import ( + get_executable_path, get_file_path, get_saveasfile_path, color_aug_changed, @@ -543,9 +545,10 @@ def train_model( lr_warmup_steps = 0 log.info(f"lr_warmup_steps = {lr_warmup_steps}") - run_cmd = "accelerate launch" + run_cmd = ["accelerate", "launch"] - run_cmd += AccelerateLaunch.run_cmd( + run_cmd = AccelerateLaunch.run_cmd( + run_cmd=run_cmd, num_processes=num_processes, num_machines=num_machines, multi_gpu=multi_gpu, @@ -557,11 +560,12 @@ def train_model( ) if sdxl: - run_cmd += rf' "{scriptdir}/sd-scripts/sdxl_train_textual_inversion.py"' + run_cmd.append(f'{scriptdir}/sd-scripts/sdxl_train_textual_inversion.py') else: - run_cmd += rf' "{scriptdir}/sd-scripts/train_textual_inversion.py"' - - run_cmd += run_cmd_advanced_training( + run_cmd.append(f'{scriptdir}/sd-scripts/train_textual_inversion.py') + + run_cmd = run_cmd_advanced_training( + run_cmd=run_cmd, adaptive_noise_scale=adaptive_noise_scale, bucket_no_upscale=bucket_no_upscale, bucket_reso_steps=bucket_reso_steps, @@ -647,17 +651,18 @@ def train_model( huber_schedule=huber_schedule, huber_c=huber_c, ) - run_cmd += f' --token_string="{token_string}"' - run_cmd += f' --init_word="{init_word}"' - run_cmd += f" --num_vectors_per_token={num_vectors_per_token}" + run_cmd.append(f'--token_string="{shlex.quote(token_string)}"') + run_cmd.append(f'--init_word="{shlex.quote(init_word)}"') + run_cmd.append(f"--num_vectors_per_token={int(num_vectors_per_token)}") if not weights == "": - run_cmd += f' --weights="{weights}"' + run_cmd.append(f'--weights="{shlex.quote(weights)}"') if template == "object template": - run_cmd += f" --use_object_template" + run_cmd.append("--use_object_template") elif template == "style template": - run_cmd += f" --use_style_template" + run_cmd.append("--use_style_template") - run_cmd += run_cmd_sample( + run_cmd = run_cmd_sample( + run_cmd, sample_every_n_steps, sample_every_n_epochs, sample_sampler, @@ -669,9 +674,12 @@ def train_model( log.warning( "Here is the trainer command as a reference. It will not be executed:\n" ) - print(run_cmd) + # Reconstruct the safe command string for display + command_to_run = ' '.join(run_cmd) + + print(command_to_run) - save_to_file(run_cmd) + save_to_file(command_to_run) else: # Saving config file for model current_datetime = datetime.now() @@ -687,7 +695,7 @@ def train_model( exclusion=["file_path", "save_as", "headless", "print_only"], ) - log.info(run_cmd) + # log.info(run_cmd) env = os.environ.copy() env["PYTHONPATH"] = ( diff --git a/kohya_gui/verify_lora_gui.py b/kohya_gui/verify_lora_gui.py index 7bc18df..0f153fc 100644 --- a/kohya_gui/verify_lora_gui.py +++ b/kohya_gui/verify_lora_gui.py @@ -35,16 +35,23 @@ def verify_lora( msgbox("The provided model A is not a file") return - run_cmd = rf'"{PYTHON}" "{scriptdir}/sd-scripts/networks/check_lora_weights.py" "{lora_model}"' + # Build the command to run check_lora_weights.py + run_cmd = [ + PYTHON, f"{scriptdir}/sd-scripts/networks/check_lora_weights.py", lora_model + ] - log.info(run_cmd) + # Log the command + log.info(' '.join(run_cmd)) + # Set the environment variable for the Python path env = os.environ.copy() env["PYTHONPATH"] = ( - rf"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}" + f"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}" ) + # Example of adding an environment variable for TensorFlow, if necessary + env["TF_ENABLE_ONEDNN_OPTS"] = "0" - # Run the command + # Run the command using subprocess.Popen for asynchronous handling process = subprocess.Popen( run_cmd, stdout=subprocess.PIPE, diff --git a/kohya_gui/wd14_caption_gui.py b/kohya_gui/wd14_caption_gui.py index c5b8269..b81e88f 100644 --- a/kohya_gui/wd14_caption_gui.py +++ b/kohya_gui/wd14_caption_gui.py @@ -45,56 +45,79 @@ def caption_images( return log.info(f"Captioning files in {train_data_dir}...") - run_cmd = rf'accelerate launch "{scriptdir}/sd-scripts/finetune/tag_images_by_wd14_tagger.py"' - # if always_first_tags: - # run_cmd += f' --always_first_tags="{always_first_tags}"' - if append_tags: - run_cmd += f" --append_tags" - run_cmd += f" --batch_size={int(batch_size)}" - run_cmd += f' --caption_extension="{caption_extension}"' - run_cmd += f' --caption_separator="{caption_separator}"' - if character_tag_expand: - run_cmd += f" --character_tag_expand" - if not character_threshold == 0.35: - run_cmd += f" --character_threshold={character_threshold}" - if debug: - run_cmd += f" --debug" - if force_download: - run_cmd += f" --force_download" - if frequency_tags: - run_cmd += f" --frequency_tags" - if not general_threshold == 0.35: - run_cmd += f" --general_threshold={general_threshold}" - run_cmd += f' --max_data_loader_n_workers="{int(max_data_loader_n_workers)}"' - if onnx: - run_cmd += f" --onnx" - if recursive: - run_cmd += f" --recursive" - if remove_underscore: - run_cmd += f" --remove_underscore" - run_cmd += f' --repo_id="{repo_id}"' - if not tag_replacement == "": - run_cmd += f" --tag_replacement={tag_replacement}" - if not thresh == 0.35: - run_cmd += f" --thresh={thresh}" - if not undesired_tags == "": - run_cmd += f' --undesired_tags="{undesired_tags}"' - if use_rating_tags: - run_cmd += f" --use_rating_tags" - if use_rating_tags_as_last_tag: - run_cmd += f" --use_rating_tags_as_last_tag" - run_cmd += rf' "{train_data_dir}"' + run_cmd = [ + "accelerate", "launch", f"{scriptdir}/sd-scripts/finetune/tag_images_by_wd14_tagger.py" + ] - log.info(run_cmd) + # Uncomment and modify if needed + # if always_first_tags: + # run_cmd.append('--always_first_tags') + # run_cmd.append(always_first_tags) + + if append_tags: + run_cmd.append("--append_tags") + run_cmd.append("--batch_size") + run_cmd.append(str(int(batch_size))) + run_cmd.append("--caption_extension") + run_cmd.append(caption_extension) + run_cmd.append("--caption_separator") + run_cmd.append(caption_separator) + + if character_tag_expand: + run_cmd.append("--character_tag_expand") + if not character_threshold == 0.35: + run_cmd.append("--character_threshold") + run_cmd.append(str(character_threshold)) + if debug: + run_cmd.append("--debug") + if force_download: + run_cmd.append("--force_download") + if frequency_tags: + run_cmd.append("--frequency_tags") + if not general_threshold == 0.35: + run_cmd.append("--general_threshold") + run_cmd.append(str(general_threshold)) + run_cmd.append("--max_data_loader_n_workers") + run_cmd.append(str(int(max_data_loader_n_workers))) + + if onnx: + run_cmd.append("--onnx") + if recursive: + run_cmd.append("--recursive") + if remove_underscore: + run_cmd.append("--remove_underscore") + run_cmd.append("--repo_id") + run_cmd.append(repo_id) + if not tag_replacement == "": + run_cmd.append("--tag_replacement") + run_cmd.append(tag_replacement) + if not thresh == 0.35: + run_cmd.append("--thresh") + run_cmd.append(str(thresh)) + if not undesired_tags == "": + run_cmd.append("--undesired_tags") + run_cmd.append(undesired_tags) + if use_rating_tags: + run_cmd.append("--use_rating_tags") + if use_rating_tags_as_last_tag: + run_cmd.append("--use_rating_tags_as_last_tag") + + # Add the directory containing the training data + run_cmd.append(train_data_dir) + + # Log the command + log.info(' '.join(run_cmd)) env = os.environ.copy() env["PYTHONPATH"] = ( rf"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}" ) + # Adding an example of an environment variable that might be relevant env["TF_ENABLE_ONEDNN_OPTS"] = "0" # Run the command subprocess.run(run_cmd, env=env) + # Add prefix and postfix add_pre_postfix( diff --git a/requirements.txt b/requirements.txt index 4bb22c2..5f15341 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ easygui==0.98.3 einops==0.7.0 fairscale==0.4.13 ftfy==6.1.1 -gradio==4.19.2 +gradio==4.26.0 huggingface-hub==0.20.1 imagesize==1.4.1 invisible-watermark==0.2.0 diff --git a/test/config/LoKR-AdamW8bit-toml.json b/test/config/LoKR-AdamW8bit-toml.json index d99201c..2b58245 100644 --- a/test/config/LoKR-AdamW8bit-toml.json +++ b/test/config/LoKR-AdamW8bit-toml.json @@ -11,17 +11,17 @@ "bypass_mode": false, "cache_latents": true, "cache_latents_to_disk": true, - "caption_dropout_every_n_epochs": 0.0, + "caption_dropout_every_n_epochs": 0, "caption_dropout_rate": 0.1, "caption_extension": ".txt", "clip_skip": "1", "color_aug": false, - "constrain": 0.0, + "constrain": 0, "conv_alpha": 1, "conv_block_alphas": "", "conv_block_dims": "", "conv_dim": 100000, - "dataset_config": "D:/kohya_ss/test/config/dataset.toml", + "dataset_config": "./test/config/dataset.toml", "debiased_estimation_loss": false, "decompose_both": false, "dim_from_weights": false, @@ -29,6 +29,7 @@ "down_lr_weight": "", "enable_bucket": true, "epoch": 150, + "extra_accelerate_launch_args": "", "factor": 6, "flip_aug": false, "fp8_base": false, @@ -37,17 +38,24 @@ "gpu_ids": "", "gradient_accumulation_steps": 1, "gradient_checkpointing": false, + "huber_c": 0.1, + "huber_schedule": "snr", + "ip_noise_gamma": 0, + "ip_noise_gamma_random_strength": false, "keep_tokens": 1, - "learning_rate": 1.0, + "learning_rate": 1, "log_tracker_config": "", "log_tracker_name": "", "logging_dir": "./test/logs", "lora_network_weights": "", + "loss_type": "l2", "lr_scheduler": "cosine", "lr_scheduler_args": "", "lr_scheduler_num_cycles": "", "lr_scheduler_power": "", "lr_warmup": 0, + "main_process_port": 0, + "masked_loss": false, "max_bucket_reso": 2048, "max_data_loader_n_workers": "0", "max_grad_norm": 1, @@ -71,6 +79,7 @@ "network_dim": 100000, "network_dropout": 0, "noise_offset": 0, + "noise_offset_random_strength": false, "noise_offset_type": "Multires", "num_cpu_threads_per_process": 2, "num_machines": 1, @@ -81,7 +90,7 @@ "output_name": "LoKR-AdamW8bit-toml", "persistent_data_loader_workers": false, "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5", - "prior_loss_weight": 1.0, + "prior_loss_weight": 1, "random_crop": false, "rank_dropout": 0, "rank_dropout_scale": false, @@ -92,6 +101,7 @@ "sample_every_n_steps": 25, "sample_prompts": "a painting of a gas mask , by darius kawasaki", "sample_sampler": "euler_a", + "save_as_bool": false, "save_every_n_epochs": 15, "save_every_n_steps": 0, "save_last_n_steps": 0, @@ -99,6 +109,7 @@ "save_model_as": "safetensors", "save_precision": "bf16", "save_state": false, + "save_state_on_train_end": false, "scale_v_pred_loss_like_noise_pred": false, "scale_weight_norms": 0, "sdxl": false, @@ -107,13 +118,13 @@ "seed": "", "shuffle_caption": true, "stop_text_encoder_training": 0, - "text_encoder_lr": 1.0, + "text_encoder_lr": 1, "train_batch_size": 2, "train_data_dir": "", "train_norm": false, "train_on_input": false, "training_comment": "KoopaTroopa", - "unet_lr": 1.0, + "unet_lr": 1, "unit": 1, "up_lr_weight": "", "use_cp": false, diff --git a/test/config/finetune-AdamW.json b/test/config/finetune-AdamW.json index 6c6cdf1..5eaf96f 100644 --- a/test/config/finetune-AdamW.json +++ b/test/config/finetune-AdamW.json @@ -10,10 +10,10 @@ "caption_dropout_every_n_epochs": 0, "caption_dropout_rate": 0, "caption_extension": ".txt", - "caption_metadata_filename": "meta-1_cap.json", + "caption_metadata_filename": "meta-1_cap5.json", "clip_skip": 1, "color_aug": false, - "create_buckets": false, + "create_buckets": true, "create_caption": true, "dataset_config": "", "dataset_repeats": "50", @@ -28,11 +28,11 @@ "gradient_checkpointing": false, "huber_c": 0.1, "huber_schedule": "snr", - "image_folder": ".\\test\\img\\10_darius kawasaki person", + "image_folder": "./test/img/10_darius kawasaki person", "ip_noise_gamma": 0, "ip_noise_gamma_random_strength": false, "keep_tokens": 0, - "latent_metadata_filename": "meta-1_lat.json", + "latent_metadata_filename": "meta-1_lat5.json", "learning_rate": 1e-05, "learning_rate_te": 5e-06, "learning_rate_te1": 5e-06,