Add new options for extract lora

library/extract_lora_gui.py

@@ -32,6 +32,9 @@ def extract_lora(
     clamp_quantile,
     min_diff,
     device,
+    load_original_model_to,
+    load_tuned_model_to,
+    load_precision,
 ):
     # Check for caption_text_input
     if model_tuned == '':
@@ -57,6 +60,7 @@ def extract_lora(
     run_cmd = (
         f'{PYTHON} "{os.path.join("networks","extract_lora_from_models.py")}"'
     )
+    run_cmd += f' --load_precision {load_precision}'
     run_cmd += f' --save_precision {save_precision}'
     run_cmd += f' --save_to "{save_to}"'
     run_cmd += f' --model_org "{model_org}"'
@@ -71,6 +75,9 @@ def extract_lora(
         run_cmd += f' --sdxl'
     run_cmd += f' --clamp_quantile {clamp_quantile}'
     run_cmd += f' --min_diff {min_diff}'
+    if sdxl:
+        run_cmd += f' --load_original_model_to {load_original_model_to}'
+        run_cmd += f' --load_tuned_model_to {load_tuned_model_to}'
 
     log.info(run_cmd)
 
@@ -87,6 +94,11 @@ def extract_lora(
 
 
 def gradio_extract_lora_tab(headless=False):
+    def change_sdxl(sdxl):
+        return gr.update(visible=sdxl), gr.update(visible=sdxl)
+            
+    
+    
     with gr.Tab('Extract LoRA'):
         gr.Markdown(
             'This utility can extract a LoRA network from a finetuned model.'
@@ -113,7 +125,15 @@ def gradio_extract_lora_tab(headless=False):
                 outputs=model_tuned,
                 show_progress=False,
             )
-
+            load_tuned_model_to = gr.Dropdown(
+                label='Load finetuned model to',
+                choices=['cpu', 'cuda', 'cuda:0'],
+                value='cpu',
+                interactive=True, scale=1,
+                info="only for SDXL",
+                visible=False,
+            )
+        with gr.Row():
             model_org = gr.Textbox(
                 label='Stable Diffusion base model',
                 placeholder='Stable Diffusion original model: ckpt or safetensors file',
@@ -130,11 +150,20 @@ def gradio_extract_lora_tab(headless=False):
                 outputs=model_org,
                 show_progress=False,
             )
+            load_original_model_to = gr.Dropdown(
+                label='Load Stable Diffusion base model to',
+                choices=['cpu', 'cuda', 'cuda:0'],
+                value='cpu',
+                interactive=True, scale=1,
+                info="only for SDXL",
+                visible=False,
+            )
         with gr.Row():
             save_to = gr.Textbox(
                 label='Save to',
                 placeholder='path where to save the extracted LoRA model...',
                 interactive=True,
+                scale=2,
             )
             button_save_to = gr.Button(
                 folder_symbol,
@@ -150,8 +179,14 @@ def gradio_extract_lora_tab(headless=False):
             save_precision = gr.Dropdown(
                 label='Save precision',
                 choices=['fp16', 'bf16', 'float'],
-                value='float',
-                interactive=True,
+                value='fp16',
+                interactive=True, scale=1,
+            )
+            load_precision = gr.Dropdown(
+                label='Load precision',
+                choices=['fp16', 'bf16', 'float'],
+                value='fp16',
+                interactive=True, scale=1,
             )
         with gr.Row():
             dim = gr.Slider(
@@ -192,6 +227,8 @@ def gradio_extract_lora_tab(headless=False):
                 value='cuda',
                 interactive=True,
             )
+            
+            sdxl.change(change_sdxl, inputs=sdxl, outputs=[load_tuned_model_to, load_original_model_to])
 
         extract_button = gr.Button('Extract LoRA model')
 
@@ -209,6 +246,9 @@ def gradio_extract_lora_tab(headless=False):
                 clamp_quantile,
                 min_diff,
                 device,
+                load_original_model_to,
+                load_tuned_model_to,
+                load_precision,
             ],
             show_progress=False,
         )

train_network_appl_weights_README.md

@@ -0,0 +1,39 @@
+# Exploring Layer-Specific Application Rates for LoRA
+## Introduction
+Added a tool, train_network_appl_weights.py, for exploring layer-specific application rates. Currently, it supports SDXL only.
+
+## Concept
+The process involves running the standard training process with varying layer-specific application rates on trained networks like LoRA. The goal is to explore which rates produce images closest to the training data.
+
+## Penalty for Total Application Rates
+It's possible to use the total of the layer-specific application rates as a penalty, aiming to reproduce images while minimizing the impact of less significant layers.
+
+## Multi-Network Exploration
+The exploration can be conducted on multiple networks and requires at least one piece of training data.
+
+Note: The effectiveness with a specific number of images has not been confirmed, but it has been tested with approximately 50 images. The training data does not necessarily have to be from LoRA's training phase, although this has not been confirmed.
+
+## Command Line Options
+The command line options are almost identical to those for `sdxl_train_network.py`, with the following additions and extensions:
+
+- `--application_loss_weight`: Weight of the layer-specific application rate when added to the loss. Default is 0.0001. Increasing this value trains the model to minimize the application rates. Setting it to 0 allows free exploration of the application rates that yield the highest fidelity.
+- `--network_module`: Allows specifying multiple modules for exploration, e.g., `--network_module networks.lora networks.lora`.
+- `--network_weights`: Allows specifying weights for multiple networks to be explored, e.g., `--network_weights model1.safetensors model2.safetensors`.
+
+## Parameters
+The number of parameters for layer-specific application rates is 20, including BASE, IN00-08, MID, OUT00-08. BASE is applied to the Text Encoder (Note: LoRA's operation on the Text Encoder has not been confirmed).
+
+Although the parameters are saved to a file, it's recommended to copy and save the values displayed on the screen.
+
+## Remarks
+Confirmed to work with the AdamW optimizer and a learning rate of 1e-1. The learning rate can be set quite high. With this setting, reasonable results can be obtained in about 1/20 to 1/10 the epochs used during LoRA training.
+Increasing `application_loss_weight` above 0.0001 significantly reduces the total application rate, meaning LoRA is applied less. Adjust as needed.
+Using negative values for the application rate can lead to minimizing the total by excessively reducing less influential layers' application rates. Negative values are weighted ten times (e.g., -0.01 is almost the same penalty as 0.1). Modify the source code to change the weighting.
+
+## Potential Uses
+Beyond reducing unnecessary layers' application rates, potential uses include:
+
+- Searching for LoRA application rates to maintain a character while changing their pose based on a reference image.
+- Exploring application rates for LoRA to maintain a character's style while altering the artistic style of the image.
+- Exploring necessary layers to reproduce a character's attributes using an image in a different style as training data.
+- Applying numerous LoRAs to an ideal image as training data and searching for the application rates that achieve the highest fidelity (though more LoRAs will slow down the training).