From 63efd3e0e374ebc2be730556d234dc31147018f4 Mon Sep 17 00:00:00 2001 From: Alexey Borsky Date: Sat, 13 May 2023 23:27:27 +0300 Subject: [PATCH] v0.9.1 --- readme.md | 8 +- scripts/base_ui.py | 215 +++++++++++++++++++++---------------- scripts/core/flow_utils.py | 14 --- scripts/core/txt2vid.py | 14 +-- scripts/core/utils.py | 2 +- scripts/core/vid2vid.py | 16 +-- 6 files changed, 135 insertions(+), 134 deletions(-) diff --git a/readme.md b/readme.md index 7e158ed..00e94da 100644 --- a/readme.md +++ b/readme.md @@ -56,10 +56,16 @@ All examples you can see here are originally generated at 512x512 resolution usi ## Installing the extension To install the extension go to 'Extensions' tab in [Automatic1111 web-ui](https://github.com/AUTOMATIC1111/stable-diffusion-webui), then go to 'Install from URL' tab. In 'URL for extension's git repository' field inter the path to this repository, i.e. 'https://github.com/volotat/SD-CN-Animation.git'. Leave 'Local directory name' field empty. Then just press 'Install' button. Restart web-ui, new 'SD-CN-Animation' tab should appear. All generated video will be saved into 'stable-diffusion-webui/outputs/sd-cn-animation' folder. +## Known issues +* If you see error like this ```IndexError: list index out of range``` try to restart webui, it should fix it. +* The extension might work incorrectly if 'Apply color correction to img2img results to match original colors.' option is enabled. Make sure to disable it in 'Settings' tab -> 'Stable Diffusion' section. + ## Last version changes: v0.9 -* Issue #76 fixed. +* Fixed issues #69, #76, #91. * Fixed an issue in vid2vid mode when an occlusion mask computed from the optical flow may include unnecessary parts (where flow is non-zero). * Added 'Extra params' in vid2vid mode for more fine-grain controls of the processing pipeline. * Better default parameters set for vid2vid pipeline. * In txt2vid mode after the first frame is generated the seed is now automatically set to -1 to prevent blurring issues. * Added an option to save resulting frames into a folder alongside the video. +* Added ability to export current parameters in a human readable form as a json. + diff --git a/scripts/base_ui.py b/scripts/base_ui.py index 29e7cb5..16894dc 100644 --- a/scripts/base_ui.py +++ b/scripts/base_ui.py @@ -1,16 +1,4 @@ import sys, os -basedirs = [os.getcwd()] - -for basedir in basedirs: - paths_to_ensure = [ - basedir, - basedir + '/extensions/sd-cn-animation/scripts', - basedir + '/extensions/SD-CN-Animation/scripts' - ] - - for scripts_path_fix in paths_to_ensure: - if not scripts_path_fix in sys.path: - sys.path.extend([scripts_path_fix]) import gradio as gr import modules @@ -27,7 +15,7 @@ import modules.scripts as scripts from modules.sd_samplers import samplers_for_img2img from modules.ui import setup_progressbar, create_sampler_and_steps_selection, ordered_ui_categories, create_output_panel -from core import vid2vid, txt2vid, utils +from scripts.core import vid2vid, txt2vid, utils import traceback def V2VArgs(): @@ -79,7 +67,7 @@ def inputs_ui(): v2v_args = SimpleNamespace(**V2VArgs()) t2v_args = SimpleNamespace(**T2VArgs()) with gr.Tabs(): - sdcn_process_mode = gr.State(value='vid2vid') + glo_sdcn_process_mode = gr.State(value='vid2vid') with gr.Tab('vid2vid') as tab_vid2vid: with gr.Row(): @@ -128,26 +116,26 @@ def inputs_ui(): t2v_fps = gr.Slider(label='Video FPS', minimum=4, maximum=64, step=4, value=12, interactive=True) - tab_vid2vid.select(fn=lambda: 'vid2vid', inputs=[], outputs=[sdcn_process_mode]) - tab_txt2vid.select(fn=lambda: 'txt2vid', inputs=[], outputs=[sdcn_process_mode]) + tab_vid2vid.select(fn=lambda: 'vid2vid', inputs=[], outputs=[glo_sdcn_process_mode]) + tab_txt2vid.select(fn=lambda: 'txt2vid', inputs=[], outputs=[glo_sdcn_process_mode]) return locals() def process(*args): msg = 'Done' try: - if args[0] == 'vid2vid': - yield from vid2vid.start_process(*args) - elif args[0] == 'txt2vid': - yield from txt2vid.start_process(*args) - else: - msg = f"Unsupported processing mode: '{args[0]}'" - raise Exception(msg) + if args[0] == 'vid2vid': + yield from vid2vid.start_process(*args) + elif args[0] == 'txt2vid': + yield from txt2vid.start_process(*args) + else: + msg = f"Unsupported processing mode: '{args[0]}'" + raise Exception(msg) except Exception as error: - # handle the exception - msg = f"An exception occurred while trying to process the frame: {error}" - print(msg) - traceback.print_exc() + # handle the exception + msg = f"An exception occurred while trying to process the frame: {error}" + print(msg) + traceback.print_exc() yield msg, gr.Image.update(), gr.Image.update(), gr.Image.update(), gr.Image.update(), gr.Video.update(), gr.Button.update(interactive=True), gr.Button.update(interactive=False) @@ -155,81 +143,126 @@ def stop_process(*args): utils.shared.is_interrupted = True return gr.Button.update(interactive=False) +import json +def get_json(obj): + return json.loads( + json.dumps(obj, default=lambda o: getattr(o, '__dict__', str(o))) + ) + +def export_settings(*args): + args_dict = utils.args_to_dict(*args) + if args[0] == 'vid2vid': + args_dict = utils.get_mode_args('v2v', args_dict) + elif args[0] == 'txt2vid': + args_dict = utils.get_mode_args('t2v', args_dict) + else: + msg = f"Unsupported processing mode: '{args[0]}'" + raise Exception(msg) + + # convert CN params into a readable dict + cn_remove_list = ['low_vram', 'is_ui', 'input_mode', 'batch_images', 'output_dir', 'loopback'] + + args_dict['ControlNets'] = [] + for script_input in args_dict['script_inputs']: + if type(script_input).__name__ == 'UiControlNetUnit': + cn_values_dict = get_json(script_input) + if cn_values_dict['enabled']: + for key in cn_remove_list: + if key in cn_values_dict: del cn_values_dict[key] + args_dict['ControlNets'].append(cn_values_dict) + + # remove unimportant values + remove_list = ['save_frames_check', 'restore_faces', 'prompt_styles', 'mask_blur', 'inpainting_fill', 'tiling', 'n_iter', 'batch_size', 'subseed', 'subseed_strength', 'seed_resize_from_h', \ + 'seed_resize_from_w', 'seed_enable_extras', 'resize_mode', 'inpaint_full_res', 'inpaint_full_res_padding', 'inpainting_mask_invert', 'file', 'denoising_strength', \ + 'override_settings', 'script_inputs', 'init_img', 'mask_img', 'mode', 'init_video'] + + for key in remove_list: + if key in args_dict: del args_dict[key] + + return json.dumps(args_dict, indent=2, default=lambda o: getattr(o, '__dict__', str(o))) + def on_ui_tabs(): - modules.scripts.scripts_current = modules.scripts.scripts_img2img - modules.scripts.scripts_img2img.initialize_scripts(is_img2img=True) + modules.scripts.scripts_current = modules.scripts.scripts_img2img + modules.scripts.scripts_img2img.initialize_scripts(is_img2img=True) - with gr.Blocks(analytics_enabled=False) as sdcnanim_interface: - components = {} - - #dv = SimpleNamespace(**T2VOutputArgs()) - with gr.Row(elem_id='sdcn-core').style(equal_height=False, variant='compact'): - with gr.Column(scale=1, variant='panel'): - with gr.Tabs(): - components = inputs_ui() + with gr.Blocks(analytics_enabled=False) as sdcnanim_interface: + components = {} - with gr.Column(scale=1, variant='compact'): - with gr.Row(variant='compact'): - run_button = gr.Button('Generate', elem_id=f"sdcn_anim_generate", variant='primary') - stop_button = gr.Button('Interrupt', elem_id=f"sdcn_anim_interrupt", variant='primary', interactive=False) - - save_frames_check = gr.Checkbox(label="Save frames into a folder nearby a video (check it before running the generation if you also want to save frames separately)", value=False, interactive=True) - gr.HTML('
') + #dv = SimpleNamespace(**T2VOutputArgs()) + with gr.Row(elem_id='sdcn-core').style(equal_height=False, variant='compact'): + with gr.Column(scale=1, variant='panel'): + #with gr.Tabs(): + components = inputs_ui() + + with gr.Accordion("Export settings", open=False): + export_settings_button = gr.Button('Export', elem_id=f"sdcn_export_settings_button") + export_setting_json = gr.JSON(value='{}') - with gr.Column(variant="panel"): - sp_progress = gr.HTML(elem_id="sp_progress", value="") - sp_progress.update() - #sp_outcome = gr.HTML(elem_id="sp_error", value="") - #sp_progressbar = gr.HTML(elem_id="sp_progressbar") - #setup_progressbar(sp_progressbar, sp_preview, 'sp', textinfo=sp_progress) - - with gr.Row(variant='compact'): - img_preview_curr_frame = gr.Image(label='Current frame', elem_id=f"img_preview_curr_frame", type='pil').style(height=240) - img_preview_curr_occl = gr.Image(label='Current occlusion', elem_id=f"img_preview_curr_occl", type='pil').style(height=240) - with gr.Row(variant='compact'): - img_preview_prev_warp = gr.Image(label='Previous frame warped', elem_id=f"img_preview_curr_frame", type='pil').style(height=240) - img_preview_processed = gr.Image(label='Processed', elem_id=f"img_preview_processed", type='pil').style(height=240) - - # html_log = gr.HTML(elem_id=f'html_log_vid2vid') - video_preview = gr.Video(interactive=False) - - with gr.Row(variant='compact'): - dummy_component = gr.Label(visible=False) - components['glo_save_frames_check'] = save_frames_check + with gr.Column(scale=1, variant='compact'): + with gr.Row(variant='compact'): + run_button = gr.Button('Generate', elem_id=f"sdcn_anim_generate", variant='primary') + stop_button = gr.Button('Interrupt', elem_id=f"sdcn_anim_interrupt", variant='primary', interactive=False) + + save_frames_check = gr.Checkbox(label="Save frames into a folder nearby a video (check it before running the generation if you also want to save frames separately)", value=False, interactive=True) + gr.HTML('
') - # Define parameters for the action methods. - method_inputs = [components[name] for name in utils.get_component_names()] + components['v2v_custom_inputs'] + with gr.Column(variant="panel"): + sp_progress = gr.HTML(elem_id="sp_progress", value="") + + with gr.Row(variant='compact'): + img_preview_curr_frame = gr.Image(label='Current frame', elem_id=f"img_preview_curr_frame", type='pil').style(height=240) + img_preview_curr_occl = gr.Image(label='Current occlusion', elem_id=f"img_preview_curr_occl", type='pil').style(height=240) + with gr.Row(variant='compact'): + img_preview_prev_warp = gr.Image(label='Previous frame warped', elem_id=f"img_preview_curr_frame", type='pil').style(height=240) + img_preview_processed = gr.Image(label='Processed', elem_id=f"img_preview_processed", type='pil').style(height=240) - method_outputs = [ - sp_progress, - img_preview_curr_frame, - img_preview_curr_occl, - img_preview_prev_warp, - img_preview_processed, - video_preview, - run_button, - stop_button, - ] + video_preview = gr.Video(interactive=False) + + with gr.Row(variant='compact'): + dummy_component = gr.Label(visible=False) - run_button.click( - fn=process, #wrap_gradio_gpu_call(start_process, extra_outputs=[None, '', '']), - inputs=method_inputs, - outputs=method_outputs, - show_progress=True, - ) + components['glo_save_frames_check'] = save_frames_check + + # Define parameters for the action methods. + method_inputs = [components[name] for name in utils.get_component_names()] + components['v2v_custom_inputs'] - stop_button.click( - fn=stop_process, - outputs=[stop_button], - show_progress=False - ) + method_outputs = [ + sp_progress, + img_preview_curr_frame, + img_preview_curr_occl, + img_preview_prev_warp, + img_preview_processed, + video_preview, + run_button, + stop_button, + ] - modules.scripts.scripts_current = None + run_button.click( + fn=process, #wrap_gradio_gpu_call(start_process, extra_outputs=[None, '', '']), + inputs=method_inputs, + outputs=method_outputs, + show_progress=True, + ) - # define queue - required for generators - sdcnanim_interface.queue(concurrency_count=1) - return [(sdcnanim_interface, "SD-CN-Animation", "sd_cn_animation_interface")] + stop_button.click( + fn=stop_process, + outputs=[stop_button], + show_progress=False + ) + + export_settings_button.click( + fn=export_settings, + inputs=method_inputs, + outputs=[export_setting_json], + show_progress=False + ) + + modules.scripts.scripts_current = None + + # define queue - required for generators + sdcnanim_interface.queue(concurrency_count=1) + return [(sdcnanim_interface, "SD-CN-Animation", "sd_cn_animation_interface")] script_callbacks.on_ui_tabs(on_ui_tabs) diff --git a/scripts/core/flow_utils.py b/scripts/core/flow_utils.py index f921af6..95750a1 100644 --- a/scripts/core/flow_utils.py +++ b/scripts/core/flow_utils.py @@ -1,18 +1,4 @@ import sys, os -basedirs = [os.getcwd()] - -for basedir in basedirs: - paths_to_ensure = [ - basedir, - basedir + '/extensions/sd-cn-animation/scripts', - basedir + '/extensions/SD-CN-Animation/scripts', - basedir + '/extensions/sd-cn-animation/RAFT', - basedir + '/extensions/SD-CN-Animation/RAFT' - ] - - for scripts_path_fix in paths_to_ensure: - if not scripts_path_fix in sys.path: - sys.path.extend([scripts_path_fix]) import numpy as np import cv2 diff --git a/scripts/core/txt2vid.py b/scripts/core/txt2vid.py index deb32c6..062f11f 100644 --- a/scripts/core/txt2vid.py +++ b/scripts/core/txt2vid.py @@ -1,16 +1,4 @@ import sys, os -basedirs = [os.getcwd()] - -for basedir in basedirs: - paths_to_ensure = [ - basedir, - basedir + '/extensions/sd-cn-animation/scripts', - basedir + '/extensions/SD-CN-Animation/scripts' - ] - - for scripts_path_fix in paths_to_ensure: - if not scripts_path_fix in sys.path: - sys.path.extend([scripts_path_fix]) import torch import gc @@ -20,7 +8,7 @@ from PIL import Image import modules.paths as ph from modules.shared import devices -from core import utils, flow_utils +from scripts.core import utils, flow_utils from FloweR.model import FloweR import skimage diff --git a/scripts/core/utils.py b/scripts/core/utils.py index 5ff777b..4d4cd16 100644 --- a/scripts/core/utils.py +++ b/scripts/core/utils.py @@ -3,7 +3,7 @@ class shared: def get_component_names(): components_list = [ - 'sdcn_process_mode', + 'glo_sdcn_process_mode', 'v2v_file', 'v2v_width', 'v2v_height', 'v2v_prompt', 'v2v_n_prompt', 'v2v_cfg_scale', 'v2v_seed', 'v2v_processing_strength', 'v2v_fix_frame_strength', 'v2v_sampler_index', 'v2v_steps', 'v2v_override_settings', 'v2v_occlusion_mask_blur', 'v2v_occlusion_mask_trailing', 'v2v_occlusion_mask_flow_multiplier', 'v2v_occlusion_mask_difo_multiplier', 'v2v_occlusion_mask_difs_multiplier', diff --git a/scripts/core/vid2vid.py b/scripts/core/vid2vid.py index 038489f..c9e8140 100644 --- a/scripts/core/vid2vid.py +++ b/scripts/core/vid2vid.py @@ -1,16 +1,4 @@ import sys, os -basedirs = [os.getcwd()] - -for basedir in basedirs: - paths_to_ensure = [ - basedir, - basedir + '/extensions/sd-cn-animation/scripts', - basedir + '/extensions/SD-CN-Animation/scripts' - ] - - for scripts_path_fix in paths_to_ensure: - if not scripts_path_fix in sys.path: - sys.path.extend([scripts_path_fix]) import math import os @@ -34,8 +22,8 @@ import time import skimage import datetime -from core.flow_utils import RAFT_estimate_flow, RAFT_clear_memory, compute_diff_map -from core import utils +from scripts.core.flow_utils import RAFT_estimate_flow, RAFT_clear_memory, compute_diff_map +from scripts.core import utils class sdcn_anim_tmp: prepear_counter = 0