From 9cedc48739c72396a50c1b4051f9a981089ebfce Mon Sep 17 00:00:00 2001 From: rewbs Date: Mon, 26 Jun 2023 19:32:37 +1000 Subject: [PATCH 1/8] Add optional preview video render which generates the vid every N frames (either sync or async. Minor: improve error message if hybrid video finds no frames to use. Minor: include 2D centre transform x/y in the console log along side other anim params. --- scripts/deforum_helpers/generate.py | 6 ++-- scripts/deforum_helpers/hybrid_video.py | 2 ++ scripts/deforum_helpers/render.py | 6 +++- scripts/deforum_helpers/render_modes.py | 4 ++- scripts/deforum_helpers/run_deforum.py | 19 ++--------- scripts/deforum_helpers/ui_settings.py | 4 ++- .../deforum_helpers/video_audio_utilities.py | 34 +++++++++++++++++++ 7 files changed, 54 insertions(+), 21 deletions(-) diff --git a/scripts/deforum_helpers/generate.py b/scripts/deforum_helpers/generate.py index b536995d..dad76b2d 100644 --- a/scripts/deforum_helpers/generate.py +++ b/scripts/deforum_helpers/generate.py @@ -276,7 +276,7 @@ def print_combined_table(args, anim_args, p, keys, frame_idx): rows2 = [] if anim_args.animation_mode not in ['Video Input', 'Interpolation']: if anim_args.animation_mode == '2D': - field_names2 = ["Angle", "Zoom"] + field_names2 = ["Angle", "Zoom", "Tr C X", "Tr C Y"] else: field_names2 = [] field_names2 += ["Tr X", "Tr Y"] @@ -291,7 +291,9 @@ def print_combined_table(args, anim_args, p, keys, frame_idx): table.add_column(field_name, justify="center") if anim_args.animation_mode == '2D': - rows2 += [f"{keys.angle_series[frame_idx]:.5g}", f"{keys.zoom_series[frame_idx]:.5g}"] + rows2 += [f"{keys.angle_series[frame_idx]:.5g}", f"{keys.zoom_series[frame_idx]:.5g}", + f"{keys.transform_center_x_series[frame_idx]:.5g}", f"{keys.transform_center_y_series[frame_idx]:.5g}"] + rows2 += [f"{keys.translation_x_series[frame_idx]:.5g}", f"{keys.translation_y_series[frame_idx]:.5g}"] if anim_args.animation_mode == '3D': diff --git a/scripts/deforum_helpers/hybrid_video.py b/scripts/deforum_helpers/hybrid_video.py index a314c537..948abb09 100644 --- a/scripts/deforum_helpers/hybrid_video.py +++ b/scripts/deforum_helpers/hybrid_video.py @@ -64,6 +64,8 @@ def hybrid_generation(args, anim_args, root): if not anim_args.hybrid_use_init_image: # determine max frames from length of input frames anim_args.max_frames = len(inputfiles) + if anim_args.max_frames < 1: + raise Exception(f"Error: No input frames found in {video_in_frame_path}! Please check your input video path and whether you've opted to extract input frames.") print(f"Using {anim_args.max_frames} input frames from {video_in_frame_path}...") # use first frame as init diff --git a/scripts/deforum_helpers/render.py b/scripts/deforum_helpers/render.py index ceaf58db..409cc5be 100644 --- a/scripts/deforum_helpers/render.py +++ b/scripts/deforum_helpers/render.py @@ -12,7 +12,7 @@ from .generate import generate, isJson from .noise import add_noise from .animation import anim_frame_warp from .animation_key_frames import DeformAnimKeys, LooperAnimKeys -from .video_audio_utilities import get_frame_name, get_next_frame +from .video_audio_utilities import get_frame_name, get_next_frame, render_preview from .depth import DepthModel from .colors import maintain_colors from .parseq_adapter import ParseqAnimKeys @@ -607,8 +607,12 @@ def render_animation(args, anim_args, video_args, parseq_args, loop_args, contro args.seed = next_seed(args, root) + render_preview(args, anim_args, video_args, root, frame_idx) + + if predict_depths and not keep_in_vram: depth_model.delete_model() # handles adabins too if load_raft: raft_model.delete_model() + diff --git a/scripts/deforum_helpers/render_modes.py b/scripts/deforum_helpers/render_modes.py index e2957c71..a2e40cb5 100644 --- a/scripts/deforum_helpers/render_modes.py +++ b/scripts/deforum_helpers/render_modes.py @@ -6,7 +6,7 @@ import numexpr from modules.shared import opts, state from .render import render_animation from .seed import next_seed -from .video_audio_utilities import vid2frames +from .video_audio_utilities import vid2frames, render_preview from .prompt import interpolate_prompts from .generate import generate from .animation_key_frames import DeformAnimKeys @@ -156,3 +156,5 @@ def render_interpolation(args, anim_args, video_args, parseq_args, loop_args, co args.seed = next_seed(args, root) frame_idx += 1 + + render_preview(args, anim_args, video_args, root, frame_idx) diff --git a/scripts/deforum_helpers/run_deforum.py b/scripts/deforum_helpers/run_deforum.py index bc734124..e41d7511 100644 --- a/scripts/deforum_helpers/run_deforum.py +++ b/scripts/deforum_helpers/run_deforum.py @@ -11,7 +11,7 @@ from .save_images import dump_frames_cache, reset_frames_cache from .frame_interpolation import process_video_interpolation from .general_utils import get_deforum_version from .upscaling import make_upscale_v2 -from .video_audio_utilities import ffmpeg_stitch_video, make_gifski_gif, handle_imgs_deletion, get_ffmpeg_params +from .video_audio_utilities import ffmpeg_stitch_video, make_gifski_gif, handle_imgs_deletion, get_ffmpeg_params, get_ffmpeg_paths from pathlib import Path from .settings import save_settings_from_animation_run @@ -103,16 +103,6 @@ def run_deforum(*args): from base64 import b64encode - real_audio_track = None - if video_args.add_soundtrack != 'None': - real_audio_track = anim_args.video_init_path if video_args.add_soundtrack == 'Init Video' else video_args.soundtrack_path - - # Establish path of subtitle file - if shared.opts.data.get("deforum_save_gen_info_as_srt", False) and shared.opts.data.get("deforum_embed_srt", False): - srt_path = os.path.join(args.outdir, f"{root.timestring}.srt") - else: - srt_path = None - # Delete folder with duplicated imgs from OS temp folder shutil.rmtree(root.tmp_deforum_run_duplicated_folder, ignore_errors=True) @@ -124,14 +114,11 @@ def run_deforum(*args): if video_args.skip_video_creation: print("\nSkipping video creation, uncheck 'Skip video creation' in 'Output' tab if you want to get a video too :)") else: - image_path = os.path.join(args.outdir, f"{root.timestring}_%09d.png") - mp4_path = os.path.join(args.outdir, f"{root.timestring}.mp4") - max_video_frames = anim_args.max_frames - # Stitch video using ffmpeg! try: f_location, f_crf, f_preset = get_ffmpeg_params() # get params for ffmpeg exec - ffmpeg_stitch_video(ffmpeg_location=f_location, fps=video_args.fps, outmp4_path=mp4_path, stitch_from_frame=0, stitch_to_frame=max_video_frames, imgs_path=image_path, add_soundtrack=video_args.add_soundtrack, audio_path=real_audio_track, crf=f_crf, preset=f_preset, srt_path=srt_path) + image_path, mp4_path, real_audio_track, srt_path = get_ffmpeg_paths(args.outdir, root.timestring, anim_args, video_args) + ffmpeg_stitch_video(ffmpeg_location=f_location, fps=video_args.fps, outmp4_path=mp4_path, stitch_from_frame=0, stitch_to_frame=anim_args.max_frames, imgs_path=image_path, add_soundtrack=video_args.add_soundtrack, audio_path=real_audio_track, crf=f_crf, preset=f_preset, srt_path=srt_path) mp4 = open(mp4_path, 'rb').read() data_url = f"data:video/mp4;base64, {b64encode(mp4).decode()}" global last_vid_data diff --git a/scripts/deforum_helpers/ui_settings.py b/scripts/deforum_helpers/ui_settings.py index 30dd4a40..f6f79862 100644 --- a/scripts/deforum_helpers/ui_settings.py +++ b/scripts/deforum_helpers/ui_settings.py @@ -16,4 +16,6 @@ def on_ui_settings(): opts.add_option("deforum_debug_mode_enabled", OptionInfo(False, "Enable Dev mode - adds extra reporting in console", gr.Checkbox, {"interactive": True}, section=section)) opts.add_option("deforum_save_gen_info_as_srt", OptionInfo(False, "Save an .srt (subtitles) file with the generation info along with each animation", gr.Checkbox, {"interactive": True}, section=section)) opts.add_option("deforum_embed_srt", OptionInfo(False, "If .srt file is saved, soft-embed the subtitles into the rendered video file", gr.Checkbox, {"interactive": True}, section=section)) - opts.add_option("deforum_save_gen_info_as_srt_params", OptionInfo(['Noise Schedule'], "Choose which animation params are to be saved to the .srt file (Frame # and Seed will always be saved):", ui_components.DropdownMulti, lambda: {"interactive": True, "choices": srt_ui_params}, section=section)) \ No newline at end of file + opts.add_option("deforum_save_gen_info_as_srt_params", OptionInfo(['Noise Schedule'], "Choose which animation params are to be saved to the .srt file (Frame # and Seed will always be saved):", ui_components.DropdownMulti, lambda: {"interactive": True, "choices": srt_ui_params}, section=section)) + opts.add_option("deforum_preview", OptionInfo("Off", "Generate preview video during generation? (Preview does not include frame interpolation or upscaling.)", gr.Dropdown, {"interactive": True, "choices": ['Off', 'On', 'On, concurrent (don\'t pause generation)']}, section=section)) + opts.add_option("deforum_preview_interval_frames", OptionInfo(100, "Generate preview every N frames", gr.Slider, {"interactive": True, "minimum": 10, "maximum": 500}, section=section)) diff --git a/scripts/deforum_helpers/video_audio_utilities.py b/scripts/deforum_helpers/video_audio_utilities.py index fd2b88e8..7d533ed1 100644 --- a/scripts/deforum_helpers/video_audio_utilities.py +++ b/scripts/deforum_helpers/video_audio_utilities.py @@ -13,6 +13,8 @@ from modules.shared import state, opts from .general_utils import checksum, clean_gradio_path_strings from basicsr.utils.download_util import load_file_from_url from .rich import console +import shutil +from threading import Thread def convert_image(input_path, output_path): # Read the input image @@ -36,6 +38,20 @@ def get_ffmpeg_params(): # get ffmpeg params from webui's settings -> deforum ta return [f_location, f_crf, f_preset] +def get_ffmpeg_paths(outdir, timestring, anim_args, video_args, output_suffix=''): + image_path = os.path.join(outdir, f"{timestring}_%09d.png") + mp4_path = os.path.join(outdir, f"{timestring}{output_suffix}.mp4") + + real_audio_track = None + if video_args.add_soundtrack != 'None': + real_audio_track = anim_args.video_init_path if video_args.add_soundtrack == 'Init Video' else video_args.soundtrack_path + + srt_path = None + if opts.data.get("deforum_save_gen_info_as_srt", False) and opts.data.get("deforum_embed_srt", False): + srt_path = os.path.join(outdir, f"{timestring}.srt") + + return [image_path, mp4_path, real_audio_track, srt_path] + # e.g gets 'x2' returns just 2 as int def extract_number(string): return int(string[1:]) if len(string) > 1 and string[1:].isdigit() else -1 @@ -399,3 +415,21 @@ def count_matching_frames(from_folder, img_batch_id): def get_matching_frame(f, img_batch_id=None): return ('png' in f or 'jpg' in f) and '-' not in f and '_depth_' not in f and ((img_batch_id is not None and f.startswith(img_batch_id) or img_batch_id is None)) + +def render_preview(args, anim_args, video_args, root, frame_idx): + if ("on" not in opts.data.get("deforum_preview", 0).lower()) or (frame_idx % opts.data.get("deforum_preview_interval_frames", 0) != 0): + return + + f_location, f_crf, f_preset = get_ffmpeg_params() # get params for ffmpeg exec + image_path, mp4_temp_path, real_audio_track, srt_path = get_ffmpeg_paths(args.outdir, root.timestring, anim_args, video_args, "_preview__rendering__") + mp4_preview_path = mp4_temp_path.replace("_preview__rendering__", "_preview") + print(f"--> Rendering preview video up to frame {frame_idx} to {mp4_preview_path}...") + + def task(): + ffmpeg_stitch_video(ffmpeg_location=f_location, fps=video_args.fps, outmp4_path=mp4_temp_path, stitch_from_frame=0, stitch_to_frame=frame_idx, imgs_path=image_path, add_soundtrack=video_args.add_soundtrack, audio_path=real_audio_track, crf=f_crf, preset=f_preset, srt_path=srt_path) + shutil.move(mp4_temp_path, mp4_preview_path) # rename so that _preview file is always in a watchable state + + if "concurrent" in opts.data.get("deforum_preview", 0).lower(): + Thread(target=task).start() + else: + task() From d2705e2edf6a73c81d426eb631e378ee7483f701 Mon Sep 17 00:00:00 2001 From: rewbs Date: Tue, 27 Jun 2023 14:19:27 +1000 Subject: [PATCH 2/8] Fixed missed variable inlining in upscale invocation --- scripts/deforum_helpers/run_deforum.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/deforum_helpers/run_deforum.py b/scripts/deforum_helpers/run_deforum.py index e41d7511..91b0a3a9 100644 --- a/scripts/deforum_helpers/run_deforum.py +++ b/scripts/deforum_helpers/run_deforum.py @@ -136,7 +136,7 @@ def run_deforum(*args): # Upscale video once generation is done: if video_args.r_upscale_video and not video_args.skip_video_creation and not video_args.store_frames_in_ram: # out mp4 path is defined in make_upscale func - make_upscale_v2(upscale_factor = video_args.r_upscale_factor, upscale_model = video_args.r_upscale_model, keep_imgs = video_args.r_upscale_keep_imgs, imgs_raw_path = args.outdir, imgs_batch_id = root.timestring, fps = video_args.fps, deforum_models_path = root.models_path, current_user_os = root.current_user_os, ffmpeg_location=f_location, stitch_from_frame=0, stitch_to_frame=max_video_frames, ffmpeg_crf=f_crf, ffmpeg_preset=f_preset, add_soundtrack = video_args.add_soundtrack ,audio_path=real_audio_track, srt_path=srt_path) + make_upscale_v2(upscale_factor = video_args.r_upscale_factor, upscale_model = video_args.r_upscale_model, keep_imgs = video_args.r_upscale_keep_imgs, imgs_raw_path = args.outdir, imgs_batch_id = root.timestring, fps = video_args.fps, deforum_models_path = root.models_path, current_user_os = root.current_user_os, ffmpeg_location=f_location, stitch_from_frame=0, stitch_to_frame=anim_args.max_frames, ffmpeg_crf=f_crf, ffmpeg_preset=f_preset, add_soundtrack = video_args.add_soundtrack ,audio_path=real_audio_track, srt_path=srt_path) # FRAME INTERPOLATION TIME if need_to_frame_interpolate: From 39cf10db6884f9249d51b185c8ab2ff39c5cf152 Mon Sep 17 00:00:00 2001 From: rewbs Date: Thu, 29 Jun 2023 06:27:33 +1000 Subject: [PATCH 3/8] Video preview improvements: don't render video on last frames (final video is being rendered anyway); in concurrent mode, detect case where previous preview is still rendering and skip the current render. --- .../deforum_helpers/video_audio_utilities.py | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/scripts/deforum_helpers/video_audio_utilities.py b/scripts/deforum_helpers/video_audio_utilities.py index 7d533ed1..51e297fa 100644 --- a/scripts/deforum_helpers/video_audio_utilities.py +++ b/scripts/deforum_helpers/video_audio_utilities.py @@ -10,7 +10,7 @@ import glob import concurrent.futures from pkg_resources import resource_filename from modules.shared import state, opts -from .general_utils import checksum, clean_gradio_path_strings +from .general_utils import checksum, clean_gradio_path_strings, debug_print from basicsr.utils.download_util import load_file_from_url from .rich import console import shutil @@ -417,17 +417,29 @@ def get_matching_frame(f, img_batch_id=None): return ('png' in f or 'jpg' in f) and '-' not in f and '_depth_' not in f and ((img_batch_id is not None and f.startswith(img_batch_id) or img_batch_id is None)) def render_preview(args, anim_args, video_args, root, frame_idx): - if ("on" not in opts.data.get("deforum_preview", 0).lower()) or (frame_idx % opts.data.get("deforum_preview_interval_frames", 0) != 0): + is_preview_on = "on" in opts.data.get("deforum_preview", 0).lower() + preview_interval_frames = opts.data.get("deforum_preview_interval_frames", 50) + is_preview_frame = (frame_idx % preview_interval_frames) == 0 + is_close_to_end = frame_idx >= (anim_args.max_frames-1) + + debug_print(f"render preview video: frame_idx={frame_idx} preview_interval_frames={preview_interval_frames} anim_args.max_frames={anim_args.max_frames} is_preview_on={is_preview_on} is_preview_frame={is_preview_frame} is_close_to_end={is_close_to_end} ") + + if not is_preview_on or not is_preview_frame or is_close_to_end: + debug_print(f"No preview video on frame {frame_idx}.") return f_location, f_crf, f_preset = get_ffmpeg_params() # get params for ffmpeg exec image_path, mp4_temp_path, real_audio_track, srt_path = get_ffmpeg_paths(args.outdir, root.timestring, anim_args, video_args, "_preview__rendering__") mp4_preview_path = mp4_temp_path.replace("_preview__rendering__", "_preview") - print(f"--> Rendering preview video up to frame {frame_idx} to {mp4_preview_path}...") - def task(): - ffmpeg_stitch_video(ffmpeg_location=f_location, fps=video_args.fps, outmp4_path=mp4_temp_path, stitch_from_frame=0, stitch_to_frame=frame_idx, imgs_path=image_path, add_soundtrack=video_args.add_soundtrack, audio_path=real_audio_track, crf=f_crf, preset=f_preset, srt_path=srt_path) - shutil.move(mp4_temp_path, mp4_preview_path) # rename so that _preview file is always in a watchable state + if os.path.exists(mp4_temp_path): + print(f"--! Skipping preview video on frame {frame_idx} (previous preview still rendering to {mp4_temp_path}...") + else: + print(f"--> Rendering preview video up to frame {frame_idx} to {mp4_preview_path}...") + try: + ffmpeg_stitch_video(ffmpeg_location=f_location, fps=video_args.fps, outmp4_path=mp4_temp_path, stitch_from_frame=0, stitch_to_frame=frame_idx, imgs_path=image_path, add_soundtrack=video_args.add_soundtrack, audio_path=real_audio_track, crf=f_crf, preset=f_preset, srt_path=srt_path) + finally: + shutil.move(mp4_temp_path, mp4_preview_path) if "concurrent" in opts.data.get("deforum_preview", 0).lower(): Thread(target=task).start() From 71917cab9ef94e7728191950b0572a83b14f24cf Mon Sep 17 00:00:00 2001 From: rewbs Date: Wed, 5 Jul 2023 10:59:57 +1000 Subject: [PATCH 4/8] Add support for Parseq to control guided images schedules. --- scripts/deforum_helpers/deforum_tqdm.py | 2 +- scripts/deforum_helpers/parseq_adapter.py | 26 +++++- .../deforum_helpers/parseq_adapter_test.py | 79 ++++++++++++++++--- scripts/deforum_helpers/render.py | 4 +- scripts/deforum_helpers/render_modes.py | 2 +- 5 files changed, 93 insertions(+), 20 deletions(-) diff --git a/scripts/deforum_helpers/deforum_tqdm.py b/scripts/deforum_helpers/deforum_tqdm.py index d29208ea..cf537e0e 100644 --- a/scripts/deforum_helpers/deforum_tqdm.py +++ b/scripts/deforum_helpers/deforum_tqdm.py @@ -16,7 +16,7 @@ class DeforumTQDM: from .parseq_adapter import ParseqAdapter deforum_total = 0 # FIXME: get only amount of steps - parseq_adapter = ParseqAdapter(self._parseq_args, self._anim_args, self._video_args, None, mute=True) + parseq_adapter = ParseqAdapter(self._parseq_args, self._anim_args, self._video_args, None, None, mute=True) keys = DeformAnimKeys(self._anim_args) if not parseq_adapter.use_parseq else parseq_adapter.anim_keys start_frame = 0 diff --git a/scripts/deforum_helpers/parseq_adapter.py b/scripts/deforum_helpers/parseq_adapter.py index 42dfffe9..46c7dcb9 100644 --- a/scripts/deforum_helpers/parseq_adapter.py +++ b/scripts/deforum_helpers/parseq_adapter.py @@ -6,13 +6,13 @@ from operator import itemgetter import numpy as np import pandas as pd import requests -from .animation_key_frames import DeformAnimKeys, ControlNetKeys +from .animation_key_frames import DeformAnimKeys, ControlNetKeys, LooperAnimKeys from .rich import console logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO) class ParseqAdapter(): - def __init__(self, parseq_args, anim_args, video_args, controlnet_args, mute=False): + def __init__(self, parseq_args, anim_args, video_args, controlnet_args, loop_args, mute=False): # Basic data extraction self.use_parseq = parseq_args.parseq_manifest and parseq_args.parseq_manifest.strip() @@ -26,6 +26,8 @@ class ParseqAdapter(): # Wrap the original schedules with Parseq decorators, so that Parseq values will override the original values IFF appropriate. self.anim_keys = ParseqAnimKeysDecorator(self, DeformAnimKeys(anim_args)) self.cn_keys = ParseqControlNetKeysDecorator(self, ControlNetKeys(anim_args, controlnet_args)) if controlnet_args else None + # -1 because seed seems to be unused in LooperAnimKeys + self.looper_keys = ParseqLooperKeysDecorator(self, LooperAnimKeys(loop_args, anim_args, -1)) if loop_args else None # Validation if (self.use_parseq): @@ -78,9 +80,11 @@ class ParseqAdapter(): table.add_column("Parseq", style="cyan") table.add_column("Deforum", style="green") - table.add_row("Anim Fields", '\n'.join(self.anim_keys.managed_fields()), '\n'.join(self.anim_keys.unmanaged_fields())) + table.add_row("Animation", '\n'.join(self.anim_keys.managed_fields()), '\n'.join(self.anim_keys.unmanaged_fields())) if self.cn_keys: - table.add_row("Cn Fields", '\n'.join(self.cn_keys.managed_fields()), '\n'.join(self.cn_keys.unmanaged_fields())) + table.add_row("ControlNet", '\n'.join(self.cn_keys.managed_fields()), '\n'.join(self.cn_keys.unmanaged_fields())) + if self.looper_keys: + table.add_row("Guided Images", '\n'.join(self.looper_keys.managed_fields()), '\n'.join(self.looper_keys.unmanaged_fields())) table.add_row("Prompts", "✅" if self.manages_prompts() else "❌", "✅" if not self.manages_prompts() else "❌") table.add_row("Frames", str(len(self.rendered_frames)), str(self.required_frames) + (" ⚠️" if str(self.required_frames) != str(len(self.rendered_frames))+"" else "")) table.add_row("FPS", str(self.config_output_fps), str(self.required_fps) + (" ⚠️" if str(self.required_fps) != str(self.config_output_fps) else "")) @@ -239,5 +243,19 @@ class ParseqAnimKeysDecorator(ParseqAbstractDecorator): self.prompts = super().parseq_to_series('deforum_prompt') # formatted as "{positive} --neg {negative}" +class ParseqLooperKeysDecorator(ParseqAbstractDecorator): + def __init__(self, adapter: ParseqAdapter, looper_keys): + super().__init__(adapter, looper_keys) + # The Deforum UI offers an "Image strength schedule" in the Guided Images section, + # which simply overrides the strength schedule if guided images is enabled. + # In Parseq, we just re-use the same strength schedule. + self.image_strength_schedule_series = super().parseq_to_series('strength') + + # We explicitly state the mapping for all other guided images fields so we can strip the prefix + # that we use in Parseq. + self.blendFactorMax_series = super().parseq_to_series('guided_blendFactorMax') + self.blendFactorSlope_series = super().parseq_to_series('guided_blendFactorSlope') + self.tweening_frames_schedule_series = super().parseq_to_series('guided_tweening_frames') + self.color_correction_factor_series = super().parseq_to_series('guidedcolor_correction_factor') diff --git a/scripts/deforum_helpers/parseq_adapter_test.py b/scripts/deforum_helpers/parseq_adapter_test.py index 1b06873a..b0f6aace 100644 --- a/scripts/deforum_helpers/parseq_adapter_test.py +++ b/scripts/deforum_helpers/parseq_adapter_test.py @@ -12,19 +12,20 @@ from types import SimpleNamespace DEFAULT_ARGS = SimpleNamespace(anim_args = SimpleNamespace(max_frames=2), video_args = SimpleNamespace(fps=30), args = SimpleNamespace(seed=-1), - loop_args = SimpleNamespace(), - controlnet_args = SimpleNamespace()) + controlnet_args = SimpleNamespace(), + loop_args = SimpleNamespace()) def buildParseqAdapter(parseq_use_deltas, parseq_manifest, setup_args=DEFAULT_ARGS): return ParseqAdapter(SimpleNamespace(parseq_use_deltas=parseq_use_deltas, parseq_manifest=parseq_manifest), - setup_args.anim_args, setup_args.video_args, setup_args.controlnet_args) + setup_args.anim_args, setup_args.video_args, setup_args.controlnet_args, setup_args.loop_args) class TestParseqAnimKeys(unittest.TestCase): @patch('deforum_helpers.parseq_adapter.DeformAnimKeys') @patch('deforum_helpers.parseq_adapter.ControlNetKeys') - def test_withprompt(self, mock_deformanimkeys, mock_controlnetkeys): + @patch('deforum_helpers.parseq_adapter.LooperAnimKeys') + def test_withprompt(self, mock_deformanimkeys, mock_controlnetkeys, mock_looperanimkeys): parseq_adapter = buildParseqAdapter(parseq_use_deltas=True, parseq_manifest=""" { "options": { @@ -47,7 +48,8 @@ class TestParseqAnimKeys(unittest.TestCase): @patch('deforum_helpers.parseq_adapter.DeformAnimKeys') @patch('deforum_helpers.parseq_adapter.ControlNetKeys') - def test_withoutprompt(self, mock_deformanimkeys, mock_controlnetkeys): + @patch('deforum_helpers.parseq_adapter.LooperAnimKeys') + def test_withoutprompt(self, mock_deformanimkeys, mock_controlnetkeys, mock_looperanimkeys): parseq_adapter = buildParseqAdapter(parseq_use_deltas=True, parseq_manifest=""" { "options": { @@ -67,7 +69,8 @@ class TestParseqAnimKeys(unittest.TestCase): @patch('deforum_helpers.parseq_adapter.DeformAnimKeys') @patch('deforum_helpers.parseq_adapter.ControlNetKeys') - def test_withseed(self, mock_deformanimkeys, mock_controlnetkeys): + @patch('deforum_helpers.parseq_adapter.LooperAnimKeys') + def test_withseed(self, mock_deformanimkeys, mock_controlnetkeys, mock_looperanimkeys): parseq_adapter = buildParseqAdapter(parseq_use_deltas=True, parseq_manifest=""" { "options": { @@ -90,7 +93,8 @@ class TestParseqAnimKeys(unittest.TestCase): @patch('deforum_helpers.parseq_adapter.DeformAnimKeys') @patch('deforum_helpers.parseq_adapter.ControlNetKeys') - def test_withoutseed(self, mock_deformanimkeys, mock_controlnetkeys): + @patch('deforum_helpers.parseq_adapter.LooperAnimKeys') + def test_withoutseed(self, mock_deformanimkeys, mock_controlnetkeys, mock_looperanimkeys): parseq_adapter = buildParseqAdapter(parseq_use_deltas=True, parseq_manifest=""" { "options": { @@ -111,7 +115,8 @@ class TestParseqAnimKeys(unittest.TestCase): @patch('deforum_helpers.parseq_adapter.DeformAnimKeys') @patch('deforum_helpers.parseq_adapter.ControlNetKeys') - def test_usedelta(self, mock_deformanimkeys, mock_controlnetkeys): + @patch('deforum_helpers.parseq_adapter.LooperAnimKeys') + def test_usedelta(self, mock_deformanimkeys, mock_controlnetkeys, mock_looperanimkeys): parseq_adapter = buildParseqAdapter(parseq_use_deltas=True, parseq_manifest=""" { "options": { @@ -135,7 +140,8 @@ class TestParseqAnimKeys(unittest.TestCase): @patch('deforum_helpers.parseq_adapter.DeformAnimKeys') @patch('deforum_helpers.parseq_adapter.ControlNetKeys') - def test_usenondelta(self, mock_deformanimkeys, mock_controlnetkeys): + @patch('deforum_helpers.parseq_adapter.LooperAnimKeys') + def test_usenondelta(self, mock_deformanimkeys, mock_controlnetkeys, mock_looperanimkeys): parseq_adapter = buildParseqAdapter(parseq_use_deltas=False, parseq_manifest=""" { "options": { @@ -159,7 +165,8 @@ class TestParseqAnimKeys(unittest.TestCase): @patch('deforum_helpers.parseq_adapter.DeformAnimKeys') @patch('deforum_helpers.parseq_adapter.ControlNetKeys') - def test_fallbackonundefined(self, mock_deformanimkeys, mock_controlnetkeys): + @patch('deforum_helpers.parseq_adapter.LooperAnimKeys') + def test_fallbackonundefined(self, mock_deformanimkeys, mock_controlnetkeys, mock_looperanimkeys): parseq_adapter = buildParseqAdapter(parseq_use_deltas=False, parseq_manifest=""" { "options": { @@ -181,7 +188,8 @@ class TestParseqAnimKeys(unittest.TestCase): @patch('deforum_helpers.parseq_adapter.DeformAnimKeys') @patch('deforum_helpers.parseq_adapter.ControlNetKeys') - def test_cn(self, mock_deformanimkeys, mock_controlnetkeys): + @patch('deforum_helpers.parseq_adapter.LooperAnimKeys') + def test_cn(self, mock_deformanimkeys, mock_controlnetkeys, mock_looperanimkeys): parseq_adapter = buildParseqAdapter(parseq_use_deltas=False, parseq_manifest=""" { "options": { @@ -203,7 +211,8 @@ class TestParseqAnimKeys(unittest.TestCase): @patch('deforum_helpers.parseq_adapter.DeformAnimKeys') @patch('deforum_helpers.parseq_adapter.ControlNetKeys') - def test_cn_fallback(self, mock_deformanimkeys, mock_controlnetkeys): + @patch('deforum_helpers.parseq_adapter.LooperAnimKeys') + def test_cn_fallback(self, mock_deformanimkeys, mock_controlnetkeys, mock_looperanimkeys): parseq_adapter = buildParseqAdapter(parseq_use_deltas=False, parseq_manifest=""" { "options": { @@ -223,5 +232,51 @@ class TestParseqAnimKeys(unittest.TestCase): #There must be a better way to inject an expected value via patch and check for that... self.assertRegex(str(parseq_adapter.cn_keys.cn_1_weight_schedule_series[0]), r'MagicMock') + @patch('deforum_helpers.parseq_adapter.DeformAnimKeys') + @patch('deforum_helpers.parseq_adapter.LooperAnimKeys') + @patch('deforum_helpers.parseq_adapter.ControlNetKeys') + def test_looper(self, mock_deformanimkeys, mock_looperanimkeys, mock_controlnetkeys): + parseq_adapter = buildParseqAdapter(parseq_use_deltas=False, parseq_manifest=""" + { + "options": { + "output_fps": 30 + }, + "rendered_frames": [ + { + "frame": 0, + "guided_blendFactorMax": 0.4 + }, + { + "frame": 1, + "guided_blendFactorMax": 0.4 + } + ] + } + """) + self.assertEqual(parseq_adapter.looper_keys.blendFactorMax_series[0], 0.4) + + @patch('deforum_helpers.parseq_adapter.DeformAnimKeys') + @patch('deforum_helpers.parseq_adapter.LooperAnimKeys') + @patch('deforum_helpers.parseq_adapter.ControlNetKeys') + def test_looper_fallback(self, mock_deformanimkeys, mock_looperanimkeys, mock_controlnetkeys): + parseq_adapter = buildParseqAdapter(parseq_use_deltas=False, parseq_manifest=""" + { + "options": { + "output_fps": 30 + }, + "rendered_frames": [ + { + "frame": 0 + }, + { + "frame": 1 + } + ] + } + """) + #TODO - this is a hacky check to make sure we're falling back to the mock. + #There must be a better way to inject an expected value via patch and check for that... + self.assertRegex(str(parseq_adapter.looper_keys.blendFactorMax_series[0]), r'MagicMock') + if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/scripts/deforum_helpers/render.py b/scripts/deforum_helpers/render.py index 7f707cc6..4ca7fa57 100644 --- a/scripts/deforum_helpers/render.py +++ b/scripts/deforum_helpers/render.py @@ -63,11 +63,11 @@ def render_animation(args, anim_args, video_args, parseq_args, loop_args, contro unpack_controlnet_vids(args, anim_args, controlnet_args) # initialise Parseq adapter - parseq_adapter = ParseqAdapter(parseq_args, anim_args, video_args, controlnet_args) + parseq_adapter = ParseqAdapter(parseq_args, anim_args, video_args, controlnet_args, loop_args) # expand key frame strings to values keys = DeformAnimKeys(anim_args, args.seed) if not parseq_adapter.use_parseq else parseq_adapter.anim_keys - loopSchedulesAndData = LooperAnimKeys(loop_args, anim_args, args.seed) + loopSchedulesAndData = LooperAnimKeys(loop_args, anim_args, args.seed) if not parseq_adapter.use_parseq else parseq_adapter.looper_keys # create output folder for the batch os.makedirs(args.outdir, exist_ok=True) diff --git a/scripts/deforum_helpers/render_modes.py b/scripts/deforum_helpers/render_modes.py index 1fc0a57f..fd4e0a16 100644 --- a/scripts/deforum_helpers/render_modes.py +++ b/scripts/deforum_helpers/render_modes.py @@ -80,7 +80,7 @@ def get_parsed_value(value, frame_idx, max_f): def render_interpolation(args, anim_args, video_args, parseq_args, loop_args, controlnet_args, root): # use parseq if manifest is provided - parseq_adapter = ParseqAdapter(parseq_args, anim_args, video_args, controlnet_args) + parseq_adapter = ParseqAdapter(parseq_args, anim_args, video_args, controlnet_args, loop_args) # expand key frame strings to values keys = DeformAnimKeys(anim_args) if not parseq_adapter.use_parseq else parseq_adapter.anim_keys From c3a3b13afc1e32868a274fc873cc4028d4bb26ec Mon Sep 17 00:00:00 2001 From: rewbs Date: Wed, 5 Jul 2023 11:32:01 +1000 Subject: [PATCH 5/8] Don't pollute list of managed/unmanaged fields with internal attributes. --- scripts/deforum_helpers/parseq_adapter.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/deforum_helpers/parseq_adapter.py b/scripts/deforum_helpers/parseq_adapter.py index 46c7dcb9..bea83b6c 100644 --- a/scripts/deforum_helpers/parseq_adapter.py +++ b/scripts/deforum_helpers/parseq_adapter.py @@ -11,6 +11,8 @@ from .rich import console logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO) +IGNORED_FIELDS = ['fi', 'use_looper', 'imagesToKeyframe', 'schedules'] + class ParseqAdapter(): def __init__(self, parseq_args, anim_args, video_args, controlnet_args, loop_args, mute=False): @@ -184,12 +186,12 @@ class ParseqAbstractDecorator(): def managed_fields(self): all_parseq_fields = self.all_parseq_fields() - deforum_fields = [self.strip_suffixes(property) for property, _ in vars(self.fallback_keys).items() if property not in ['fi'] and not property.startswith('_')] + deforum_fields = [self.strip_suffixes(property) for property, _ in vars(self.fallback_keys).items() if property not in IGNORED_FIELDS and not property.startswith('_')] return [field for field in deforum_fields if field in all_parseq_fields] def unmanaged_fields(self): all_parseq_fields = self.all_parseq_fields() - deforum_fields = [self.strip_suffixes(property) for property, _ in vars(self.fallback_keys).items() if property not in ['fi'] and not property.startswith('_')] + deforum_fields = [self.strip_suffixes(property) for property, _ in vars(self.fallback_keys).items() if property not in IGNORED_FIELDS and not property.startswith('_')] return [field for field in deforum_fields if field not in all_parseq_fields] From 0fafce9c57762dc14e6a3e11012678e1ccd0981b Mon Sep 17 00:00:00 2001 From: Robin Fernandes Date: Fri, 21 Jul 2023 23:48:00 +1000 Subject: [PATCH 6/8] Don't skip creating preview video if the preview frame is a cadence frame (create it on the next generation frame) --- scripts/deforum_helpers/render.py | 4 ++-- scripts/deforum_helpers/render_modes.py | 5 ++++- scripts/deforum_helpers/video_audio_utilities.py | 8 +++++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/scripts/deforum_helpers/render.py b/scripts/deforum_helpers/render.py index ce0833b2..7afd0ba4 100644 --- a/scripts/deforum_helpers/render.py +++ b/scripts/deforum_helpers/render.py @@ -192,6 +192,7 @@ def render_animation(args, anim_args, video_args, parseq_args, loop_args, contro # Webui state.job_count = anim_args.max_frames + last_preview_frame = 0 while frame_idx < anim_args.max_frames: # Webui @@ -608,8 +609,7 @@ def render_animation(args, anim_args, video_args, parseq_args, loop_args, contro args.seed = next_seed(args, root) - render_preview(args, anim_args, video_args, root, frame_idx) - + last_preview_frame = render_preview(args, anim_args, video_args, root, frame_idx, last_preview_frame) if predict_depths and not keep_in_vram: depth_model.delete_model() # handles adabins too diff --git a/scripts/deforum_helpers/render_modes.py b/scripts/deforum_helpers/render_modes.py index 7f9c40bf..efd936f9 100644 --- a/scripts/deforum_helpers/render_modes.py +++ b/scripts/deforum_helpers/render_modes.py @@ -102,6 +102,7 @@ def render_interpolation(args, anim_args, video_args, parseq_args, loop_args, co state.job_count = anim_args.max_frames frame_idx = 0 + last_preview_frame = 0 # INTERPOLATION MODE while frame_idx < anim_args.max_frames: # print data to cli @@ -155,6 +156,8 @@ def render_interpolation(args, anim_args, video_args, parseq_args, loop_args, co if args.seed_behavior != 'schedule': args.seed = next_seed(args, root) + last_preview_frame = render_preview(args, anim_args, video_args, root, frame_idx, last_preview_frame) + frame_idx += 1 - render_preview(args, anim_args, video_args, root, frame_idx) + diff --git a/scripts/deforum_helpers/video_audio_utilities.py b/scripts/deforum_helpers/video_audio_utilities.py index 51e297fa..e7a20bc1 100644 --- a/scripts/deforum_helpers/video_audio_utilities.py +++ b/scripts/deforum_helpers/video_audio_utilities.py @@ -416,17 +416,17 @@ def count_matching_frames(from_folder, img_batch_id): def get_matching_frame(f, img_batch_id=None): return ('png' in f or 'jpg' in f) and '-' not in f and '_depth_' not in f and ((img_batch_id is not None and f.startswith(img_batch_id) or img_batch_id is None)) -def render_preview(args, anim_args, video_args, root, frame_idx): +def render_preview(args, anim_args, video_args, root, frame_idx, last_preview_frame): is_preview_on = "on" in opts.data.get("deforum_preview", 0).lower() preview_interval_frames = opts.data.get("deforum_preview_interval_frames", 50) - is_preview_frame = (frame_idx % preview_interval_frames) == 0 + is_preview_frame = (frame_idx % preview_interval_frames) == 0 or (frame_idx - last_preview_frame) >= preview_interval_frames is_close_to_end = frame_idx >= (anim_args.max_frames-1) debug_print(f"render preview video: frame_idx={frame_idx} preview_interval_frames={preview_interval_frames} anim_args.max_frames={anim_args.max_frames} is_preview_on={is_preview_on} is_preview_frame={is_preview_frame} is_close_to_end={is_close_to_end} ") if not is_preview_on or not is_preview_frame or is_close_to_end: debug_print(f"No preview video on frame {frame_idx}.") - return + return last_preview_frame f_location, f_crf, f_preset = get_ffmpeg_params() # get params for ffmpeg exec image_path, mp4_temp_path, real_audio_track, srt_path = get_ffmpeg_paths(args.outdir, root.timestring, anim_args, video_args, "_preview__rendering__") @@ -445,3 +445,5 @@ def render_preview(args, anim_args, video_args, root, frame_idx): Thread(target=task).start() else: task() + + return frame_idx From 6682572d960b967cdc2ac495545f9adf4888821f Mon Sep 17 00:00:00 2001 From: rewbs Date: Thu, 27 Jul 2023 23:00:52 +1000 Subject: [PATCH 7/8] Fix case when preview option has never been set. --- scripts/deforum_helpers/args.py | 2 +- scripts/deforum_helpers/video_audio_utilities.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/deforum_helpers/args.py b/scripts/deforum_helpers/args.py index 6a003b11..aed6edf9 100644 --- a/scripts/deforum_helpers/args.py +++ b/scripts/deforum_helpers/args.py @@ -648,7 +648,7 @@ def DeforumAnimArgs(): "label": "Consistency mask blur", "type": "slider", "minimum": 0, - "maximum": 16, + "maximum": 64, "step": 1, "value": 2, "visible": False diff --git a/scripts/deforum_helpers/video_audio_utilities.py b/scripts/deforum_helpers/video_audio_utilities.py index e7a20bc1..1e45cbb0 100644 --- a/scripts/deforum_helpers/video_audio_utilities.py +++ b/scripts/deforum_helpers/video_audio_utilities.py @@ -417,7 +417,7 @@ def get_matching_frame(f, img_batch_id=None): return ('png' in f or 'jpg' in f) and '-' not in f and '_depth_' not in f and ((img_batch_id is not None and f.startswith(img_batch_id) or img_batch_id is None)) def render_preview(args, anim_args, video_args, root, frame_idx, last_preview_frame): - is_preview_on = "on" in opts.data.get("deforum_preview", 0).lower() + is_preview_on = "on" in opts.data.get("deforum_preview", "off").lower() preview_interval_frames = opts.data.get("deforum_preview_interval_frames", 50) is_preview_frame = (frame_idx % preview_interval_frames) == 0 or (frame_idx - last_preview_frame) >= preview_interval_frames is_close_to_end = frame_idx >= (anim_args.max_frames-1) @@ -441,7 +441,7 @@ def render_preview(args, anim_args, video_args, root, frame_idx, last_preview_fr finally: shutil.move(mp4_temp_path, mp4_preview_path) - if "concurrent" in opts.data.get("deforum_preview", 0).lower(): + if "concurrent" in opts.data.get("deforum_preview", "off").lower(): Thread(target=task).start() else: task() From 943a10ad3be7215819d30a7a681ddeec5fbe4393 Mon Sep 17 00:00:00 2001 From: Robin Fernandes Date: Fri, 28 Jul 2023 13:56:39 +1000 Subject: [PATCH 8/8] Update args.py - undo unrelated max value change. --- scripts/deforum_helpers/args.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/deforum_helpers/args.py b/scripts/deforum_helpers/args.py index aed6edf9..6a003b11 100644 --- a/scripts/deforum_helpers/args.py +++ b/scripts/deforum_helpers/args.py @@ -648,7 +648,7 @@ def DeforumAnimArgs(): "label": "Consistency mask blur", "type": "slider", "minimum": 0, - "maximum": 64, + "maximum": 16, "step": 1, "value": 2, "visible": False