From a45e6efa67e8a53d6103495ad8fe804309d8771b Mon Sep 17 00:00:00 2001 From: Charles Fettinger Date: Wed, 17 May 2023 16:38:32 -0700 Subject: [PATCH] Add music to video --- iz_helpers/helpers.py | 16 ++++++++++++++++ iz_helpers/run.py | 22 ++++++++++++++-------- iz_helpers/ui.py | 41 ++++++++++++++++++++++++++++------------- iz_helpers/video.py | 8 +++++++- 4 files changed, 65 insertions(+), 22 deletions(-) diff --git a/iz_helpers/helpers.py b/iz_helpers/helpers.py index 28afc51..176da96 100644 --- a/iz_helpers/helpers.py +++ b/iz_helpers/helpers.py @@ -4,6 +4,7 @@ import modules.shared as shared import modules.sd_models import gradio as gr from scripts import postprocessing_upscale +from pkg_resources import resource_filename from .prompt_util import readJsonPrompt, process_keys from .static_variables import jsonprompt_schemafile import asyncio @@ -141,3 +142,18 @@ def value_to_bool(value): if value in (0, 1): return bool(value) return False + +def find_ffmpeg_binary(): + try: + import google.colab + return 'ffmpeg' + except: + pass + for package in ['imageio_ffmpeg', 'imageio-ffmpeg']: + try: + package_path = resource_filename(package, 'binaries') + files = [os.path.join(package_path, f) for f in os.listdir(package_path) if f.startswith("ffmpeg-")] + files.sort(key=lambda x: os.path.getmtime(x), reverse=True) + return files[0] if files else 'ffmpeg' + except: + return 'ffmpeg' \ No newline at end of file diff --git a/iz_helpers/run.py b/iz_helpers/run.py index 28b200d..9246da9 100644 --- a/iz_helpers/run.py +++ b/iz_helpers/run.py @@ -9,11 +9,11 @@ from .helpers import ( fix_env_Path_ffprobe, closest_upper_divisible_by_eight, load_model_from_setting, - do_upscaleImg,value_to_bool + do_upscaleImg,value_to_bool, find_ffmpeg_binary ) from .sd_helpers import renderImg2Img, renderTxt2Img from .image import shrink_and_paste_on_blank, open_image, apply_alpha_mask, draw_gradient_ellipse, resize_and_crop_image, crop_fethear_ellipse, crop_inner_image -from .video import write_video +from .video import write_video, add_audio_to_video def outpaint_steps( width, @@ -41,7 +41,7 @@ def outpaint_steps( mask_height, custom_exit_image, frame_correction=True, # TODO: add frame_Correction in UI - blend_gradient_size = 61, + blend_gradient_size = 61 ): main_frames = [init_img.convert("RGBA")] prev_image = init_img.convert("RGBA") @@ -87,7 +87,7 @@ def outpaint_steps( f"{common_prompt_pre}\n{pr}\n{common_prompt_suf}".strip(), negative_prompt, sampler, - num_inference_steps, + int(num_inference_steps), guidance_scale, seed, width, @@ -209,10 +209,11 @@ def create_zoom( blend_gradient_size, blend_invert_do, blend_color, + audio_filename=None, inpainting_denoising_strength=1, inpainting_full_res=0, inpainting_padding=0, - progress=None, + progress=None, ): for i in range(batchcount): print(f"Batch {i+1}/{batchcount}") @@ -223,7 +224,7 @@ def create_zoom( negative_prompt, num_outpainting_steps, guidance_scale, - num_inference_steps, + int(num_inference_steps), custom_init_image, custom_exit_image, video_frame_rate, @@ -249,6 +250,7 @@ def create_zoom( inpainting_full_res, inpainting_padding, progress, + audio_filename ) return result @@ -325,6 +327,7 @@ def create_zoom_single( inpainting_full_res, inpainting_padding, progress, + audio_filename = None ): # try: # if gr.Progress() is not None: @@ -429,7 +432,7 @@ def create_zoom_single( negative_prompt, current_seed, sampler, - num_inference_steps, + int(num_inference_steps), guidance_scale, inpainting_denoising_strength, inpainting_mask_blur, @@ -443,7 +446,7 @@ def create_zoom_single( mask_height, custom_exit_image, False, - blend_gradient_size, + blend_gradient_size ) #for k in range(len(main_frames)): @@ -551,6 +554,9 @@ def create_zoom_single( blend_gradient_size, ImageColor.getcolor(blend_color, "RGBA"), ) + if audio_filename is not None: + out_config["video_filename"] = add_audio_to_video(out_config["video_filename"], audio_filename, str.replace(out_config["video_filename"], ".mp4", "_audio.mp4"), find_ffmpeg_binary()) + print("Video saved in: " + os.path.join(script_path, out_config["video_filename"])) return ( out_config["video_filename"], diff --git a/iz_helpers/ui.py b/iz_helpers/ui.py index ab25806..f8c9f57 100644 --- a/iz_helpers/ui.py +++ b/iz_helpers/ui.py @@ -1,4 +1,5 @@ import json +from msilib.schema import File import gradio as gr from .run import create_zoom import modules.shared as shared @@ -201,7 +202,7 @@ def on_ui_tabs(): info="Frames to freeze at the start of the video", value=0, minimum=1, - maximum=60, + maximum=120, step=1 ) video_last_frame_dupe_amount = gr.Slider( @@ -209,7 +210,7 @@ def on_ui_tabs(): info="Frames to freeze at the end of the video", value=0, minimum=1, - maximum=60, + maximum=120, step=1 ) video_zoom_speed = gr.Slider( @@ -242,16 +243,27 @@ def on_ui_tabs(): label='Blend Edge Color', default='#ffff00' ) - with gr.Accordion("Blend Info"): - gr.Markdown( - """# Important Blend Info: - Number of Start and Stop Frame Duplication number of frames used for the blend/wipe effect. At 30 Frames per second, 30 frames is 1 second. - Blend Gradient size determines if blends extend to the border of the images. 61 is typical, higher values may result in frames around steps of your video + with gr.Accordion("Blend Info", open=False): + gr.Markdown( + """# Important Blend Info: +Number of Start and Stop Frame Duplication number of frames used for the blend/wipe effect. At 30 Frames per second, 30 frames is 1 second. +Blend Gradient size determines if blends extend to the border of the images. 61 is typical, higher values may result in frames around steps of your video - Free to use grayscale blend images can be found here: https://github.com/Oncorporation/obs-studio/tree/master/plugins/obs-transitions/data/luma_wipes - Ideas for custom blend images: https://www.pexels.com/search/gradient/ - """ - ) +Free to use grayscale blend images can be found here: https://github.com/Oncorporation/obs-studio/tree/master/plugins/obs-transitions/data/luma_wipes +Ideas for custom blend images: https://www.pexels.com/search/gradient/ +""" + ) + + with gr.Tab("Audio"): + with gr.Row(): + audio_filename = gr.Textbox(value=None, label="Audio File Name") + audio_file = gr.File( + value=None, + file_count="single", + file_types=["audio"], + type="file", + label="Audio File") + audio_file.change(get_filename, inputs=[audio_file], outputs=[audio_filename]) with gr.Tab("Outpaint"): inpainting_mask_blur = gr.Slider( @@ -268,7 +280,6 @@ def on_ui_tabs(): type="index", ) - with gr.Tab("Post proccess"): upscale_do = gr.Checkbox(False, label="Enable Upscale") upscaler_name = gr.Dropdown( @@ -335,6 +346,7 @@ Our best experience and trade-off is the R-ERSGAn4x upscaler. blend_gradient_size, blend_invert_do, blend_color, + audio_filename, ], outputs=[output_video, out_image, generation_info, html_info, html_log], ) @@ -352,4 +364,7 @@ def checkPrompts(p): return gr.Button.update( interactive=any(0 in sublist for sublist in p) or any("0" in sublist for sublist in p) - ) \ No newline at end of file + ) + +def get_filename(file): + return file.name \ No newline at end of file diff --git a/iz_helpers/video.py b/iz_helpers/video.py index 45ca9e0..9f06ca9 100644 --- a/iz_helpers/video.py +++ b/iz_helpers/video.py @@ -1,5 +1,6 @@ import numpy as np import imageio +import subprocess from .image import draw_gradient_ellipse, alpha_composite_images, blend_images, PSLumaWipe_images2 import math @@ -103,4 +104,9 @@ def write_video(file_path, frames, fps, reversed=True, start_frame_dupe_amount=1 for i in range(last_frame_dupe_amount): self._writer.append_data(np.array(frame)) - self._writer.close() \ No newline at end of file + self._writer.close() + +def add_audio_to_video(video_path, audio_path, output_path, ffmpeg_location = 'ffmpeg'): + command = [ffmpeg_location, '-i', video_path, '-i', audio_path, '-c:v', 'copy', '-c:a', 'aac', '-map', '0:v:0', '-map', '1:a:0', '-shortest', output_path] + subprocess.run(command) + return output_path \ No newline at end of file