From b26628a5be6b47e9f9b24dda5672001e9b03d7ed Mon Sep 17 00:00:00 2001 From: Logan zoellner Date: Wed, 24 May 2023 11:28:25 -0400 Subject: [PATCH] add init_frame to txt2vid --- scripts/base_ui.py | 1 + scripts/core/txt2vid.py | 18 ++++++++++++------ scripts/core/utils.py | 2 +- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/scripts/base_ui.py b/scripts/base_ui.py index 8dd8b91..7ccd392 100644 --- a/scripts/base_ui.py +++ b/scripts/base_ui.py @@ -114,6 +114,7 @@ def inputs_ui(): gr.HTML('Control video (each frame will be used as input image to CN): *NOT REQUIRED') with gr.Row(): t2v_file = gr.File(label="Input video", interactive=True, file_count="single", file_types=["video"], elem_id="tex_to_vid_chosen_file") + t2v_init_image = gr.Image(label="Input image", interactive=True, file_count="single", file_types=["image"], elem_id="tex_to_vid_init_image") t2v_width, t2v_height, t2v_prompt, t2v_n_prompt, t2v_cfg_scale, t2v_seed, t2v_processing_strength, t2v_fix_frame_strength, t2v_sampler_index, t2v_steps = setup_common_values('txt2vid', t2v_args) diff --git a/scripts/core/txt2vid.py b/scripts/core/txt2vid.py index bcc53d8..40c2551 100644 --- a/scripts/core/txt2vid.py +++ b/scripts/core/txt2vid.py @@ -86,12 +86,18 @@ def start_process(*args): curr_video_frame = cv2.resize(curr_video_frame, (args_dict['width'], args_dict['height'])) utils.set_CNs_input_image(args_dict, Image.fromarray(curr_video_frame)) - processed_frames, _, _, _ = utils.txt2img(args_dict) - processed_frame = np.array(processed_frames[0])[...,:3] - #if input_video is not None: - # processed_frame = skimage.exposure.match_histograms(processed_frame, curr_video_frame, channel_axis=-1) - processed_frame = np.clip(processed_frame, 0, 255).astype(np.uint8) - init_frame = processed_frame.copy() + if args_dict['init_image'] is not None: + #resize array to args_dict['width'], args_dict['height'] + image_array=args_dict['init_image']#this is a numpy array + init_frame = np.array(Image.fromarray(image_array).resize((args_dict['width'], args_dict['height'])).convert('RGB')) + processed_frame = init_frame.copy() + else: + processed_frames, _, _, _ = utils.txt2img(args_dict) + processed_frame = np.array(processed_frames[0])[...,:3] + #if input_video is not None: + # processed_frame = skimage.exposure.match_histograms(processed_frame, curr_video_frame, channel_axis=-1) + processed_frame = np.clip(processed_frame, 0, 255).astype(np.uint8) + init_frame = processed_frame.copy() output_video = cv2.VideoWriter(output_video_name, cv2.VideoWriter_fourcc(*'mp4v'), args_dict['fps'], (args_dict['width'], args_dict['height'])) output_video.write(cv2.cvtColor(processed_frame, cv2.COLOR_RGB2BGR)) diff --git a/scripts/core/utils.py b/scripts/core/utils.py index ae7c200..e06c171 100644 --- a/scripts/core/utils.py +++ b/scripts/core/utils.py @@ -10,7 +10,7 @@ def get_component_names(): 'v2v_sampler_index', 'v2v_steps', 'v2v_override_settings', 'v2v_occlusion_mask_blur', 'v2v_occlusion_mask_trailing', 'v2v_occlusion_mask_flow_multiplier', 'v2v_occlusion_mask_difo_multiplier', 'v2v_occlusion_mask_difs_multiplier', 'v2v_step_1_processing_mode', 'v2v_step_1_blend_alpha', 'v2v_step_1_seed', 'v2v_step_2_seed', - 't2v_file', 't2v_width', 't2v_height', 't2v_prompt', 't2v_n_prompt', 't2v_cfg_scale', 't2v_seed', 't2v_processing_strength', 't2v_fix_frame_strength', + 't2v_file','t2v_init_image', 't2v_width', 't2v_height', 't2v_prompt', 't2v_n_prompt', 't2v_cfg_scale', 't2v_seed', 't2v_processing_strength', 't2v_fix_frame_strength', 't2v_sampler_index', 't2v_steps', 't2v_length', 't2v_fps', 'glo_save_frames_check' ]