From 19686f17317907919d998f5a073f77c5bff8854a Mon Sep 17 00:00:00 2001 From: GeorgLegato Date: Tue, 18 Apr 2023 03:39:27 +0200 Subject: [PATCH 01/23] added model for txt2img and inpaint setting. clear table added --- scripts/inifnite-zoom.py | 51 +++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/scripts/inifnite-zoom.py b/scripts/inifnite-zoom.py index a102d72..cf4e306 100644 --- a/scripts/inifnite-zoom.py +++ b/scripts/inifnite-zoom.py @@ -21,6 +21,7 @@ from modules.processing import ( ) from modules.ui import create_output_panel, plaintext_to_html +import modules.sd_models default_prompt = "A psychedelic jungle with trees that have glowing, fractal-like patterns, Simon stalenhag poster 1920s style, street level view, hyper futuristic, 8k resolution, hyper realistic" default_negative_prompt = "frames, borderline, text, character, duplicate, error, out of frame, watermark, low quality, ugly, deformed, blur" @@ -28,6 +29,7 @@ default_negative_prompt = "frames, borderline, text, character, duplicate, error def renderTxt2Img(prompt, negative_prompt, sampler, steps, cfg_scale, width, height): processed = None + p = StableDiffusionProcessingTxt2Img( sd_model=shared.sd_model, outpath_samples=shared.opts.outdir_txt2img_samples, @@ -63,6 +65,7 @@ def renderImg2Img( inpainting_padding, ): processed = None + p = StableDiffusionProcessingImg2Img( sd_model=shared.sd_model, outpath_samples=shared.opts.outdir_img2img_samples, @@ -118,6 +121,7 @@ def create_zoom( zoom_speed, outputsizeW, outputsizeH, + custom_exit_image ): fix_env_Path_ffprobe() @@ -145,6 +149,13 @@ def create_zoom( (width, height), resample=Image.LANCZOS ) else: + + # switch to txt2img model + checkinfo = modules.sd_models.checkpoint_alisases[shared.opts.data.get("infzoom_txt2img_model")] + if (not checkinfo): + raise NameError("Checklist not found in registry") + modules.sd_models.load_model(checkinfo) + processed = renderTxt2Img( prompts[min(k for k in prompts.keys() if k >= 0)], negative_prompt, @@ -163,6 +174,13 @@ def create_zoom( all_frames = [] all_frames.append(current_image) + + # switch to inpaint model now + checkinfo = modules.sd_models.checkpoint_alisases[shared.opts.data.get("infzoom_inpainting_model", "sd-v1-5-inpainting.ckpt")] + if (not checkinfo): + raise NameError("Checklist not found in registry") + modules.sd_models.load_model(checkinfo) + for i in range(num_outpainting_steps): print("Outpaint step: " + str(i + 1) + " / " + str(num_outpainting_steps)) @@ -278,18 +296,16 @@ def create_zoom( plaintext_to_html(""), ) - -def exportPrompts(p,np): - print("prompts:" + str(p) +"\n"+str(np)) - def putPrompts(files): - file_paths = [file.name for file in files] with open(files.name, 'r') as f: file_contents = f.read() data = json.loads(file_contents) - print(data) return [gr.DataFrame.update(data["prompts"]), gr.Textbox.update(data["negPrompt"])] +def clearPrompts(): + return [gr.DataFrame.update(value=[[0,"Infinite Zoom. Start over"]]), gr.Textbox.update("")] + + def on_ui_tabs(): with gr.Blocks(analytics_enabled=False) as infinite_zoom_interface: gr.HTML( @@ -313,12 +329,12 @@ def on_ui_tabs(): datatype=["number", "str"], row_count=1, col_count=(2, "fixed"), - value=[[0, default_prompt]], + value=[shared.opts.data.get("infzoom_defPrompt",default_prompt)], wrap=True, ) outpaint_negative_prompt = gr.Textbox( - value=default_negative_prompt, label="Negative Prompt" + value=shared.opts.data.get("infzoom_defNegPrompt",default_negative_prompt), label="Negative Prompt" ) # these button will be moved using JS unde the dataframe view as small ones @@ -327,6 +343,9 @@ def on_ui_tabs(): exportPrompts_button.click(None,_js="exportPrompts",inputs=[outpaint_prompts,outpaint_negative_prompt],outputs=None) importPrompts_button.upload(fn=putPrompts,outputs=[outpaint_prompts,outpaint_negative_prompt], inputs=[importPrompts_button]) + clearPrompts_button= gr.Button(value="Clear prompts",variant="secondary",elem_classes="sm infzoom_tab_butt", elem_id="infzoom_clP_butt") + clearPrompts_button.click(fn=clearPrompts,inputs=[],outputs=[outpaint_prompts,outpaint_negative_prompt]) + outpaint_steps = gr.Slider( minimum=2, maximum=100, @@ -352,6 +371,7 @@ def on_ui_tabs(): label="Sampling Steps for each outpaint", ) init_image = gr.Image(type="pil", label="custom initial image") + exit_image = gr.Image(type="pil", label="custom exit image") with gr.Tab("Video"): video_frame_rate = gr.Slider( label="Frames per second", @@ -436,7 +456,8 @@ def on_ui_tabs(): inpainting_padding, zoom_speed_slider, outsizeW_slider, - outsizeH_slider + outsizeH_slider, + exit_image ], outputs=[output_video, out_image, generation_info, html_info, html_log], ) @@ -465,5 +486,17 @@ def on_ui_settings(): shared.opts.add_option("infzoom_ffprobepath", shared.OptionInfo( "", "Writing videos has dependency to an existing FFPROBE executable on your machine. D/L here (https://github.com/BtbN/FFmpeg-Builds/releases) your OS variant and point to your installation path", gr.Textbox, {"interactive": True}, section=section)) + shared.opts.add_option("infzoom_txt2img_model", shared.OptionInfo( + "", "Name of your desired model to render keyframes (txt2img), if empty current model used", gr.Dropdown, lambda: {"choices": shared.list_checkpoint_tiles()}, section=section)) + + shared.opts.add_option("infzoom_inpainting_model", shared.OptionInfo( + "sd-v1-5-inpainting.ckpt", "Name of your desired inpaint model (img2img-inpaint). Default is vanilla sd-v1-5-inpainting.ckpt ", gr.Dropdown, lambda: {"choices": shared.list_checkpoint_tiles()}, section=section)) + + shared.opts.add_option("infzoom_defPrompt", shared.OptionInfo( + default_prompt, "Default prompt to start with'", gr.TextArea, {"interactive": True}, section=section)) + + shared.opts.add_option("infzoom_defNegPrompt", shared.OptionInfo( + default_negative_prompt, "Default negative prompt to start with'", gr.TextArea, {"interactive": True}, section=section)) + script_callbacks.on_ui_tabs(on_ui_tabs) script_callbacks.on_ui_settings(on_ui_settings) From 90d367b30273a77a1a0c27a2cad36d151a40859d Mon Sep 17 00:00:00 2001 From: GeorgLegato Date: Tue, 18 Apr 2023 04:20:12 +0200 Subject: [PATCH 02/23] adding upscaler --- scripts/inifnite-zoom.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/inifnite-zoom.py b/scripts/inifnite-zoom.py index cf4e306..34958b6 100644 --- a/scripts/inifnite-zoom.py +++ b/scripts/inifnite-zoom.py @@ -20,6 +20,8 @@ from modules.processing import ( StableDiffusionProcessingImg2Img, ) +import scripts.postprocessing_upscale + from modules.ui import create_output_panel, plaintext_to_html import modules.sd_models @@ -45,6 +47,14 @@ def renderTxt2Img(prompt, negative_prompt, sampler, steps, cfg_scale, width, hei height=height, ) processed = process_images(p) + + curImg = processed.images[0] + pp= scripts.postprocessing_upscale.scripts_postprocessing.PostprocessedImage(curImg,{}) + ups = scripts.postprocessing_upscale.ScriptPostprocessingUpscale() + ups.process(pp, upscale_mode=1, upscale_by=2.0, upscale_to_width=None, upscale_to_height=None, upscale_crop=False, upscaler_1_name="4x-UltraSharp", upscaler_2_name=None, upscaler_2_visibility=0.0) + processed.images[0]=pp.image + + return processed From 4ae9c15d207c029a155a632c673a65d30e121b64 Mon Sep 17 00:00:00 2001 From: GeorgLegato Date: Wed, 19 Apr 2023 05:53:54 +0200 Subject: [PATCH 03/23] prepare 2nd image (disabled yet), default settings, samplers, UPSCALE, Clear prompts --- scripts/infonite-zoom.py | 149 +++++++++++++++++++++++++++++---------- 1 file changed, 110 insertions(+), 39 deletions(-) diff --git a/scripts/infonite-zoom.py b/scripts/infonite-zoom.py index f2b137b..1465ba6 100644 --- a/scripts/infonite-zoom.py +++ b/scripts/infonite-zoom.py @@ -20,31 +20,15 @@ from modules.processing import ( StableDiffusionProcessingImg2Img, ) +import scripts.postprocessing_upscale + from modules.ui import create_output_panel, plaintext_to_html +import modules.sd_models +import modules.sd_samplers -available_samplers = [ - "DDIM", - "Euler a", - "Euler", - "LMS", - "Heun", - "DPM2", - "DPM2 a", - "DPM++ 2S a", - "DPM++ 2M", - "DPM++ SDE", - "DPM fast", - "DPM adaptive", - "LMS Karras", - "DPM2 Karras", - "DPM2 a Karras", - "DPM++ 2S a Karras", - "DPM++ 2M Karras", - "DPM++ SDE Karras", -] -default_prompt = "A psychedelic jungle with trees that have glowing, fractal-like patterns, Simon stalenhag poster 1920s style, street level view, hyper futuristic, 8k resolution, hyper realistic" -default_negative_prompt = "frames, borderline, text, character, duplicate, error, out of frame, watermark, low quality, ugly, deformed, blur" +available_samplers = [s.name for s in modules.sd_samplers.samplers] +default_prompt = {"prompts":{"data":[[0,"Cat"],["1","Dog"],["2","Happy Pets"]],"headers":["outpaint steps","prompt"]},"negPrompt":"ugly"} def closest_upper_divisible_by_eight(num): if num % 8 == 0: @@ -52,6 +36,13 @@ def closest_upper_divisible_by_eight(num): else: return math.ceil(num / 8) * 8 +def do_upscaleImg(curImg,upscale_do, upscaler_name,upscale_by): + if (not upscale_do): return curImg + pp= scripts.postprocessing_upscale.scripts_postprocessing.PostprocessedImage(curImg) + ups = scripts.postprocessing_upscale.ScriptPostprocessingUpscale() + ups.process(pp, upscale_mode=2, upscale_by=upscale_by, upscale_to_width=None, upscale_to_height=None, upscale_crop=False, upscaler_1_name=upscaler_name, upscaler_2_name=None, upscaler_2_visibility=0.0) + return pp.image + def renderTxt2Img(prompt, negative_prompt, sampler, steps, cfg_scale, width, height): processed = None @@ -72,7 +63,6 @@ def renderTxt2Img(prompt, negative_prompt, sampler, steps, cfg_scale, width, hei processed = process_images(p) return processed - def renderImg2Img( prompt, negative_prompt, @@ -90,6 +80,7 @@ def renderImg2Img( inpainting_padding, ): processed = None + p = StableDiffusionProcessingImg2Img( sd_model=shared.sd_model, outpath_samples=shared.opts.outdir_img2img_samples, @@ -133,6 +124,7 @@ def create_zoom( guidance_scale, num_inference_steps, custom_init_image, + custom_exit_image, video_frame_rate, video_zoom_mode, video_start_frame_dupe_amount, @@ -147,6 +139,9 @@ def create_zoom( outputsizeH, batchcount, sampler, + upscale_do, + upscaler_name, + upscale_by, progress=gr.Progress(), ): for i in range(batchcount): @@ -158,6 +153,7 @@ def create_zoom( guidance_scale, num_inference_steps, custom_init_image, + custom_exit_image, video_frame_rate, video_zoom_mode, video_start_frame_dupe_amount, @@ -171,7 +167,11 @@ def create_zoom( outputsizeW, outputsizeH, sampler, + upscale_do, + upscaler_name, + upscale_by, progress, + ) return result @@ -183,6 +183,7 @@ def create_zoom_single( guidance_scale, num_inference_steps, custom_init_image, + custom_exit_image, video_frame_rate, video_zoom_mode, video_start_frame_dupe_amount, @@ -196,6 +197,9 @@ def create_zoom_single( outputsizeW, outputsizeH, sampler, + upscale_do, + upscaler_name, + upscale_by, progress=gr.Progress(), ): fix_env_Path_ffprobe() @@ -224,6 +228,13 @@ def create_zoom_single( (width, height), resample=Image.LANCZOS ) else: + # switch to txt2img model + checkinfo = modules.sd_models.checkpoint_alisases[shared.opts.data.get("infzoom_txt2img_model")] + if (not checkinfo): + raise NameError("Checklist not found in registry") + progress(0, desc="Loading Model for txt2img: " + checkinfo.name) + modules.sd_models.load_model(checkinfo) + processed = renderTxt2Img( prompts[min(k for k in prompts.keys() if k >= 0)], negative_prompt, @@ -235,13 +246,27 @@ def create_zoom_single( ) current_image = processed.images[0] + mask_width = math.trunc(width / 4) # was initially 512px => 128px mask_height = math.trunc(height / 4) # was initially 512px => 128px num_interpol_frames = round(video_frame_rate * zoom_speed) all_frames = [] - all_frames.append(current_image) + + + if upscale_do: + progress(0,desc="upscaling inital image") + + all_frames.append(do_upscaleImg(current_image,upscale_do, upscaler_name,upscale_by) if upscale_do else current_image) + + # switch to inpaint model now + checkinfo = modules.sd_models.checkpoint_alisases[shared.opts.data.get("infzoom_inpainting_model", "sd-v1-5-inpainting.ckpt")] + if (not checkinfo): + raise NameError("Checklist not found in registry") + progress(0, desc="Loading Model for inpainting/img2img: " + checkinfo.name) + modules.sd_models.load_model(checkinfo) + for i in range(num_outpainting_steps): print_out = "Outpaint step: " + str(i + 1) + " / " + str(num_outpainting_steps) print(print_out) @@ -261,6 +286,7 @@ def create_zoom_single( # inpainting step current_image = current_image.convert("RGB") + processed = renderImg2Img( prompts[max(k for k in prompts.keys() if k <= i)], negative_prompt, @@ -335,8 +361,21 @@ def create_zoom_single( interpol_image.paste(prev_image_fix_crop, mask=prev_image_fix_crop) - all_frames.append(interpol_image) - all_frames.append(current_image) + if upscale_do: + progress( + ((i + 1) / num_outpainting_steps), + desc="upscaling interpol", + ) + all_frames.append(do_upscaleImg(interpol_image, upscale_do, upscaler_name,upscale_by) if upscale_do else current_image) + + + if (upscale_do): + progress( + ((i + 1) / num_outpainting_steps), + desc="upscaling current", + ) + + all_frames.append(do_upscaleImg(current_image,upscale_do, upscaler_name,upscale_by) if upscale_do else current_image) video_file_name = "infinite_zoom_" + str(int(time.time())) + ".mp4" output_path = shared.opts.data.get( @@ -365,19 +404,15 @@ def create_zoom_single( plaintext_to_html(""), ) - -def exportPrompts(p, np): - print("prompts:" + str(p) + "\n" + str(np)) - - def putPrompts(files): - file_paths = [file.name for file in files] - with open(files.name, "r") as f: + with open(files.name, 'r') as f: file_contents = f.read() data = json.loads(file_contents) - print(data) return [gr.DataFrame.update(data["prompts"]), gr.Textbox.update(data["negPrompt"])] +def clearPrompts(): + return [gr.DataFrame.update(value=[[0,"Infinite Zoom. Start over"]]), gr.Textbox.update("")] + def on_ui_tabs(): with gr.Blocks(analytics_enabled=False) as infinite_zoom_interface: @@ -390,10 +425,12 @@ def on_ui_tabs(): """ ) - generate_btn = gr.Button(value="Generate video", variant="primary") - interrupt = gr.Button(value="Interrupt", elem_id="interrupt_training") + with gr.Row(): + generate_btn = gr.Button(value="Generate video", variant="primary") + interrupt = gr.Button(value="Interrupt", elem_id="interrupt_training") with gr.Row(): with gr.Column(scale=1, variant="panel"): + with gr.Tab("Main"): main_outpaint_steps = gr.Slider( minimum=2, @@ -409,12 +446,12 @@ def on_ui_tabs(): datatype=["number", "str"], row_count=1, col_count=(2, "fixed"), - value=[[0, default_prompt]], + value=json.loads(shared.opts.data.get("infzoom_defPrompt",default_prompt))["prompts"], wrap=True, ) main_negative_prompt = gr.Textbox( - value=default_negative_prompt, label="Negative Prompt" + value=json.loads(shared.opts.data.get("infzoom_defPrompt",default_prompt))["negPrompt"], label="Negative Prompt" ) # these button will be moved using JS unde the dataframe view as small ones @@ -441,6 +478,10 @@ def on_ui_tabs(): outputs=[main_prompts, main_negative_prompt], inputs=[importPrompts_button], ) + + clearPrompts_button= gr.Button(value="Clear prompts",variant="secondary",elem_classes="sm infzoom_tab_butt", elem_id="infzoom_clP_butt") + clearPrompts_button.click(fn=clearPrompts,inputs=[],outputs=[main_prompts,main_negative_prompt]) + main_sampler = gr.Dropdown( label="Sampler", choices=available_samplers, @@ -477,7 +518,10 @@ def on_ui_tabs(): value=50, label="Sampling Steps for each outpaint", ) - init_image = gr.Image(type="pil", label="custom initial image") + with gr.Row(): + init_image = gr.Image(type="pil", label="custom initial image") + exit_image = gr.Image(type="pil", label="custom exit image", visible=False) #TODO: implement exit-image rendering + batchcount_slider = gr.Slider( minimum=1, maximum=25, @@ -539,6 +583,19 @@ def on_ui_tabs(): label="masked padding", minimum=0, maximum=256, value=0 ) + with gr.Tab("Post proccess"): + upscale_do = gr.Checkbox(False, label="Enable Upscale") + upscaler_name = gr.Dropdown(label='Upscaler', elem_id="infZ_upscaler", choices=[x.name for x in shared.sd_upscalers], value=shared.sd_upscalers[0].name) + + upscale_by = gr.Slider( + label="Upscale by factor", minimum=1, maximum=8, value=1 + ) + with gr.Accordion("Help",open=False): + gr.Markdown("""# Performance critical +Depending on amount of frames and which upscaler you choose it might took a long time to render. +Our best experience and trade-off is the R-ERSGAn4x upscaler. +""") + with gr.Column(scale=1, variant="compact"): output_video = gr.Video(label="Output").style(width=512, height=512) ( @@ -558,6 +615,7 @@ def on_ui_tabs(): main_guidance_scale, sampling_step, init_image, + exit_image, video_frame_rate, video_zoom_mode, video_start_frame_dupe_amount, @@ -572,6 +630,10 @@ def on_ui_tabs(): main_height, batchcount_slider, main_sampler, + upscale_do, + upscaler_name, + upscale_by + ], outputs=[output_video, out_image, generation_info, html_info, html_log], ) @@ -638,6 +700,15 @@ def on_ui_settings(): ), ) + shared.opts.add_option("infzoom_txt2img_model", shared.OptionInfo( + "", "Name of your desired model to render keyframes (txt2img), if empty current model used", gr.Dropdown, lambda: {"choices": shared.list_checkpoint_tiles()}, section=section)) + + shared.opts.add_option("infzoom_inpainting_model", shared.OptionInfo( + "sd-v1-5-inpainting.ckpt", "Name of your desired inpaint model (img2img-inpaint). Default is vanilla sd-v1-5-inpainting.ckpt ", gr.Dropdown, lambda: {"choices": shared.list_checkpoint_tiles()}, section=section)) + + shared.opts.add_option("infzoom_defPrompt", shared.OptionInfo( + default_prompt, "Default prompt-setup to start with'", gr.Code, {"interactive": True, "language":"json"}, section=section)) + script_callbacks.on_ui_tabs(on_ui_tabs) script_callbacks.on_ui_settings(on_ui_settings) From 58d01a182a61cec047336782d689901e80180305 Mon Sep 17 00:00:00 2001 From: GeorgLegato Date: Wed, 19 Apr 2023 07:18:51 +0200 Subject: [PATCH 04/23] handlilng default prompts, fix in upscaleing images --- scripts/infonite-zoom.py | 47 ++++++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/scripts/infonite-zoom.py b/scripts/infonite-zoom.py index 1465ba6..3fac238 100644 --- a/scripts/infonite-zoom.py +++ b/scripts/infonite-zoom.py @@ -28,7 +28,7 @@ import modules.sd_samplers available_samplers = [s.name for s in modules.sd_samplers.samplers] -default_prompt = {"prompts":{"data":[[0,"Cat"],["1","Dog"],["2","Happy Pets"]],"headers":["outpaint steps","prompt"]},"negPrompt":"ugly"} +default_prompt = json.loads('{"prompts":{"data":[[0,"Cat"],["1","Dog"],["2","Happy Pets"]],"headers":["outpaint steps","prompt"]},"negPrompt":"ugly"}') def closest_upper_divisible_by_eight(num): if num % 8 == 0: @@ -366,7 +366,7 @@ def create_zoom_single( ((i + 1) / num_outpainting_steps), desc="upscaling interpol", ) - all_frames.append(do_upscaleImg(interpol_image, upscale_do, upscaler_name,upscale_by) if upscale_do else current_image) + all_frames.append(do_upscaleImg(interpol_image, upscale_do, upscaler_name,upscale_by) if upscale_do else interpol_image) if (upscale_do): @@ -604,7 +604,7 @@ Our best experience and trade-off is the R-ERSGAn4x upscaler. html_info, html_log, ) = create_output_panel( - "infinit-zoom", shared.opts.outdir_img2img_samples + "infinite-zoom", shared.opts.outdir_img2img_samples ) generate_btn.click( fn=create_zoom, @@ -696,19 +696,42 @@ def on_ui_settings(): "Writing videos has dependency to an existing FFPROBE executable on your machine. D/L here (https://github.com/BtbN/FFmpeg-Builds/releases) your OS variant and point to your installation path", gr.Textbox, {"interactive": True}, - section=section, - ), + section=section + ) ) - shared.opts.add_option("infzoom_txt2img_model", shared.OptionInfo( - "", "Name of your desired model to render keyframes (txt2img), if empty current model used", gr.Dropdown, lambda: {"choices": shared.list_checkpoint_tiles()}, section=section)) + shared.opts.add_option( + "infzoom_txt2img_model", + shared.OptionInfo( + "", + "Name of your desired model to render keyframes (txt2img), if empty current model used", + gr.Dropdown, + lambda: {"choices": shared.list_checkpoint_tiles()}, + section=section + ) + ) - shared.opts.add_option("infzoom_inpainting_model", shared.OptionInfo( - "sd-v1-5-inpainting.ckpt", "Name of your desired inpaint model (img2img-inpaint). Default is vanilla sd-v1-5-inpainting.ckpt ", gr.Dropdown, lambda: {"choices": shared.list_checkpoint_tiles()}, section=section)) + shared.opts.add_option( + "infzoom_inpainting_model", + shared.OptionInfo( + "sd-v1-5-inpainting.ckpt", + "Name of your desired inpaint model (img2img-inpaint). Default is vanilla sd-v1-5-inpainting.ckpt ", + gr.Dropdown, + lambda: {"choices": shared.list_checkpoint_tiles()}, + section=section + ) + ) - shared.opts.add_option("infzoom_defPrompt", shared.OptionInfo( - default_prompt, "Default prompt-setup to start with'", gr.Code, {"interactive": True, "language":"json"}, section=section)) + shared.opts.add_option( + "infzoom_defPrompt", + shared.OptionInfo( + "", + "Default prompt-setup to start with'", + gr.Code, + {"interactive": True, "language":"json"}, + section=section + ) + ) - script_callbacks.on_ui_tabs(on_ui_tabs) script_callbacks.on_ui_settings(on_ui_settings) From 648121a3a43f7d806278c4b17762e92978adbabb Mon Sep 17 00:00:00 2001 From: GeorgLegato Date: Wed, 19 Apr 2023 07:27:24 +0200 Subject: [PATCH 05/23] fix typo in filename --- scripts/{infonite-zoom.py => infinite-zoom.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{infonite-zoom.py => infinite-zoom.py} (100%) diff --git a/scripts/infonite-zoom.py b/scripts/infinite-zoom.py similarity index 100% rename from scripts/infonite-zoom.py rename to scripts/infinite-zoom.py From b143c18058f29034dcf50dc55d6949c5193ad9a9 Mon Sep 17 00:00:00 2001 From: GeorgLegato Date: Wed, 19 Apr 2023 14:47:13 +0200 Subject: [PATCH 06/23] json-prompt error handling from settings --- scripts/infinite-zoom.py | 49 ++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/scripts/infinite-zoom.py b/scripts/infinite-zoom.py index 5f0cdbc..3c8183d 100644 --- a/scripts/infinite-zoom.py +++ b/scripts/infinite-zoom.py @@ -28,7 +28,11 @@ import modules.sd_samplers available_samplers = [s.name for s in modules.sd_samplers.samplers] -default_prompt = json.loads('{"prompts":{"data":[[0,"Cat"],["1","Dog"],["2","Happy Pets"]],"headers":["outpaint steps","prompt"]},"negPrompt":"ugly"}') +default_prompt = '{"prompts":{"data":[[0,"Cat"],["1","Dog"],["2","Happy Pets"]],"headers":["outpaint steps","prompt"]},"negPrompt":"ugly"}' +empty_prompt = '{"prompts":{"data":[],"headers":["outpaint steps","prompt"]},"negPrompt":""}' + +#must be python dict +invalid_prompt ={"prompts":{"data":[[0,"Your prompt-json is invalid, please check Settings"]],"headers":["outpaint steps","prompt"]},"negPrompt":"Invalid prompt-json"} def closest_upper_divisible_by_eight(num): if num % 8 == 0: @@ -237,7 +241,7 @@ def create_zoom_single( checkinfo = modules.sd_models.checkpoint_alisases[shared.opts.data.get("infzoom_txt2img_model")] if (not checkinfo): raise NameError("Checklist not found in registry") - progress(0, desc="Loading Model for txt2img: " + checkinfo.name) + if progress: progress(0, desc="Loading Model for txt2img: " + checkinfo.name) modules.sd_models.load_model(checkinfo) processed = renderTxt2Img( @@ -260,8 +264,8 @@ def create_zoom_single( all_frames = [] - if upscale_do: - progress(0,desc="upscaling inital image") + if upscale_do and progress: + progress(0, desc="upscaling inital image") all_frames.append(do_upscaleImg(current_image,upscale_do, upscaler_name,upscale_by) if upscale_do else current_image) @@ -269,21 +273,16 @@ def create_zoom_single( checkinfo = modules.sd_models.checkpoint_alisases[shared.opts.data.get("infzoom_inpainting_model", "sd-v1-5-inpainting.ckpt")] if (not checkinfo): raise NameError("Checklist not found in registry") - progress(0, desc="Loading Model for inpainting/img2img: " + checkinfo.name) + if progress: progress(0, desc="Loading Model for inpainting/img2img: " + checkinfo.name) modules.sd_models.load_model(checkinfo) for i in range(num_outpainting_steps): print_out = "Outpaint step: " + str(i + 1) + " / " + str(num_outpainting_steps) print(print_out) - # if progress is not None: - # progress( - # ((i + 1) / num_outpainting_steps), - # desc=print_out, - # ) + if progress: progress( ((i + 1) / num_outpainting_steps), desc=print_out) + prev_image_fix = current_image - prev_image = shrink_and_paste_on_blank(current_image, mask_width, mask_height) - current_image = prev_image # create mask (black image with white mask_width width edges) @@ -367,18 +366,18 @@ def create_zoom_single( interpol_image.paste(prev_image_fix_crop, mask=prev_image_fix_crop) - if upscale_do: + if upscale_do and progress: progress( ((i + 1) / num_outpainting_steps), - desc="upscaling interpol", + desc="upscaling interpol" ) + all_frames.append(do_upscaleImg(interpol_image, upscale_do, upscaler_name,upscale_by) if upscale_do else interpol_image) - - if (upscale_do): + if upscale_do and progress: progress( ((i + 1) / num_outpainting_steps), - desc="upscaling current", + desc="upscaling current" ) all_frames.append(do_upscaleImg(current_image,upscale_do, upscaler_name,upscale_by) if upscale_do else current_image) @@ -446,18 +445,28 @@ def on_ui_tabs(): label="Total Outpaint Steps", info="The more it is, the longer your videos will be", ) + + # safe reading json prompt + pr = shared.opts.data.get("infzoom_defPrompt",default_prompt) + if (not pr): pr = empty_prompt + + try: + jpr = json.loads(pr) + except Exception: + jpr = invalid_prompt + main_prompts = gr.Dataframe( type="array", headers=["outpaint step", "prompt"], datatype=["number", "str"], row_count=1, col_count=(2, "fixed"), - value=json.loads(shared.opts.data.get("infzoom_defPrompt",default_prompt))["prompts"], + value=jpr["prompts"], wrap=True, ) main_negative_prompt = gr.Textbox( - value=json.loads(shared.opts.data.get("infzoom_defPrompt",default_prompt))["negPrompt"], label="Negative Prompt" + value=jpr["negPrompt"], label="Negative Prompt" ) # these button will be moved using JS unde the dataframe view as small ones @@ -731,7 +740,7 @@ def on_ui_settings(): shared.opts.add_option( "infzoom_defPrompt", shared.OptionInfo( - "", + default_prompt, "Default prompt-setup to start with'", gr.Code, {"interactive": True, "language":"json"}, From 9c49de7a4852d97c30a1d7b70b39130c131a85b8 Mon Sep 17 00:00:00 2001 From: GeorgLegato Date: Thu, 20 Apr 2023 02:44:36 +0200 Subject: [PATCH 07/23] added jsonschema for validating. fixed hardcoded basedir path --- scripts/infinite-zoom.py | 31 +++++++++++++++++----- scripts/promptschema.json | 55 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 7 deletions(-) create mode 100644 scripts/promptschema.json diff --git a/scripts/infinite-zoom.py b/scripts/infinite-zoom.py index 3c8183d..bc16156 100644 --- a/scripts/infinite-zoom.py +++ b/scripts/infinite-zoom.py @@ -1,9 +1,9 @@ import sys import os import time +import json +from jsonschema import validate -basedir = os.getcwd() -sys.path.extend(basedir + "/extensions/infinite-zoom-automatic1111-webui/") import numpy as np import gradio as gr from PIL import Image @@ -12,7 +12,7 @@ import json from iz_helpers import shrink_and_paste_on_blank, write_video from webui import wrap_gradio_gpu_call -from modules import script_callbacks +from modules import script_callbacks, scripts import modules.shared as shared from modules.processing import ( process_images, @@ -26,6 +26,10 @@ from modules.ui import create_output_panel, plaintext_to_html import modules.sd_models import modules.sd_samplers +from modules import scripts +usefulDirs = scripts.basedir().split(os.sep)[-2:] # contains install and our extension foldername +jsonprompt_schemafile = usefulDirs[0]+"/"+usefulDirs[1]+"/scripts/promptschema.json" + available_samplers = [s.name for s in modules.sd_samplers.samplers] default_prompt = '{"prompts":{"data":[[0,"Cat"],["1","Dog"],["2","Happy Pets"]],"headers":["outpaint steps","prompt"]},"negPrompt":"ugly"}' @@ -409,11 +413,23 @@ def create_zoom_single( plaintext_to_html(""), ) + +def validatePromptJson_throws(data): + with open(jsonprompt_schemafile, "r") as s: schema = json.load(s) + validate(instance=data, schema=schema) + def putPrompts(files): - with open(files.name, 'r') as f: - file_contents = f.read() - data = json.loads(file_contents) - return [gr.DataFrame.update(data["prompts"]), gr.Textbox.update(data["negPrompt"])] + + try: + with open(files.name, 'r') as f: + file_contents = f.read() + data = json.loads(file_contents) + validatePromptJson_throws(data) + return [gr.DataFrame.update(data["prompts"]), gr.Textbox.update(data["negPrompt"])] + + except Exception: + gr.Error("loading your prompt failed. It seems to be invalid. Your prompt table is preserved.") + return [gr.DataFrame.update(), gr.Textbox.update()] def clearPrompts(): return [gr.DataFrame.update(value=[[0,"Infinite Zoom. Start over"]]), gr.Textbox.update("")] @@ -452,6 +468,7 @@ def on_ui_tabs(): try: jpr = json.loads(pr) + validatePromptJson_throws(jpr) except Exception: jpr = invalid_prompt diff --git a/scripts/promptschema.json b/scripts/promptschema.json new file mode 100644 index 0000000..fcafb0f --- /dev/null +++ b/scripts/promptschema.json @@ -0,0 +1,55 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "prompts": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "type": "array", + "items": [ + { + "oneOf": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "string" + } + ] + }, + { + "type": "string" + } + ], + "minItems": 2, + "maxItems": 2, + "uniqueItems": true + }, + "minItems": 1 + }, + "headers": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 2 + } + }, + "required": [ + "data", + "headers" + ] + }, + "negPrompt": { + "type": "string" + } + }, + "required": [ + "prompts", + "negPrompt" + ] +} \ No newline at end of file From ad1a7edc43533fbbe0842212df5a0b0fe2eda2f1 Mon Sep 17 00:00:00 2001 From: GeorgLegato Date: Thu, 20 Apr 2023 02:46:20 +0200 Subject: [PATCH 08/23] gradeio error not shown, added a print on invalid json --- scripts/infinite-zoom.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/infinite-zoom.py b/scripts/infinite-zoom.py index bc16156..945f674 100644 --- a/scripts/infinite-zoom.py +++ b/scripts/infinite-zoom.py @@ -429,6 +429,7 @@ def putPrompts(files): except Exception: gr.Error("loading your prompt failed. It seems to be invalid. Your prompt table is preserved.") + print("[InfiniteZoom:] Loading your prompt failed. It seems to be invalid. Your prompt table is preserved.") return [gr.DataFrame.update(), gr.Textbox.update()] def clearPrompts(): From f0a68e997d854ca46057c157e1d8e20e1e820537 Mon Sep 17 00:00:00 2001 From: GeorgLegato Date: Thu, 20 Apr 2023 03:35:12 +0200 Subject: [PATCH 09/23] use First Checkpiint if setting was never written --- scripts/infinite-zoom.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/infinite-zoom.py b/scripts/infinite-zoom.py index 945f674..2239825 100644 --- a/scripts/infinite-zoom.py +++ b/scripts/infinite-zoom.py @@ -242,7 +242,7 @@ def create_zoom_single( ) else: # switch to txt2img model - checkinfo = modules.sd_models.checkpoint_alisases[shared.opts.data.get("infzoom_txt2img_model")] + checkinfo = modules.sd_models.checkpoint_alisases[shared.opts.data.get("infzoom_txt2img_model", modules.sd_models.checkpoint_alisases[0])] if (not checkinfo): raise NameError("Checklist not found in registry") if progress: progress(0, desc="Loading Model for txt2img: " + checkinfo.name) @@ -736,8 +736,8 @@ def on_ui_settings(): shared.opts.add_option( "infzoom_txt2img_model", shared.OptionInfo( - "", - "Name of your desired model to render keyframes (txt2img), if empty current model used", + shared.list_checkpoint_tiles[0], + "Name of your desired model to render keyframes (txt2img)", gr.Dropdown, lambda: {"choices": shared.list_checkpoint_tiles()}, section=section From fa0e196fef26d430beddbc4d55a67e60b816876c Mon Sep 17 00:00:00 2001 From: GeorgLegato Date: Thu, 20 Apr 2023 03:36:22 +0200 Subject: [PATCH 10/23] use explizit Euler a on tetx2img when no setting ever written --- scripts/infinite-zoom.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/infinite-zoom.py b/scripts/infinite-zoom.py index 2dcfd84..45d6c2a 100644 --- a/scripts/infinite-zoom.py +++ b/scripts/infinite-zoom.py @@ -242,7 +242,7 @@ def create_zoom_single( ) else: # switch to txt2img model - checkinfo = modules.sd_models.checkpoint_alisases[shared.opts.data.get("infzoom_txt2img_model", modules.sd_models.checkpoint_alisases[0])] + checkinfo = modules.sd_models.checkpoint_alisases[shared.opts.data.get("infzoom_txt2img_model", "Euler a")] if (not checkinfo): raise NameError("Checklist not found in registry") if progress: progress(0, desc="Loading Model for txt2img: " + checkinfo.name) From 8482a7a6deec40c79c1d6c1bc2a19f5459064a0f Mon Sep 17 00:00:00 2001 From: GeorgLegato Date: Thu, 20 Apr 2023 04:03:46 +0200 Subject: [PATCH 11/23] handle NO MODEL secelected in setting, just use the current loaded model for both. --- scripts/infinite-zoom.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/scripts/infinite-zoom.py b/scripts/infinite-zoom.py index 45d6c2a..0b65d36 100644 --- a/scripts/infinite-zoom.py +++ b/scripts/infinite-zoom.py @@ -241,12 +241,14 @@ def create_zoom_single( (width, height), resample=Image.LANCZOS ) else: - # switch to txt2img model - checkinfo = modules.sd_models.checkpoint_alisases[shared.opts.data.get("infzoom_txt2img_model", "Euler a")] - if (not checkinfo): - raise NameError("Checklist not found in registry") - if progress: progress(0, desc="Loading Model for txt2img: " + checkinfo.name) - modules.sd_models.load_model(checkinfo) + modelname = shared.opts.data.get("infzoom_txt2img_model") + if (modelname): + # switch to txt2img model + checkinfo = modules.sd_models.checkpoint_alisases[modelname] + if (not checkinfo): + raise NameError("Checklist not found in registry") + if progress: progress(0, desc="Loading Model for txt2img: " + checkinfo.name) + modules.sd_models.load_model(checkinfo) processed = renderTxt2Img( prompts[min(k for k in prompts.keys() if k >= 0)], @@ -273,12 +275,14 @@ def create_zoom_single( all_frames.append(do_upscaleImg(current_image,upscale_do, upscaler_name,upscale_by) if upscale_do else current_image) - # switch to inpaint model now - checkinfo = modules.sd_models.checkpoint_alisases[shared.opts.data.get("infzoom_inpainting_model", "sd-v1-5-inpainting.ckpt")] - if (not checkinfo): - raise NameError("Checklist not found in registry") - if progress: progress(0, desc="Loading Model for inpainting/img2img: " + checkinfo.name) - modules.sd_models.load_model(checkinfo) + inmodelname = shared.opts.data.get("infzoom_inpainting_model") + if (inmodelname): + # switch to inpaint model now + checkinfo = modules.sd_models.checkpoint_alisases[inmodelname] + if (not checkinfo): + raise NameError("Checklist not found in registry") + if progress: progress(0, desc="Loading Model for inpainting/img2img: " + checkinfo.name) + modules.sd_models.load_model(checkinfo) for i in range(num_outpainting_steps): print_out = "Outpaint step: " + str(i + 1) + " / " + str(num_outpainting_steps) From 3543fcd2d389b051ce4eacba9d07da6a5b9a447c Mon Sep 17 00:00:00 2001 From: GeorgLegato Date: Thu, 20 Apr 2023 04:09:46 +0200 Subject: [PATCH 12/23] default value for models should None, when never written in onfig.json --- scripts/infinite-zoom.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/infinite-zoom.py b/scripts/infinite-zoom.py index 0b65d36..34b0d63 100644 --- a/scripts/infinite-zoom.py +++ b/scripts/infinite-zoom.py @@ -740,7 +740,7 @@ def on_ui_settings(): shared.opts.add_option( "infzoom_txt2img_model", shared.OptionInfo( - shared.list_checkpoint_tiles[0], + None, "Name of your desired model to render keyframes (txt2img)", gr.Dropdown, lambda: {"choices": shared.list_checkpoint_tiles()}, @@ -751,7 +751,7 @@ def on_ui_settings(): shared.opts.add_option( "infzoom_inpainting_model", shared.OptionInfo( - "sd-v1-5-inpainting.ckpt", + None, "Name of your desired inpaint model (img2img-inpaint). Default is vanilla sd-v1-5-inpainting.ckpt ", gr.Dropdown, lambda: {"choices": shared.list_checkpoint_tiles()}, From 614657b31994c4b820a776fd9fffc4c51444a6c2 Mon Sep 17 00:00:00 2001 From: GeorgLegato Date: Thu, 20 Apr 2023 04:17:08 +0200 Subject: [PATCH 13/23] minimize 1st column, only numbers here --- style.css | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/style.css b/style.css index 37f8dde..752c2d4 100644 --- a/style.css +++ b/style.css @@ -1,4 +1,9 @@ #tab_iz_interface .gradio-dataframe .controls-wrap { flex-direction: row-reverse; justify-content: space-between; +} + +/* first column min width */ +#tab_iz_interface th { + width: 0; } \ No newline at end of file From 9ba6f17ce3e2d9e9b8ee75387d5a09b71337b64b Mon Sep 17 00:00:00 2001 From: GeorgLegato Date: Thu, 20 Apr 2023 05:02:48 +0200 Subject: [PATCH 14/23] now 1st column is minimized --- style.css | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/style.css b/style.css index 752c2d4..4cc416a 100644 --- a/style.css +++ b/style.css @@ -4,6 +4,7 @@ } /* first column min width */ -#tab_iz_interface th { +#tab_iz_interface th:first-child { + flex: 0 0 0%; width: 0; -} \ No newline at end of file +} From 2d3bbe91cde9593665ceaa38673153d669359e41 Mon Sep 17 00:00:00 2001 From: "vahid K. nejad" Date: Thu, 20 Apr 2023 09:36:29 +0400 Subject: [PATCH 15/23] load_model_from_setting function --- scripts/infinite-zoom.py | 246 ++++++++++++++++++++++++--------------- 1 file changed, 154 insertions(+), 92 deletions(-) diff --git a/scripts/infinite-zoom.py b/scripts/infinite-zoom.py index 34b0d63..18a9547 100644 --- a/scripts/infinite-zoom.py +++ b/scripts/infinite-zoom.py @@ -27,16 +27,30 @@ import modules.sd_models import modules.sd_samplers from modules import scripts -usefulDirs = scripts.basedir().split(os.sep)[-2:] # contains install and our extension foldername -jsonprompt_schemafile = usefulDirs[0]+"/"+usefulDirs[1]+"/scripts/promptschema.json" + +usefulDirs = scripts.basedir().split(os.sep)[ + -2: +] # contains install and our extension foldername +jsonprompt_schemafile = ( + usefulDirs[0] + "/" + usefulDirs[1] + "/scripts/promptschema.json" +) available_samplers = [s.name for s in modules.sd_samplers.samplers] default_prompt = '{"prompts":{"data":[[0,"Cat"],["1","Dog"],["2","Happy Pets"]],"headers":["outpaint steps","prompt"]},"negPrompt":"ugly"}' -empty_prompt = '{"prompts":{"data":[],"headers":["outpaint steps","prompt"]},"negPrompt":""}' +empty_prompt = ( + '{"prompts":{"data":[],"headers":["outpaint steps","prompt"]},"negPrompt":""}' +) + +# must be python dict +invalid_prompt = { + "prompts": { + "data": [[0, "Your prompt-json is invalid, please check Settings"]], + "headers": ["outpaint steps", "prompt"], + }, + "negPrompt": "Invalid prompt-json", +} -#must be python dict -invalid_prompt ={"prompts":{"data":[[0,"Your prompt-json is invalid, please check Settings"]],"headers":["outpaint steps","prompt"]},"negPrompt":"Invalid prompt-json"} def closest_upper_divisible_by_eight(num): if num % 8 == 0: @@ -44,11 +58,25 @@ def closest_upper_divisible_by_eight(num): else: return math.ceil(num / 8) * 8 -def do_upscaleImg(curImg,upscale_do, upscaler_name,upscale_by): - if (not upscale_do): return curImg - pp= scripts.postprocessing_upscale.scripts_postprocessing.PostprocessedImage(curImg) + +def do_upscaleImg(curImg, upscale_do, upscaler_name, upscale_by): + if not upscale_do: + return curImg + pp = scripts.postprocessing_upscale.scripts_postprocessing.PostprocessedImage( + curImg + ) ups = scripts.postprocessing_upscale.ScriptPostprocessingUpscale() - ups.process(pp, upscale_mode=2, upscale_by=upscale_by, upscale_to_width=None, upscale_to_height=None, upscale_crop=False, upscaler_1_name=upscaler_name, upscaler_2_name=None, upscaler_2_visibility=0.0) + ups.process( + pp, + upscale_mode=2, + upscale_by=upscale_by, + upscale_to_width=None, + upscale_to_height=None, + upscale_crop=False, + upscaler_1_name=upscaler_name, + upscaler_2_name=None, + upscaler_2_visibility=0.0, + ) return pp.image @@ -71,6 +99,7 @@ def renderTxt2Img(prompt, negative_prompt, sampler, steps, cfg_scale, width, hei processed = process_images(p) return processed + def renderImg2Img( prompt, negative_prompt, @@ -125,6 +154,15 @@ def fix_env_Path_ffprobe(): os.environ["PATH"] = envpath + path_sep + ffppath +def load_model_from_setting(model_field_name): + model_name = shared.opts.data.get(model_field_name) + if model_name is not None and model_name != "": + checkinfo = modules.sd_models.checkpoint_alisases[model_name] + if not checkinfo: + raise NameError(model_field_name + " Does not exist in your models.") + modules.sd_models.load_model(checkinfo) + + def create_zoom( prompts_array, negative_prompt, @@ -179,7 +217,6 @@ def create_zoom( upscaler_name, upscale_by, progress, - ) return result @@ -241,14 +278,9 @@ def create_zoom_single( (width, height), resample=Image.LANCZOS ) else: - modelname = shared.opts.data.get("infzoom_txt2img_model") - if (modelname): - # switch to txt2img model - checkinfo = modules.sd_models.checkpoint_alisases[modelname] - if (not checkinfo): - raise NameError("Checklist not found in registry") - if progress: progress(0, desc="Loading Model for txt2img: " + checkinfo.name) - modules.sd_models.load_model(checkinfo) + load_model_from_setting("infzoom_txt2img_model") + if progress: + progress(0, desc="Loading Model for txt2img: " + checkinfo.name) processed = renderTxt2Img( prompts[min(k for k in prompts.keys() if k >= 0)], @@ -261,7 +293,6 @@ def create_zoom_single( ) current_image = processed.images[0] - mask_width = math.trunc(width / 4) # was initially 512px => 128px mask_height = math.trunc(height / 4) # was initially 512px => 128px @@ -269,25 +300,24 @@ def create_zoom_single( all_frames = [] - if upscale_do and progress: progress(0, desc="upscaling inital image") - all_frames.append(do_upscaleImg(current_image,upscale_do, upscaler_name,upscale_by) if upscale_do else current_image) + all_frames.append( + do_upscaleImg(current_image, upscale_do, upscaler_name, upscale_by) + if upscale_do + else current_image + ) - inmodelname = shared.opts.data.get("infzoom_inpainting_model") - if (inmodelname): - # switch to inpaint model now - checkinfo = modules.sd_models.checkpoint_alisases[inmodelname] - if (not checkinfo): - raise NameError("Checklist not found in registry") - if progress: progress(0, desc="Loading Model for inpainting/img2img: " + checkinfo.name) - modules.sd_models.load_model(checkinfo) + load_model_from_setting("infzoom_inpainting_model") + if progress: + progress(0, desc="Loading Model for inpainting/img2img: " + checkinfo.name) for i in range(num_outpainting_steps): print_out = "Outpaint step: " + str(i + 1) + " / " + str(num_outpainting_steps) print(print_out) - if progress: progress( ((i + 1) / num_outpainting_steps), desc=print_out) + if progress: + progress(((i + 1) / num_outpainting_steps), desc=print_out) prev_image_fix = current_image prev_image = shrink_and_paste_on_blank(current_image, mask_width, mask_height) @@ -374,21 +404,23 @@ def create_zoom_single( interpol_image.paste(prev_image_fix_crop, mask=prev_image_fix_crop) - if upscale_do and progress: - progress( - ((i + 1) / num_outpainting_steps), - desc="upscaling interpol" - ) + if upscale_do and progress: + progress(((i + 1) / num_outpainting_steps), desc="upscaling interpol") - all_frames.append(do_upscaleImg(interpol_image, upscale_do, upscaler_name,upscale_by) if upscale_do else interpol_image) - - if upscale_do and progress: - progress( - ((i + 1) / num_outpainting_steps), - desc="upscaling current" + all_frames.append( + do_upscaleImg(interpol_image, upscale_do, upscaler_name, upscale_by) + if upscale_do + else interpol_image ) - all_frames.append(do_upscaleImg(current_image,upscale_do, upscaler_name,upscale_by) if upscale_do else current_image) + if upscale_do and progress: + progress(((i + 1) / num_outpainting_steps), desc="upscaling current") + + all_frames.append( + do_upscaleImg(current_image, upscale_do, upscaler_name, upscale_by) + if upscale_do + else current_image + ) video_file_name = "infinite_zoom_" + str(int(time.time())) + ".mp4" output_path = shared.opts.data.get( @@ -419,25 +451,37 @@ def create_zoom_single( def validatePromptJson_throws(data): - with open(jsonprompt_schemafile, "r") as s: schema = json.load(s) + with open(jsonprompt_schemafile, "r") as s: + schema = json.load(s) validate(instance=data, schema=schema) - -def putPrompts(files): + +def putPrompts(files): try: - with open(files.name, 'r') as f: + with open(files.name, "r") as f: file_contents = f.read() data = json.loads(file_contents) validatePromptJson_throws(data) - return [gr.DataFrame.update(data["prompts"]), gr.Textbox.update(data["negPrompt"])] - + return [ + gr.DataFrame.update(data["prompts"]), + gr.Textbox.update(data["negPrompt"]), + ] + except Exception: - gr.Error("loading your prompt failed. It seems to be invalid. Your prompt table is preserved.") - print("[InfiniteZoom:] Loading your prompt failed. It seems to be invalid. Your prompt table is preserved.") + gr.Error( + "loading your prompt failed. It seems to be invalid. Your prompt table is preserved." + ) + print( + "[InfiniteZoom:] Loading your prompt failed. It seems to be invalid. Your prompt table is preserved." + ) return [gr.DataFrame.update(), gr.Textbox.update()] + def clearPrompts(): - return [gr.DataFrame.update(value=[[0,"Infinite Zoom. Start over"]]), gr.Textbox.update("")] + return [ + gr.DataFrame.update(value=[[0, "Infinite Zoom. Start over"]]), + gr.Textbox.update(""), + ] def on_ui_tabs(): @@ -456,7 +500,6 @@ def on_ui_tabs(): interrupt = gr.Button(value="Interrupt", elem_id="interrupt_training") with gr.Row(): with gr.Column(scale=1, variant="panel"): - with gr.Tab("Main"): main_outpaint_steps = gr.Slider( minimum=2, @@ -468,8 +511,9 @@ def on_ui_tabs(): ) # safe reading json prompt - pr = shared.opts.data.get("infzoom_defPrompt",default_prompt) - if (not pr): pr = empty_prompt + pr = shared.opts.data.get("infzoom_defPrompt", default_prompt) + if not pr: + pr = empty_prompt try: jpr = json.loads(pr) @@ -488,7 +532,7 @@ def on_ui_tabs(): ) main_negative_prompt = gr.Textbox( - value=jpr["negPrompt"], label="Negative Prompt" + value=jpr["negPrompt"], label="Negative Prompt" ) # these button will be moved using JS unde the dataframe view as small ones @@ -516,8 +560,17 @@ def on_ui_tabs(): inputs=[importPrompts_button], ) - clearPrompts_button= gr.Button(value="Clear prompts",variant="secondary",elem_classes="sm infzoom_tab_butt", elem_id="infzoom_clP_butt") - clearPrompts_button.click(fn=clearPrompts,inputs=[],outputs=[main_prompts,main_negative_prompt]) + clearPrompts_button = gr.Button( + value="Clear prompts", + variant="secondary", + elem_classes="sm infzoom_tab_butt", + elem_id="infzoom_clP_butt", + ) + clearPrompts_button.click( + fn=clearPrompts, + inputs=[], + outputs=[main_prompts, main_negative_prompt], + ) main_sampler = gr.Dropdown( label="Sampler", @@ -557,7 +610,9 @@ def on_ui_tabs(): ) with gr.Row(): init_image = gr.Image(type="pil", label="custom initial image") - exit_image = gr.Image(type="pil", label="custom exit image", visible=False) #TODO: implement exit-image rendering + exit_image = gr.Image( + type="pil", label="custom exit image", visible=False + ) # TODO: implement exit-image rendering batchcount_slider = gr.Slider( minimum=1, @@ -622,16 +677,23 @@ def on_ui_tabs(): with gr.Tab("Post proccess"): upscale_do = gr.Checkbox(False, label="Enable Upscale") - upscaler_name = gr.Dropdown(label='Upscaler', elem_id="infZ_upscaler", choices=[x.name for x in shared.sd_upscalers], value=shared.sd_upscalers[0].name) + upscaler_name = gr.Dropdown( + label="Upscaler", + elem_id="infZ_upscaler", + choices=[x.name for x in shared.sd_upscalers], + value=shared.sd_upscalers[0].name, + ) upscale_by = gr.Slider( label="Upscale by factor", minimum=1, maximum=8, value=1 ) - with gr.Accordion("Help",open=False): - gr.Markdown("""# Performance critical + with gr.Accordion("Help", open=False): + gr.Markdown( + """# Performance critical Depending on amount of frames and which upscaler you choose it might took a long time to render. Our best experience and trade-off is the R-ERSGAn4x upscaler. -""") +""" + ) with gr.Column(scale=1, variant="compact"): output_video = gr.Video(label="Output").style(width=512, height=512) @@ -644,7 +706,7 @@ Our best experience and trade-off is the R-ERSGAn4x upscaler. "infinite-zoom", shared.opts.outdir_img2img_samples ) generate_btn.click( - fn=wrap_gradio_gpu_call(create_zoom, extra_outputs=[None, '', '']), + fn=wrap_gradio_gpu_call(create_zoom, extra_outputs=[None, "", ""]), inputs=[ main_prompts, main_negative_prompt, @@ -669,8 +731,7 @@ Our best experience and trade-off is the R-ERSGAn4x upscaler. main_sampler, upscale_do, upscaler_name, - upscale_by - + upscale_by, ], outputs=[output_video, out_image, generation_info, html_info, html_log], ) @@ -733,42 +794,43 @@ def on_ui_settings(): "Writing videos has dependency to an existing FFPROBE executable on your machine. D/L here (https://github.com/BtbN/FFmpeg-Builds/releases) your OS variant and point to your installation path", gr.Textbox, {"interactive": True}, - section=section - ) + section=section, + ), ) shared.opts.add_option( - "infzoom_txt2img_model", + "infzoom_txt2img_model", shared.OptionInfo( - None, - "Name of your desired model to render keyframes (txt2img)", - gr.Dropdown, + None, + "Name of your desired model to render keyframes (txt2img)", + gr.Dropdown, lambda: {"choices": shared.list_checkpoint_tiles()}, - section=section - ) - ) - - shared.opts.add_option( - "infzoom_inpainting_model", - shared.OptionInfo( - None, - "Name of your desired inpaint model (img2img-inpaint). Default is vanilla sd-v1-5-inpainting.ckpt ", - gr.Dropdown, - lambda: {"choices": shared.list_checkpoint_tiles()}, - section=section - ) + section=section, + ), ) shared.opts.add_option( - "infzoom_defPrompt", + "infzoom_inpainting_model", shared.OptionInfo( - default_prompt, - "Default prompt-setup to start with'", - gr.Code, - {"interactive": True, "language":"json"}, - section=section - ) + None, + "Name of your desired inpaint model (img2img-inpaint). Default is vanilla sd-v1-5-inpainting.ckpt ", + gr.Dropdown, + lambda: {"choices": shared.list_checkpoint_tiles()}, + section=section, + ), ) - + + shared.opts.add_option( + "infzoom_defPrompt", + shared.OptionInfo( + default_prompt, + "Default prompt-setup to start with'", + gr.Code, + {"interactive": True, "language": "json"}, + section=section, + ), + ) + + script_callbacks.on_ui_tabs(on_ui_tabs) script_callbacks.on_ui_settings(on_ui_settings) From ed86daf515eba30ab1ddccad3576ac296550d4d3 Mon Sep 17 00:00:00 2001 From: GeorgLegato Date: Thu, 20 Apr 2023 17:43:59 +0200 Subject: [PATCH 16/23] fixed load_setting_model to keep al logic inside AND compatible with Vlad1111 --- scripts/infinite-zoom.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/scripts/infinite-zoom.py b/scripts/infinite-zoom.py index 18a9547..87f05b0 100644 --- a/scripts/infinite-zoom.py +++ b/scripts/infinite-zoom.py @@ -154,12 +154,25 @@ def fix_env_Path_ffprobe(): os.environ["PATH"] = envpath + path_sep + ffppath -def load_model_from_setting(model_field_name): +def load_model_from_setting(model_field_name, progress, progress_desc): + # fix typo in Automatic1111 vs Vlad111 + if hasattr(modules.sd_models, "checkpoint_alisases"): + checkPList = modules.sd_models.checkpoint_alisases + if hasattr(modules.sd_models, "checkpoint_aliases"): + checkPList = modules.sd_models.checkpoint_aliases + else: + raise Exception("This is not a compatible StableDiffusion Platform, can not access checkpoints") + model_name = shared.opts.data.get(model_field_name) if model_name is not None and model_name != "": - checkinfo = modules.sd_models.checkpoint_alisases[model_name] + checkinfo = checkPList[model_name] + if not checkinfo: raise NameError(model_field_name + " Does not exist in your models.") + + if progress: + progress(0, desc=progress_desc + checkinfo.name) + modules.sd_models.load_model(checkinfo) @@ -278,9 +291,7 @@ def create_zoom_single( (width, height), resample=Image.LANCZOS ) else: - load_model_from_setting("infzoom_txt2img_model") - if progress: - progress(0, desc="Loading Model for txt2img: " + checkinfo.name) + load_model_from_setting("infzoom_txt2img_model", progress, "Loading Model for txt2img: ") processed = renderTxt2Img( prompts[min(k for k in prompts.keys() if k >= 0)], @@ -309,9 +320,7 @@ def create_zoom_single( else current_image ) - load_model_from_setting("infzoom_inpainting_model") - if progress: - progress(0, desc="Loading Model for inpainting/img2img: " + checkinfo.name) + load_model_from_setting("infzoom_inpainting_model", progress, "Loading Model for inpainting/img2img: " ) for i in range(num_outpainting_steps): print_out = "Outpaint step: " + str(i + 1) + " / " + str(num_outpainting_steps) From 8bcea44d2122449a9205facfebf7ce39af0a8583 Mon Sep 17 00:00:00 2001 From: GeorgLegato Date: Thu, 20 Apr 2023 18:51:59 +0200 Subject: [PATCH 17/23] less restrictive json, pretty print on default json --- scripts/infinite-zoom.py | 15 ++++++- scripts/promptschema.json | 84 ++++++++++++++++++--------------------- 2 files changed, 53 insertions(+), 46 deletions(-) diff --git a/scripts/infinite-zoom.py b/scripts/infinite-zoom.py index 87f05b0..f756635 100644 --- a/scripts/infinite-zoom.py +++ b/scripts/infinite-zoom.py @@ -37,7 +37,20 @@ jsonprompt_schemafile = ( available_samplers = [s.name for s in modules.sd_samplers.samplers] -default_prompt = '{"prompts":{"data":[[0,"Cat"],["1","Dog"],["2","Happy Pets"]],"headers":["outpaint steps","prompt"]},"negPrompt":"ugly"}' +default_prompt = """ +{ + "prompts":{ + "headers":["outpaint steps","prompt"], + "data":[ + [0,"Cat"], + ["1","Dog"], + [2,"Happy Pets"] + ] + }, + "negPrompt":"ugly" +} +""" + empty_prompt = ( '{"prompts":{"data":[],"headers":["outpaint steps","prompt"]},"negPrompt":""}' ) diff --git a/scripts/promptschema.json b/scripts/promptschema.json index fcafb0f..b7fcfd4 100644 --- a/scripts/promptschema.json +++ b/scripts/promptschema.json @@ -2,54 +2,48 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "prompts": { - "type": "object", - "properties": { - "data": { - "type": "array", - "items": { - "type": "array", - "items": [ - { - "oneOf": [ - { - "type": "integer", - "minimum": 0 - }, - { - "type": "string" - } - ] - }, - { - "type": "string" - } - ], - "minItems": 2, - "maxItems": 2, - "uniqueItems": true + "prompts": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "type": "array", + "items": [ + { + "oneOf": [ + { + "type": "integer", + "minimum": 0 }, - "minItems": 1 + { + "type": "string" + } + ] }, - "headers": { - "type": "array", - "items": { - "type": "string" - }, - "minItems": 2 + { + "type": "string" } + ], + "minItems": 0, + "maxItems": 999, + "uniqueItems": false }, - "required": [ - "data", - "headers" - ] + "minItems": 0 + }, + "headers": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 2 + } }, - "negPrompt": { - "type": "string" - } + "required": ["data", "headers"] + }, + "negPrompt": { + "type": "string" + } }, - "required": [ - "prompts", - "negPrompt" - ] -} \ No newline at end of file + "required": ["prompts", "negPrompt"] + } \ No newline at end of file From aa1cfb00d424c21a6a245ce33c71ce7267e5ecb3 Mon Sep 17 00:00:00 2001 From: GeorgLegato Date: Thu, 20 Apr 2023 19:13:21 +0200 Subject: [PATCH 18/23] assert all frame have same size as first one --- iz_helpers/video.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/iz_helpers/video.py b/iz_helpers/video.py index 63aad92..c3d61a3 100644 --- a/iz_helpers/video.py +++ b/iz_helpers/video.py @@ -13,8 +13,9 @@ def write_video(file_path, frames, fps, reversed=True, start_frame_dupe_amount=1 if reversed == True: frames = frames[::-1] - # Get dimensions of the frames - # w, h = frames[0].size + # Get dimensions of the first frames, all subsequent has to be same sized + for k in frames: + assert (k.size == frames[0].size,"Different frame sizes found!") # Create an imageio video writer, avoid block size of 512. writer = imageio.get_writer(file_path, fps=fps, macro_block_size=None) From 95ed85cd8a1bb62c43e8360d5bbba7d0d8398106 Mon Sep 17 00:00:00 2001 From: GeorgLegato Date: Fri, 21 Apr 2023 00:07:22 +0200 Subject: [PATCH 19/23] fixed glitch in checking checkpoint_aliases vs checkpoint_alisases (typo) --- scripts/infinite-zoom.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/infinite-zoom.py b/scripts/infinite-zoom.py index f756635..4932038 100644 --- a/scripts/infinite-zoom.py +++ b/scripts/infinite-zoom.py @@ -171,7 +171,7 @@ def load_model_from_setting(model_field_name, progress, progress_desc): # fix typo in Automatic1111 vs Vlad111 if hasattr(modules.sd_models, "checkpoint_alisases"): checkPList = modules.sd_models.checkpoint_alisases - if hasattr(modules.sd_models, "checkpoint_aliases"): + elif hasattr(modules.sd_models, "checkpoint_aliases"): checkPList = modules.sd_models.checkpoint_aliases else: raise Exception("This is not a compatible StableDiffusion Platform, can not access checkpoints") From 77f64f994c3ce3fc8484b527e934b9674965ccb8 Mon Sep 17 00:00:00 2001 From: GeorgLegato Date: Fri, 21 Apr 2023 00:56:21 +0200 Subject: [PATCH 20/23] fixed upscaling-import for Vlad1111, fixed name for "Import Prompt" button --- scripts/infinite-zoom.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/infinite-zoom.py b/scripts/infinite-zoom.py index 4932038..c28ed53 100644 --- a/scripts/infinite-zoom.py +++ b/scripts/infinite-zoom.py @@ -20,7 +20,7 @@ from modules.processing import ( StableDiffusionProcessingImg2Img, ) -import scripts.postprocessing_upscale +from scripts import postprocessing_upscale from modules.ui import create_output_panel, plaintext_to_html import modules.sd_models @@ -75,10 +75,10 @@ def closest_upper_divisible_by_eight(num): def do_upscaleImg(curImg, upscale_do, upscaler_name, upscale_by): if not upscale_do: return curImg - pp = scripts.postprocessing_upscale.scripts_postprocessing.PostprocessedImage( + pp = postprocessing_upscale.scripts_postprocessing.PostprocessedImage( curImg ) - ups = scripts.postprocessing_upscale.ScriptPostprocessingUpscale() + ups = postprocessing_upscale.ScriptPostprocessingUpscale() ups.process( pp, upscale_mode=2, @@ -565,7 +565,7 @@ def on_ui_tabs(): elem_id="infzoom_exP_butt", ) importPrompts_button = gr.UploadButton( - value="Import prompts", + label="Import prompts", variant="secondary", elem_classes="sm infzoom_tab_butt", elem_id="infzoom_imP_butt", From 93b91327ab62873036d5d1329b8baa451e27b520 Mon Sep 17 00:00:00 2001 From: GeorgLegato Date: Fri, 21 Apr 2023 04:04:53 +0200 Subject: [PATCH 21/23] fix #45, odd width/height crashed ffmpeg --- scripts/infinite-zoom.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/scripts/infinite-zoom.py b/scripts/infinite-zoom.py index c28ed53..f50163a 100644 --- a/scripts/infinite-zoom.py +++ b/scripts/infinite-zoom.py @@ -72,19 +72,41 @@ def closest_upper_divisible_by_eight(num): return math.ceil(num / 8) * 8 +# example fail: 720 px width * 1.66 upscale => 1195.2 => 1195 crash +# 512 px * 1.66 = 513.66 = ? +# assume ffmpeg will CUT to integer +# 721 /720 + def do_upscaleImg(curImg, upscale_do, upscaler_name, upscale_by): if not upscale_do: return curImg + + # ensure even width and even height for ffmpeg + # if odd, switch to scale to mode + rwidth = round(curImg.width * upscale_by) + rheight = round(curImg.height * upscale_by) + + ups_mode = 2 # upscale_by + if ( (rwidth %2) == 1 ): + ups_mode = 1 + rwidth += 1 + if ( (rheight %2) == 1 ): + ups_mode = 1 + rheight += 1 + + if (1 == ups_mode ): + print ("Infinite Zoom: aligning output size to even width and height: " + str(rwidth) +" x "+str(rheight), end='\r' ) + pp = postprocessing_upscale.scripts_postprocessing.PostprocessedImage( curImg ) ups = postprocessing_upscale.ScriptPostprocessingUpscale() ups.process( pp, - upscale_mode=2, + upscale_mode=ups_mode, upscale_by=upscale_by, - upscale_to_width=None, - upscale_to_height=None, + upscale_to_width=rwidth, + upscale_to_height=rheight, upscale_crop=False, upscaler_1_name=upscaler_name, upscaler_2_name=None, @@ -216,6 +238,7 @@ def create_zoom( upscale_by, progress=None, ): + for i in range(batchcount): print(f"Batch {i+1}/{batchcount}") result = create_zoom_single( From 29dccff5fd20a47b06c14caa63b4492ce809414e Mon Sep 17 00:00:00 2001 From: vahid khroasani <62482657+v8hid@users.noreply.github.com> Date: Fri, 21 Apr 2023 07:10:30 +0400 Subject: [PATCH 22/23] More convenient default prompt added Signed-off-by: vahid khroasani <62482657+v8hid@users.noreply.github.com> --- scripts/infinite-zoom.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/infinite-zoom.py b/scripts/infinite-zoom.py index f50163a..8333a4e 100644 --- a/scripts/infinite-zoom.py +++ b/scripts/infinite-zoom.py @@ -42,12 +42,10 @@ default_prompt = """ "prompts":{ "headers":["outpaint steps","prompt"], "data":[ - [0,"Cat"], - ["1","Dog"], - [2,"Happy Pets"] + [0,"Huge spectacular Waterfall in a dense tropical forest,epic perspective,(vegetation overgrowth:1.3)(intricate, ornamentation:1.1),(baroque:1.1), fantasy, (realistic:1) digital painting , (magical,mystical:1.2) , (wide angle shot:1.4), (landscape composed:1.2)(medieval:1.1), divine,cinematic,(tropical forest:1.4),(river:1.3)mythology,india, volumetric lighting, Hindu ,epic, Alex Horley Wenjun Lin greg rutkowski Ruan Jia (Wayne Barlowe:1.2) "], ] }, - "negPrompt":"ugly" + "negPrompt":"frames, borderline, text, character, duplicate, error, out of frame, watermark, low quality, ugly, deformed, blur bad-artist" } """ From c5f77a50e3f5d1dadb0c4e0971b76f702776f755 Mon Sep 17 00:00:00 2001 From: GeorgLegato Date: Fri, 21 Apr 2023 05:42:32 +0200 Subject: [PATCH 23/23] filter UniPC. Set def output folder to "Outputs" --- scripts/infinite-zoom.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/infinite-zoom.py b/scripts/infinite-zoom.py index 8333a4e..6ebe1ad 100644 --- a/scripts/infinite-zoom.py +++ b/scripts/infinite-zoom.py @@ -35,7 +35,7 @@ jsonprompt_schemafile = ( usefulDirs[0] + "/" + usefulDirs[1] + "/scripts/promptschema.json" ) -available_samplers = [s.name for s in modules.sd_samplers.samplers] +available_samplers = [s.name for s in modules.sd_samplers.samplers if "UniPc" not in s.name] default_prompt = """ { @@ -787,10 +787,11 @@ def on_ui_settings(): section = ("infinite-zoom", "Infinite Zoom") shared.opts.add_option( + "outputs" "infzoom_outpath", shared.OptionInfo( "", - "Path where to store your infinite video. Let empty to use img2img-output", + "Path where to store your infinite video. Default is Outputs", gr.Textbox, {"interactive": True}, section=section,