Include alpha masks - preview

iz_helpers/image.py

@@ -42,3 +42,14 @@ def open_image(image_path):
         img = Image.open(image_path)
     
     return img
+
+def apply_alpha_mask(current_image, mask_image):
+
+    # Resize the mask to match the current image size
+    mask_image = mask_image.resize(current_image.size)
+
+    # Apply the mask as the alpha layer of the current image
+    result_image = current_image.copy()
+    result_image.putalpha(mask_image.convert('L')) # convert to grayscale
+
+    return result_image

iz_helpers/promptschema.json

@@ -26,6 +26,12 @@
                 },
                 {
                   "type": "string"
+                },
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "boolean"
                 }
               ],
               "minItems": 0,
@@ -39,7 +45,7 @@
             "items": {
               "type": "string"
             },
-            "minItems": 3
+            "minItems": 5
           }
         },
         "required": ["data", "headers"]

iz_helpers/run.py

@@ -11,7 +11,7 @@ from .helpers import (
     do_upscaleImg,
 )
 from .sd_helpers import renderImg2Img, renderTxt2Img
-from .image import shrink_and_paste_on_blank, open_image
+from .image import shrink_and_paste_on_blank, open_image, apply_alpha_mask
 from .video import write_video
 
 
@@ -112,14 +112,20 @@ def create_zoom_single(
 
     prompts = {}
     prompt_images = {}
+    prompt_alpha_mask_images = {}
+    prompt_image_is_keyframe = {}
 
     for x in prompts_array:
         try:
             key = int(x[0])
             value = str(x[1])
             file_loc = str(x[2])
+            alpha_mask_loc = str(x[3])
+            is_keyframe = bool(x[4])
             prompts[key] = value
             prompt_images[key] = file_loc
+            prompt_alpha_mask_images[key] = alpha_mask_loc
+            prompt_image_is_keyframe[key] = is_keyframe
         except ValueError:
             pass
     assert len(prompts_array) > 0, "prompts is empty"
@@ -143,9 +149,9 @@ def create_zoom_single(
         print("using Custom Initial Image")
     else:
         if prompt_images[min(k for k in prompt_images.keys() if k >= 0)] == "":
-        load_model_from_setting(
+            load_model_from_setting(
-            "infzoom_txt2img_model", progress, "Loading Model for txt2img: "
+                "infzoom_txt2img_model", progress, "Loading Model for txt2img: "
-        )
+            )
 
             processed, current_seed = renderTxt2Img(
             prompts[min(k for k in prompts.keys() if k >= 0)],
@@ -156,13 +162,18 @@ def create_zoom_single(
             current_seed,
             width,
             height,
-        )
+            )
-        current_image = processed.images[0]
+            current_image = processed.images[0]
         else:
             current_image = open_image(prompt_images[min(k for k in prompt_images.keys() if k >= 0)]).resize(
                 (width, height), resample=Image.LANCZOS
             )
 
+    # apply available alpha mask
+    if prompt_alpha_mask_images[min(k for k in prompt_alpha_mask_images.keys() if k >= 0)] != "":
+        current_image = apply_alpha_mask(current_image, open_image(prompt_alpha_mask_images[min(k for k in prompt_alpha_mask_images.keys() if k >= 0)]))
+
+
     mask_width = math.trunc(width / 4)  # was initially 512px => 128px
     mask_height = math.trunc(height / 4)  # was initially 512px => 128px
 
@@ -208,6 +219,8 @@ def create_zoom_single(
         # inpainting step
         current_image = current_image.convert("RGB")
 
+        paste_previous_image = prompt_image_is_keyframe[max(k for k in prompt_image_is_keyframe.keys() if k <= (i + 1))]
+        
         # Custom and specified images work like keyframes
         if custom_exit_image and (i + 1) >= (num_outpainting_steps + extra_frames):
             current_image = custom_exit_image.resize(
@@ -218,31 +231,37 @@ def create_zoom_single(
             if prompt_images[max(k for k in prompt_images.keys() if k <= (i + 1))] == "":
                 processed, current_seed = renderImg2Img(
                     prompts[max(k for k in prompts.keys() if k <= (i + 1))],
-                negative_prompt,
+                    negative_prompt,
-                sampler,
+                    sampler,
-                num_inference_steps,
+                    num_inference_steps,
-                guidance_scale,
+                    guidance_scale,
-                current_seed,
+                    current_seed,
-                width,
+                    width,
-                height,
+                    height,
-                current_image,
+                    current_image,
-                mask_image,
+                    mask_image,
-                inpainting_denoising_strength,
+                    inpainting_denoising_strength,
-                inpainting_mask_blur,
+                    inpainting_mask_blur,
-                inpainting_fill_mode,
+                    inpainting_fill_mode,
-                inpainting_full_res,
+                    inpainting_full_res,
-                inpainting_padding,
+                    inpainting_padding,
-            )
+                )
-            current_image = processed.images[0]
+                current_image = processed.images[0]
                 # only paste previous image when generating a new image
-                current_image.paste(prev_image, mask=prev_image)
+                #current_image.paste(prev_image, mask=prev_image)
+                paste_previous_image = True
             else:
                 current_image = open_image(prompt_images[max(k for k in prompt_images.keys() if k <= (i + 1))]).resize(
                     (width, height), resample=Image.LANCZOS
                 )
 
+        # apply available alpha mask
+        if prompt_alpha_mask_images[max(k for k in prompt_alpha_mask_images.keys() if k <= (i + 1))] != "":
+            current_image = apply_alpha_mask(current_image, open_image(prompt_alpha_mask_images[max(k for k in prompt_alpha_mask_images.keys() if  k <= (i + 1))]))
 
-        current_image.paste(prev_image, mask=prev_image)
+        # paste previous image on current image
+        if paste_previous_image:
+            current_image.paste(prev_image, mask=prev_image)
 
         # interpolation steps between 2 inpainted images (=sequential zoom and crop)
         for j in range(num_interpol_frames - 1):

iz_helpers/static_variables.py

@@ -5,12 +5,12 @@ import modules.sd_samplers
 default_prompt = """
 {
     "prompts":{
-        "headers":["outpaint steps","prompt"],
+        "headers":["outpaint steps","prompt","image location","blend mask location", "is keyframe"],
         "data":[
-            [0,"Huge spectacular Waterfall in a dense tropical forest,epic perspective,(vegetation overgrowth:1.3)(intricate, ornamentation:1.1),(baroque:1.1), fantasy, (realistic:1) digital painting , (magical,mystical:1.2) , (wide angle shot:1.4), (landscape composed:1.2)(medieval:1.1), divine,cinematic,(tropical forest:1.4),(river:1.3)mythology,india, volumetric lighting, Hindu ,epic,  Alex Horley Wenjun Lin greg rutkowski Ruan Jia (Wayne Barlowe:1.2) <lora:epiNoiseoffset_v2:0.6> "]
+            [0,"Huge spectacular Waterfall in a dense tropical forest,epic perspective,(vegetation overgrowth:1.3)(intricate, ornamentation:1.1),(baroque:1.1), fantasy, (realistic:1) digital painting , (magical,mystical:1.2) , (wide angle shot:1.4), (landscape composed:1.2)(medieval:1.1), divine,cinematic,(tropical forest:1.4),(river:1.3)mythology,india, volumetric lighting, Hindu ,epic,  Alex Horley Wenjun Lin greg rutkowski Ruan Jia (Wayne Barlowe:1.2) <lora:epiNoiseoffset_v2:0.6> ","C:\\path\\to\\image.png", "C:\\path\\to\\mask_image.png", false]
         ]
     },
-    "negPrompt":"frames, borderline, text, character, duplicate, error, out of frame, watermark, low quality, ugly, deformed, blur  bad-artist"
+    "negPrompt":"frames, borderline, text, character, duplicate, error, out of frame, watermark, low quality, ugly, deformed, blur bad-artist"
 }
 """
 available_samplers = [
@@ -18,13 +18,13 @@ available_samplers = [
 ]
 
 empty_prompt = (
-    '{"prompts":{"data":[],"headers":["outpaint steps","prompt"]},"negPrompt":""}'
+    '{"prompts":{"data":[],"headers":["outpaint steps","prompt","image location", "blend mask location", "is keyframe"]},"negPrompt":""}'
 )
 
 invalid_prompt = {
     "prompts": {
-        "data": [[0, "Your prompt-json is invalid, please check Settings"]],
+        "data": [[0, "Your prompt-json is invalid, please check Settings","", "", False]],
-        "headers": ["outpaint steps", "prompt"],
+        "headers": ["outpaint steps", "prompt","image location","blend mask location", "is keyframe"],
     },
     "negPrompt": "Invalid prompt-json",
 }

iz_helpers/ui.py

@@ -51,10 +51,10 @@ def on_ui_tabs():
 
                     main_prompts = gr.Dataframe(
                         type="array",
-                        headers=["outpaint step", "prompt"],
+                        headers=["outpaint step", "prompt", "image location", "blend mask", "is keyframe"],
-                        datatype=["number", "str"],
+                        datatype=["number", "str", "str", "str", "bool"],                        
                         row_count=1,
-                        col_count=(2, "fixed"),
+                        col_count=(5, "fixed"),
                         value=jpr["prompts"],
                         wrap=True,
                     )

iz_helpers/video.py

@@ -14,7 +14,7 @@ def write_video(file_path, frames, fps, reversed=True, start_frame_dupe_amount=1
         frames = frames[::-1]
 
     # Drop missformed frames
-    frames = [frame for frame in frames if frame.size == frames[0].size]
+    frames = [frame.convert("RGBA") for frame in frames if frame.size == frames[0].size]
 
     # Create an imageio video writer, avoid block size of 512.
     writer = imageio.get_writer(file_path, fps=fps, macro_block_size=None)