Add stage 8(blend with any background)

Add Invert mask mode(img2img/ebsynth for background)
Add mask creation option
Parameters are now saved in a text file.
Mask now affects keyframe selection.
update readme

README.md

@@ -5,12 +5,19 @@
 #### This extension allows you to output edited videos using ebsynth.(AE is not required)
 
 ## Example
+- The following sample is raw output of this extension.
 #### sample 1
 <div><video controls src="https://user-images.githubusercontent.com/118420657/213474231-38cac10e-7e75-43e1-b912-4e7727074d39.mp4" muted="false"></video></div>
 
 #### sample 2
 <div><video controls src="https://user-images.githubusercontent.com/118420657/213474343-e49e797d-386e-459f-9be9-2241b2d6266d.mp4" muted="false"></video></div>
 
+#### sample 3 blend background
+- person : masterpiece, best quality, masterpiece, 1girl, masterpiece, best quality,anime screencap, anime style
+- background : cyberpunk, factory, room ,anime screencap, anime style
+It is also possible to blend with your favorite videos.
+
+
 ## Installation
 - Install [ffmpeg](https://ffmpeg.org/) for your operating system
   (https://www.geeksforgeeks.org/how-to-install-ffmpeg-on-windows/)
@@ -37,7 +44,9 @@ All frames of the video and mask images for all frames are generated.
   
 #### Stage 2
 In the implementation of this extension, the keyframe interval is chosen to be shorter where there is a lot of motion and longer where there is little motion.  
+If the animation breaks up, increase the keyframe, if it flickers, decrease the keyframe.  
 First, generate one time with the default settings and go straight ahead without worrying about the result.  
+
   
 #### Stage 3
 Select one of the keyframes, throw it to img2img, and run [Interrogate DeepBooru].  
@@ -45,19 +54,21 @@ Delete unwanted words such as blur from the displayed prompt.
 Fill in the rest of the settings as you would normally do for image generation.  
   
 Here is the settings I used.  
-- Sampling method : DDIM  
+- Sampling method : Euler a  
 - Sampling Steps : 50  
 - Width : 960  
 - Height : 512  
 - CFG Scale : 20  
-- Denoising strength : 0.35  
+- Denoising strength : 0.2  
   
 Here is the settings for extension.  
-- Img2Img Repeat Count : 1 ( or 3 with Euler a / lower Denoising strength)  
+- Mask Mode(Override img2img Mask mode) : Normal
+- Img2Img Repeat Count : 5  
+- Add N to seed when repeating : 1
 - use Face Crop img2img : True  
 - Face Detection Method : YuNet  
 - Max Crop Size : 1024  
-- Face Denoising Strength : 0.35  
+- Face Denoising Strength : 0.25  
 - Face Area Magnification : 1.5 (The larger the number, the closer to the model's painting style, but the more likely it is to shift when merged with the body.)  
 - Enable Face Prompt : False  
   

ebsynth_utility.py

@@ -5,10 +5,12 @@ from modules.ui import plaintext_to_html
 import cv2
 import glob
 
-from extensions.ebsynth_utility.stage1 import ebsynth_utility_stage1
+from extensions.ebsynth_utility.stage1 import ebsynth_utility_stage1,ebsynth_utility_stage1_invert
 from extensions.ebsynth_utility.stage2 import ebsynth_utility_stage2
 from extensions.ebsynth_utility.stage5 import ebsynth_utility_stage5
 from extensions.ebsynth_utility.stage7 import ebsynth_utility_stage7
+from extensions.ebsynth_utility.stage8 import ebsynth_utility_stage8
+
 
 def x_ceiling(value, step):
   return -(-value // step) * step
@@ -21,11 +23,12 @@ def dump_dict(string, d:dict):
 class debug_string:
     txt = ""
     def print(self, comment):
+        print(comment)
         self.txt += comment + '\n'
     def to_string(self):
         return self.txt
 
-def ebsynth_utility_process(stage_index: int, project_dir:str, original_movie_path:str, key_min_gap:int, key_max_gap:int, key_th:float, key_add_last_frame:bool, blend_rate:float, export_type:str, no_mask_mode:bool):
+def ebsynth_utility_process(stage_index: int, project_dir:str, original_movie_path:str, tb_use_fast_mode:bool, tb_use_jit:bool, key_min_gap:int, key_max_gap:int, key_th:float, key_add_last_frame:bool, blend_rate:float, export_type:str, bg_src:str, bg_type:str, mask_blur_size:int, mask_mode:str):
     args = locals()
     info = ""
     info = dump_dict(info, args)
@@ -43,22 +46,40 @@ def ebsynth_utility_process(stage_index: int, project_dir:str, original_movie_pa
         dbg.print("original_movie_path not found")
         return process_end( dbg, info )
     
+    is_invert_mask = False
+    if mask_mode == "Invert":
+        is_invert_mask = True
+
     frame_path = os.path.join(project_dir , "video_frame")
     frame_mask_path = os.path.join(project_dir, "video_mask")
-    org_key_path = os.path.join(project_dir, "video_key")
-    img2img_key_path = os.path.join(project_dir, "img2img_key")
-    img2img_upscale_key_path = os.path.join(project_dir, "img2img_upscale_key")
 
-    if no_mask_mode:
+    if is_invert_mask:
+        inv_path = os.path.join(project_dir, "inv")
+        os.makedirs(inv_path, exist_ok=True)
+
+        org_key_path = os.path.join(inv_path, "video_key")
+        img2img_key_path = os.path.join(inv_path, "img2img_key")
+        img2img_upscale_key_path = os.path.join(inv_path, "img2img_upscale_key")
+    else:
+        org_key_path = os.path.join(project_dir, "video_key")
+        img2img_key_path = os.path.join(project_dir, "img2img_key")
+        img2img_upscale_key_path = os.path.join(project_dir, "img2img_upscale_key")
+
+    if mask_mode == "None":
         frame_mask_path = ""
+    
 
     project_args = [project_dir, original_movie_path, frame_path, frame_mask_path, org_key_path, img2img_key_path, img2img_upscale_key_path]
 
 
     if stage_index == 0:
-        ebsynth_utility_stage1(dbg, project_args)
+        ebsynth_utility_stage1(dbg, project_args, tb_use_fast_mode, tb_use_jit, is_invert_mask)
+        if is_invert_mask:
+            inv_mask_path = os.path.join(inv_path, "inv_video_mask")
+            ebsynth_utility_stage1_invert(dbg, frame_mask_path, inv_mask_path)
+
     elif stage_index == 1:
-        ebsynth_utility_stage2(dbg, project_args, key_min_gap, key_max_gap, key_th, key_add_last_frame)
+        ebsynth_utility_stage2(dbg, project_args, key_min_gap, key_max_gap, key_th, key_add_last_frame, is_invert_mask)
     elif stage_index == 2:
 
         sample_image = glob.glob( os.path.join(frame_path , "*.png" ) )[0]
@@ -78,18 +99,24 @@ def ebsynth_utility_process(stage_index: int, project_dir:str, original_movie_pa
         dbg.print("1. Go to img2img tab")
         dbg.print("2. Select [ebsynth utility] in the script combo box")
         dbg.print("3. Fill in the \"Project directory\" field with [" + project_dir + "]" )
-        dbg.print("4. I recommend to fill in the \"Width\" field with [" + str(img_width) + "]" )
-        dbg.print("5. I recommend to fill in the \"Height\" field with [" + str(img_height) + "]" )
-        dbg.print("6. I recommend to fill in the \"Denoising strength\" field with lower than 0.35" )
+        dbg.print("4. Select in the \"Mask Mode(Override img2img Mask mode)\" field with [" + ("Invert" if is_invert_mask else "Normal") + "]" )
+        dbg.print("5. I recommend to fill in the \"Width\" field with [" + str(img_width) + "]" )
+        dbg.print("6. I recommend to fill in the \"Height\" field with [" + str(img_height) + "]" )
+        dbg.print("7. I recommend to fill in the \"Denoising strength\" field with lower than 0.35" )
         dbg.print("   (It's okay to put a Large value in \"Face Denoising Strength\")")
-        dbg.print("7. Fill in the remaining configuration fields of img2img. No image and mask settings are required.")
-        dbg.print("8. Generate")
+        dbg.print("8. Fill in the remaining configuration fields of img2img. No image and mask settings are required.")
+        dbg.print("9. Drop any image onto the img2img main screen. This is necessary to avoid errors, but does not affect the results of img2img.")
+        dbg.print("10. Generate")
         dbg.print("(Images are output to [" + img2img_key_path + "])")
         dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
         return process_end( dbg, "" )
     elif stage_index == 3:
         sample_image = glob.glob( os.path.join(frame_path , "*.png" ) )[0]
         img_height, img_width, _ = cv2.imread(sample_image).shape
+
+        if is_invert_mask:
+            project_dir = inv_path
+
         dbg.print("stage 4")
         dbg.print("")
         dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
@@ -98,32 +125,44 @@ def ebsynth_utility_process(stage_index: int, project_dir:str, original_movie_pa
         dbg.print("  Saving images/grids ->")
         dbg.print("    Use original name for output filename during batch process in extras tab")
         dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
-        dbg.print("1. Go to Extras tab")
-        dbg.print("2. Go to Batch from Directory tab")
-        dbg.print("3. Fill in the \"Input directory\" field with [" + img2img_key_path + "]" )
-        dbg.print("4. Fill in the \"Output directory\" field with [" + img2img_upscale_key_path + "]" )
-        dbg.print("5. Go to Scale to tab")
-        dbg.print("6. Fill in the \"Width\" field with [" + str(img_width) + "]" )
-        dbg.print("7. Fill in the \"Height\" field with [" + str(img_height) + "]" )
-        dbg.print("8. Fill in the remaining configuration fields of Upscaler.")
-        dbg.print("9. Generate")
+        dbg.print("1. If \"img2img_upscale_key\" directory already exists in the %s, delete it manually before executing."%(project_dir))
+        dbg.print("2. Go to Extras tab")
+        dbg.print("3. Go to Batch from Directory tab")
+        dbg.print("4. Fill in the \"Input directory\" field with [" + img2img_key_path + "]" )
+        dbg.print("5. Fill in the \"Output directory\" field with [" + img2img_upscale_key_path + "]" )
+        dbg.print("6. Go to Scale to tab")
+        dbg.print("7. Fill in the \"Width\" field with [" + str(img_width) + "]" )
+        dbg.print("8. Fill in the \"Height\" field with [" + str(img_height) + "]" )
+        dbg.print("9. Fill in the remaining configuration fields of Upscaler.")
+        dbg.print("10. Generate")
         dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
         return process_end( dbg, "" )
     elif stage_index == 4:
-        ebsynth_utility_stage5(dbg, project_args)
+        ebsynth_utility_stage5(dbg, project_args, is_invert_mask)
     elif stage_index == 5:
+
+        if is_invert_mask:
+            project_dir = inv_path
+
         dbg.print("stage 6")
         dbg.print("")
         dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
         dbg.print("Running ebsynth.(on your self)")
-        dbg.print("Open the generated .ebs under project directory and press [Run All] button.")
-        dbg.print("If ""out-*"" directory already exists in the Project directory, delete it manually before executing.")
+        dbg.print("Open the generated .ebs under %s and press [Run All] button."%(project_dir))
+        dbg.print("If ""out-*"" directory already exists in the %s, delete it manually before executing."%(project_dir))
         dbg.print("If multiple .ebs files are generated, run them all.")
         dbg.print("(I recommend associating the .ebs file with EbSynth.exe.)")
         dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
         return process_end( dbg, "" )
     elif stage_index == 6:
-        ebsynth_utility_stage7(dbg, project_args, blend_rate, export_type)
+        ebsynth_utility_stage7(dbg, project_args, blend_rate, export_type, is_invert_mask)
+    elif stage_index == 7:
+        if mask_mode != "Normal":
+            dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
+            dbg.print("Please reset [configuration]->[etc]->[Mask Mode] to Normal.")
+            dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
+            return process_end( dbg, "" )
+        ebsynth_utility_stage8(dbg, project_args, bg_src, bg_type, mask_blur_size, export_type)
     else:
         pass
 

scripts/custom_script.py

@@ -4,7 +4,7 @@ import os
 import torch
 import random
 
-from modules.processing import process_images
+from modules.processing import process_images,Processed
 from modules.paths import models_path
 from modules.textual_inversion import autocrop
 import cv2
@@ -80,9 +80,10 @@ class Script(scripts.Script):
 
     def ui(self, is_img2img):
         project_dir = gr.Textbox(label='Project directory', lines=1)
+        mask_mode = gr.Dropdown(choices=["Normal","Invert","None","Don't Override"], value="Normal" ,label="Mask Mode(Override img2img Mask mode)")
 
-        img2img_repeat_count = gr.Slider(minimum=1, maximum=10, step=1, value=1, label="Img2Img Repeat Count")
-        inc_seed = gr.Slider(minimum=0, maximum=9999999, step=1, value=0, label="Add N to seed when repeating")
+        img2img_repeat_count = gr.Slider(minimum=1, maximum=10, step=1, value=1, label="Img2Img Repeat Count(Loop Back)")
+        inc_seed = gr.Slider(minimum=0, maximum=9999999, step=1, value=1, label="Add N to seed when repeating ")
 
         with gr.Group():
             is_facecrop = gr.Checkbox(False, label="use Face Crop img2img")
@@ -93,7 +94,7 @@ class Script(scripts.Script):
                     </p>")
             max_crop_size = gr.Slider(minimum=0, maximum=2048, step=1, value=1024, label="Max Crop Size")
             face_denoising_strength = gr.Slider(minimum=0.00, maximum=1.00, step=0.01, value=0.5, label="Face Denoising Strength")
-            face_area_magnification = gr.Slider(minimum=1.00, maximum=10.00, step=0.01, value=1.5, label="Face Area Magnification")
+            face_area_magnification = gr.Slider(minimum=1.00, maximum=10.00, step=0.01, value=1.5, label="Face Area Magnification ")
             
             with gr.Column():
                 enable_face_prompt = gr.Checkbox(False, label="Enable Face Prompt")
@@ -102,7 +103,7 @@ class Script(scripts.Script):
                     value = "face close up,"
                 )
 
-        return [project_dir, img2img_repeat_count, inc_seed, is_facecrop, face_detection_method, max_crop_size, face_denoising_strength, face_area_magnification, enable_face_prompt, face_prompt]
+        return [project_dir, mask_mode, img2img_repeat_count, inc_seed, is_facecrop, face_detection_method, max_crop_size, face_denoising_strength, face_area_magnification, enable_face_prompt, face_prompt]
 
 
     def detect_face(self, img_array):
@@ -282,7 +283,8 @@ class Script(scripts.Script):
 # to be used in processing. The return value should be a Processed object, which is
 # what is returned by the process_images method.
 
-    def run(self, p, project_dir, img2img_repeat_count, inc_seed, is_facecrop, face_detection_method, max_crop_size, face_denoising_strength, face_area_magnification, enable_face_prompt, face_prompt):
+    def run(self, p, project_dir, mask_mode, img2img_repeat_count, inc_seed, is_facecrop, face_detection_method, max_crop_size, face_denoising_strength, face_area_magnification, enable_face_prompt, face_prompt):
+        args = locals()
 
         def detect_face(img, mask, face_detection_method, max_crop_size):
             img_array = np.array(img)
@@ -325,15 +327,39 @@ class Script(scripts.Script):
 
         if not os.path.isdir(project_dir):
             print("project_dir not found")
-            return process_images(p)
-
+            return Processed()
+        
         if p.seed == -1:
             p.seed = int(random.randrange(4294967294))
-        
-        frame_mask_path = os.path.join(project_dir, "video_mask")
-        org_key_path = os.path.join(project_dir, "video_key")
-        img2img_key_path = os.path.join(project_dir, "img2img_key")
 
+        if mask_mode == "Normal":
+            p.inpainting_mask_invert = 0
+        elif mask_mode == "Invert":
+            p.inpainting_mask_invert = 1
+
+        is_invert_mask = False
+        if mask_mode == "Invert":
+            is_invert_mask = True
+
+            inv_path = os.path.join(project_dir, "inv")
+            if not os.path.isdir(inv_path):
+                print("project_dir/inv not found")
+                return Processed()
+            
+            org_key_path = os.path.join(inv_path, "video_key")
+            img2img_key_path = os.path.join(inv_path, "img2img_key")
+        else:
+            org_key_path = os.path.join(project_dir, "video_key")
+            img2img_key_path = os.path.join(project_dir, "img2img_key")
+
+        frame_mask_path = os.path.join(project_dir, "video_mask")
+
+        if not os.path.isdir(org_key_path):
+            print(org_key_path + " not found")
+            print("Generate key frames first." if is_invert_mask == False else \
+                    "Generate key frames first.(with [Ebsynth Utility] Tab -> [configuration] -> [etc]-> [Mask Mode] = Invert setting)")
+            return Processed()
+        
         remove_pngs_in_dir(img2img_key_path)
         os.makedirs(img2img_key_path, exist_ok=True)
 
@@ -343,11 +369,13 @@ class Script(scripts.Script):
             image = Image.open(img)
 
             img_basename = os.path.basename(img)
-            mask_path = os.path.join( frame_mask_path , img_basename )
             
             mask = None
-            if os.path.isfile( mask_path ):
-                mask = Image.open(mask_path)
+
+            if mask_mode != "None":
+                mask_path = os.path.join( frame_mask_path , img_basename )
+                if os.path.isfile( mask_path ):
+                    mask = Image.open(mask_path)
             
             _p = copy.copy(p)
             
@@ -379,7 +407,6 @@ class Script(scripts.Script):
                 else:
                     proc = process_images(_p)
                     print(proc.seed)
-
                 
                 repeat_count -= 1
 
@@ -394,4 +421,9 @@ class Script(scripts.Script):
 
             proc.images[0].save( os.path.join( img2img_key_path , img_basename ) )
 
+        with open( os.path.join( project_dir if is_invert_mask == False else inv_path,"param.txt" ), "w") as f:
+            f.write(proc.info)
+        with open( os.path.join( project_dir if is_invert_mask == False else inv_path ,"args.txt" ), "w") as f:
+            f.write(str(args))
+
         return proc

scripts/ui.py

@@ -17,18 +17,32 @@ def on_ui_tabs():
                             project_dir = gr.Textbox(label='Project directory', lines=1)
                             original_movie_path = gr.Textbox(label='Original Movie Path', lines=1)
                         with gr.TabItem('configuration', id='ebs_configuration'):
+                            with gr.Accordion(label="stage 1"):
+                                # https://pypi.org/project/transparent-background/
+                                gr.HTML(value="<p style='margin-bottom: 0.7em'>\
+                                        configuration for \
+                                        <font color=\"blue\"><a href=\"https://pypi.org/project/transparent-background\">[transparent-background]</a></font>\
+                                        </p>")
+                                tb_use_fast_mode = gr.Checkbox(label="Use Fast Mode(It will be faster, but the quality of the mask will be lower.)", value=False)
+                                tb_use_jit = gr.Checkbox(label="Use Jit", value=False)
+
                             with gr.Accordion(label="stage 2"):
                                 key_min_gap = gr.Slider(minimum=0, maximum=500, step=1, label='Minimum keyframe gap', value=10)
                                 key_max_gap = gr.Slider(minimum=0, maximum=1000, step=1, label='Maximum keyframe gap', value=300)
-                                key_th = gr.Slider(minimum=5.0, maximum=100.0, step=0.1, label='Threshold of delta frame edge', value=27.0)
+                                key_th = gr.Slider(minimum=0.0, maximum=100.0, step=0.1, label='Threshold of delta frame edge', value=8.5)
                                 key_add_last_frame = gr.Checkbox(label="Add last frame to keyframes", value=True)
 
                             with gr.Accordion(label="stage 7"):
                                 blend_rate = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='Crossfade blend rate', value=1.0)
                                 export_type = gr.Dropdown(choices=["mp4","webm","gif","rawvideo"], value="mp4" ,label="Export type")
 
+                            with gr.Accordion(label="stage 8"):
+                                bg_src = gr.Textbox(label='Background source(mp4 or directory containing images)', lines=1)
+                                bg_type = gr.Dropdown(choices=["Fit video length","Loop"], value="Fit video length" ,label="Background type")
+                                mask_blur_size = gr.Slider(minimum=0, maximum=150, step=1, label='Mask Blur Kernel Size', value=5)
+
                             with gr.Accordion(label="etc"):
-                                no_mask_mode = gr.Checkbox(label="No Mask Mode", value=False)
+                                mask_mode = gr.Dropdown(choices=["Normal","Invert","None"], value="Normal" ,label="Mask Mode")
 
                     with gr.Column(variant='panel'):
                         with gr.Column(scale=1):
@@ -36,7 +50,7 @@ def on_ui_tabs():
                                 debug_info = gr.HTML(elem_id="ebs_info_area", value=".")
 
                             with gr.Column(scale=2):
-                                stage_index = gr.Radio(label='Process Stage', choices=["stage 1","stage 2","stage 3","stage 4","stage 5","stage 6","stage 7"], value="stage 1", type="index")
+                                stage_index = gr.Radio(label='Process Stage', choices=["stage 1","stage 2","stage 3","stage 4","stage 5","stage 6","stage 7","stage 8"], value="stage 1", type="index")
                                 gr.HTML(value="<p style='margin-bottom: 0.7em'>\
                                                 The process of creating a video can be divided into the following stages.<br>\
                                                 (Stage 3, 4, and 6 only show a guide and do nothing actual processing.)<br><br>\
@@ -59,7 +73,13 @@ def on_ui_tabs():
                                                     If multiple .ebs files are generated, run them all.<br><br>\
                                                 <b>stage 7</b> <br>\
                                                     Concatenate each frame while crossfading.<br>\
-                                                    Composite audio files extracted from the original video onto the concatenated video.<br>\
+                                                    Composite audio files extracted from the original video onto the concatenated video.<br><br>\
+                                                <b>stage 8</b> <br>\
+                                                    This is an extra stage.<br>\
+                                                    You can put any image or images or video you like in the background.<br>\
+                                                    You can specify in this field -> [Ebsynth Utility]->[configuration]->[stage 8]->[Background source]<br>\
+                                                    If you have already created a background video in Invert Mask Mode([Ebsynth Utility]->[configuration]->[etc]->[Mask Mode]),<br>\
+                                                    You can specify \"path_to_project_dir/inv/crossfade_tmp\".<br>\
                                                 </p>")
                             
                             with gr.Row():
@@ -77,6 +97,9 @@ def on_ui_tabs():
                     project_dir,
                     original_movie_path,
 
+                    tb_use_fast_mode,
+                    tb_use_jit,
+
                     key_min_gap,
                     key_max_gap,
                     key_th,
@@ -85,7 +108,11 @@ def on_ui_tabs():
                     blend_rate,
                     export_type,
 
-                    no_mask_mode,
+                    bg_src,
+                    bg_type,
+                    mask_blur_size,
+
+                    mask_mode,
 
                 ],
                 outputs=[

stage1.py

@@ -12,12 +12,17 @@ def remove_pngs_in_dir(path):
     for png in pngs:
         os.remove(png)
 
-def ebsynth_utility_stage1(dbg, project_args):
+def ebsynth_utility_stage1(dbg, project_args, tb_use_fast_mode, tb_use_jit, is_invert_mask):
     dbg.print("stage1")
     dbg.print("")
 
     _, original_movie_path, frame_path, frame_mask_path, _, _, _ = project_args
 
+    if is_invert_mask:
+        if os.path.isdir( frame_path ) and os.path.isdir( frame_mask_path ):
+            dbg.print("Skip as it appears that the frame and normal masks have already been generated.")
+            return
+
     remove_pngs_in_dir(frame_path)
 
     if frame_mask_path:
@@ -35,7 +40,9 @@ def ebsynth_utility_stage1(dbg, project_args):
     dbg.print("frame extracted")
 
     if frame_mask_path:
-        subprocess.call("venv\\Scripts\\transparent-background --source " + frame_path + " --dest " + frame_mask_path + " --type map --fast", shell=True)
+        fast_str = " --fast" if tb_use_fast_mode else ""
+        jit_str = " --jit" if tb_use_jit else ""
+        subprocess.call("venv\\Scripts\\transparent-background --source " + frame_path + " --dest " + frame_mask_path + " --type map" + fast_str + jit_str, shell=True)
 
         mask_imgs = glob.glob( os.path.join(frame_mask_path, "*.png") )
         
@@ -57,3 +64,26 @@ def ebsynth_utility_stage1(dbg, project_args):
     dbg.print("completed.")
 
 
+def ebsynth_utility_stage1_invert(dbg, frame_mask_path, inv_mask_path):
+    dbg.print("stage 1 create_invert_mask")
+    dbg.print("")
+
+    if not os.path.isdir( frame_mask_path ):
+        dbg.print( frame_mask_path + " not found")
+        dbg.print("Normal masks must be generated previously.")
+        dbg.print("Do stage 1 with [Ebsynth Utility] Tab -> [configuration] -> [etc]-> [Mask Mode] = Normal setting first")
+        return
+
+    os.makedirs(inv_mask_path, exist_ok=True)
+
+    mask_imgs = glob.glob( os.path.join(frame_mask_path, "*.png") )
+    
+    for m in mask_imgs:
+        img = cv2.imread(m)
+        inv = cv2.bitwise_not(img)
+
+        base_name = os.path.basename(m)
+        cv2.imwrite(os.path.join(inv_mask_path,base_name), inv)
+
+    dbg.print("")
+    dbg.print("completed.")

stage2.py

@@ -54,18 +54,32 @@ def _detect_edges(lum: np.ndarray) -> np.ndarray:
 
 #---------------------------------
 
-def detect_edges(img_path):
-    hue, sat, lum = cv2.split(cv2.cvtColor( cv2.imread(img_path) , cv2.COLOR_BGR2HSV))
+def detect_edges(img_path, mask_path, is_invert_mask):
+    im = cv2.imread(img_path)
+    if mask_path:
+        mask = cv2.imread(mask_path)[:,:,0]
+        mask = mask[:, :, np.newaxis]
+        im = im * ( (mask == 0) if is_invert_mask else (mask > 0) )
+#        im = im * (mask/255)
+#        im = im.astype(np.uint8)
+#        cv2.imwrite( os.path.join( os.path.dirname(mask_path) , "tmp.png" ) , im)
+
+    hue, sat, lum = cv2.split(cv2.cvtColor( im , cv2.COLOR_BGR2HSV))
     return _detect_edges(lum)
 
-def analyze_key_frames(png_dir, th, min_gap, max_gap, add_last_frame):
+def get_mask_path_of_img(img_path, mask_dir):
+    img_basename = os.path.basename(img_path)
+    mask_path = os.path.join( mask_dir , img_basename )
+    return mask_path if os.path.isfile( mask_path ) else None
+
+def analyze_key_frames(png_dir, mask_dir, th, min_gap, max_gap, add_last_frame, is_invert_mask):
     keys = []
     
     frames = sorted(glob.glob( os.path.join(png_dir, "[0-9]*.png") ))
     
     key_frame = frames[0]
     keys.append( int(os.path.splitext(os.path.basename(key_frame))[0]) )
-    key_edges = detect_edges( key_frame )
+    key_edges = detect_edges( key_frame, get_mask_path_of_img( key_frame, mask_dir ), is_invert_mask )
     gap = 0
     
     for frame in frames:
@@ -73,7 +87,7 @@ def analyze_key_frames(png_dir, th, min_gap, max_gap, add_last_frame):
         if gap < min_gap:
             continue
         
-        edges = detect_edges( frame )
+        edges = detect_edges( frame, get_mask_path_of_img( frame, mask_dir ), is_invert_mask )
         
         delta = mean_pixel_distance( edges, key_edges )
         
@@ -102,11 +116,11 @@ def remove_pngs_in_dir(path):
     for png in pngs:
         os.remove(png)
 
-def ebsynth_utility_stage2(dbg, project_args, key_min_gap, key_max_gap, key_th, key_add_last_frame):
+def ebsynth_utility_stage2(dbg, project_args, key_min_gap, key_max_gap, key_th, key_add_last_frame, is_invert_mask):
     dbg.print("stage2")
     dbg.print("")
 
-    _, original_movie_path, frame_path, _, org_key_path, _, _ = project_args
+    _, original_movie_path, frame_path, frame_mask_path, org_key_path, _, _ = project_args
 
     remove_pngs_in_dir(org_key_path)
     os.makedirs(org_key_path, exist_ok=True)
@@ -136,7 +150,7 @@ def ebsynth_utility_stage2(dbg, project_args, key_min_gap, key_max_gap, key_th,
     dbg.print("key_max_gap: {}".format(key_max_gap))
     dbg.print("key_th: {}".format(key_th))
 
-    keys = analyze_key_frames(frame_path, key_th, key_min_gap, key_max_gap, key_add_last_frame)
+    keys = analyze_key_frames(frame_path, frame_mask_path, key_th, key_min_gap, key_max_gap, key_add_last_frame, is_invert_mask)
 
     dbg.print("keys : " + str(keys))
     
@@ -147,6 +161,10 @@ def ebsynth_utility_stage2(dbg, project_args, key_min_gap, key_max_gap, key_th,
 
     dbg.print("")
     dbg.print("Keyframes are output to [" + org_key_path + "]")
+    dbg.print("")
+    dbg.print("[Ebsynth Utility]->[configuration]->[stage 2]->[Threshold of delta frame edge]")
+    dbg.print("The smaller this value, the narrower the keyframe spacing, and if set to 0, the keyframes will be equally spaced at the value of [Minimum keyframe gap].")
+    dbg.print("")
     dbg.print("If you do not like the selection, you can modify it manually.")
     dbg.print("(Delete keyframe, or Add keyframe from ["+frame_path+"])")
 

stage5.py

@@ -167,7 +167,7 @@ def rename_keys(key_dir):
             dirname = os.path.dirname(img)
             os.rename(img, os.path.join(dirname, f))
 
-def ebsynth_utility_stage5(dbg, project_args):
+def ebsynth_utility_stage5(dbg, project_args, is_invert_mask):
     dbg.print("stage5")
     dbg.print("")
     
@@ -212,13 +212,13 @@ def ebsynth_utility_stage5(dbg, project_args):
         prev_key = key
     
     project = {
-        "proj_dir" : project_dir,
+        "proj_dir" : project_dir if is_invert_mask == False else os.path.join(project_dir, "inv"),
         "file_name" : "/[" + "#" *  number_of_digits + "].png",
         "number_of_digits" : number_of_digits,
         
         "key_dir" : "img2img_upscale_key",
-        "video_dir" : "video_frame",
-        "mask_dir" : "video_mask",
+        "video_dir" : "video_frame" if is_invert_mask == False else "../video_frame",
+        "mask_dir" : "video_mask" if is_invert_mask == False else "inv_video_mask",
         "key_weight" : 1.0,
         "video_weight" : 4.0,
         "mask_weight" : 1.0,
@@ -234,6 +234,8 @@ def ebsynth_utility_stage5(dbg, project_args):
         project["mask_dir"] = ""
 
     proj_base_name = time.strftime("%Y%m%d-%H%M%S")
+    if is_invert_mask:
+        proj_base_name = "inv_" + proj_base_name
 
     tmp=[]
     proj_index = 0

stage7.py

@@ -91,8 +91,21 @@ def get_ext(export_type):
         return "." + export_type
     else:
         return ".avi"
+
+def trying_to_add_audio(original_movie_path, no_snd_movie_path, output_path, tmp_dir ):
+    if os.path.isfile(original_movie_path):
+        sound_path = os.path.join(tmp_dir , 'sound.mp4')
+        subprocess.call("ffmpeg -i " + original_movie_path + " -vn -acodec copy " + sound_path, shell=True)
+        
+        if os.path.isfile(sound_path):
+            # ffmpeg -i video.mp4 -i audio.wav -c:v copy -c:a aac -map 0:v:0 -map 1:a:0 output.mp4
+
+            subprocess.call("ffmpeg -i " + no_snd_movie_path + " -i " + sound_path + " -c:v copy -c:a aac -map 0:v:0 -map 1:a:0 " + output_path, shell=True)
+            return True
     
-def ebsynth_utility_stage7(dbg, project_args, blend_rate,export_type):
+    return False
+
+def ebsynth_utility_stage7(dbg, project_args, blend_rate,export_type,is_invert_mask):
     dbg.print("stage7")
     dbg.print("")
 
@@ -110,7 +123,11 @@ def ebsynth_utility_stage7(dbg, project_args, blend_rate,export_type):
     dbg.print("export_type: {}".format(export_type))
     dbg.print("fps: {}".format(fps))
     
+    if is_invert_mask:
+        project_dir = os.path.join( project_dir , "inv")
+
     tmp_dir = os.path.join( project_dir , "crossfade_tmp")
+
     
     if os.path.isdir(tmp_dir):
         shutil.rmtree(tmp_dir)
@@ -191,9 +208,11 @@ def ebsynth_utility_stage7(dbg, project_args, blend_rate,export_type):
         filename = str(i).zfill(number_of_digits) + ".png"
         shutil.copy( os.path.join(out_dirs[cur_clip]['path'] , filename) , os.path.join(tmp_dir , filename) )
     
-    
     ### create movie
     movie_base_name = time.strftime("%Y%m%d-%H%M%S")
+    if is_invert_mask:
+        movie_base_name = "inv_" + movie_base_name
+    
     nosnd_path = os.path.join(project_dir , movie_base_name + get_ext(export_type))
     
     start = out_dirs[0]['startframe']
@@ -204,18 +223,11 @@ def ebsynth_utility_stage7(dbg, project_args, blend_rate,export_type):
     dbg.print("exported : " + nosnd_path)
     
     if export_type == "mp4":
-        if os.path.isfile(original_movie_path):
-            sound_path = os.path.join(tmp_dir , 'sound.mp4')
-            subprocess.call("ffmpeg -i " + original_movie_path + " -vn -acodec copy " + sound_path, shell=True)
-            
-            if os.path.isfile(sound_path):
-                # ffmpeg -i video.mp4 -i audio.wav -c:v copy -c:a aac -map 0:v:0 -map 1:a:0 output.mp4
-                
-                with_snd_path = os.path.join(project_dir , movie_base_name + '_with_snd.mp4')
-                
-                subprocess.call("ffmpeg -i " + nosnd_path + " -i " + sound_path + " -c:v copy -c:a aac -map 0:v:0 -map 1:a:0 " + with_snd_path, shell=True)
 
-                dbg.print("exported : " + with_snd_path)
+        with_snd_path = os.path.join(project_dir , movie_base_name + '_with_snd.mp4')
+
+        if trying_to_add_audio(original_movie_path, nosnd_path, with_snd_path, tmp_dir):
+            dbg.print("exported : " + with_snd_path)
     
     dbg.print("")
     dbg.print("completed.")

stage8.py

@@ -0,0 +1,141 @@
+import os
+import re
+import subprocess
+import glob
+import shutil
+import time
+import cv2
+import numpy as np
+import itertools
+from extensions.ebsynth_utility.stage7 import create_movie_from_frames, get_ext, trying_to_add_audio
+
+def clamp(n, smallest, largest):
+    return sorted([smallest, n, largest])[1]
+
+def resize_img(img, w, h):
+    if img.shape[0] + img.shape[1] < h + w:
+        interpolation = interpolation=cv2.INTER_CUBIC
+    else:
+        interpolation = interpolation=cv2.INTER_AREA
+
+    return cv2.resize(img, (w, h), interpolation=interpolation)
+
+def merge_bg_src(base_frame_dir, bg_dir, frame_mask_path, tmp_dir, bg_type, mask_blur_size):
+
+    base_frames = sorted(glob.glob( os.path.join(base_frame_dir, "[0-9]*.png"), recursive=False) )
+
+    bg_frames = sorted(glob.glob( os.path.join(bg_dir, "*.png"), recursive=False) )
+    
+    def bg_frame(total_frames):
+        bg_len = len(bg_frames)
+
+        if bg_type == "Loop":
+            itr = itertools.cycle(bg_frames)
+            while True:
+                yield next(itr)
+        else:
+            for i in range(total_frames):
+                yield bg_frames[ int(bg_len * (i/total_frames))]
+    
+    bg_itr = bg_frame(len(base_frames))
+
+    for base_frame in base_frames:
+        im = cv2.imread(base_frame)
+        bg = cv2.imread( next(bg_itr) )
+        bg = resize_img(bg, im.shape[1], im.shape[0] )
+
+        basename = os.path.basename(base_frame)
+        mask_path = os.path.join(frame_mask_path, basename)
+        mask = cv2.imread(mask_path)[:,:,0]
+        if mask_blur_size > 0:
+            mask_blur_size = mask_blur_size//2 * 2 + 1
+            mask = cv2.GaussianBlur(mask, (mask_blur_size, mask_blur_size), 0)
+        mask = mask[:, :, np.newaxis]
+
+        im = im * (mask/255) + bg * (1- mask/255)
+        im = im.astype(np.uint8)
+        cv2.imwrite( os.path.join( tmp_dir , basename ) , im)
+
+def extract_frames(movie_path , output_dir, fps):
+    png_path = os.path.join(output_dir , "%05d.png")
+    # ffmpeg.exe -ss 00:00:00  -y -i %1 -qscale 0 -f image2 -c:v png "%05d.png"
+    subprocess.call("ffmpeg.exe -ss 00:00:00  -y -i " + movie_path + " -vf fps=" + str( round(fps, 2)) + " -qscale 0 -f image2 -c:v png " + png_path, shell=True)
+
+def ebsynth_utility_stage8(dbg, project_args, bg_src, bg_type, mask_blur_size, export_type):
+    dbg.print("stage8")
+    dbg.print("")
+
+    if not bg_src:
+        dbg.print("Fill [configuration] -> [stage 8] -> [Background source]")
+        return
+
+    project_dir, original_movie_path, _, frame_mask_path, _, _, _ = project_args
+
+    fps = 30
+    clip = cv2.VideoCapture(original_movie_path)
+    if clip:
+        fps = clip.get(cv2.CAP_PROP_FPS)
+        clip.release()
+
+    dbg.print("bg_src: {}".format(bg_src))
+    dbg.print("bg_type: {}".format(bg_type))
+    dbg.print("mask_blur_size: {}".format(mask_blur_size))
+    dbg.print("export_type: {}".format(export_type))
+    dbg.print("fps: {}".format(fps))
+    
+    base_frame_dir = os.path.join( project_dir , "crossfade_tmp")
+    
+    if not os.path.isdir(base_frame_dir):
+        dbg.print(base_frame_dir + " base frame not found")
+        return
+    
+    tmp_dir = os.path.join( project_dir , "bg_merge_tmp")
+    if os.path.isdir(tmp_dir):
+        shutil.rmtree(tmp_dir)
+    os.mkdir(tmp_dir)
+
+    ### create frame imgs
+    if os.path.isfile(bg_src):
+        bg_ext = os.path.splitext(os.path.basename(bg_src))[1]
+        if bg_ext == ".mp4":
+            bg_tmp_dir = os.path.join( project_dir , "bg_extract_tmp")
+            if os.path.isdir(bg_tmp_dir):
+                shutil.rmtree(bg_tmp_dir)
+            os.mkdir(bg_tmp_dir)
+
+            extract_frames(bg_src, bg_tmp_dir, fps)
+
+            bg_src = bg_tmp_dir
+        else:
+            dbg.print(bg_src + " must be mp4 or directory")
+            return
+    elif not os.path.isdir(bg_src):
+        dbg.print(bg_src + " must be mp4 or directory")
+        return
+
+    merge_bg_src(base_frame_dir, bg_src, frame_mask_path, tmp_dir, bg_type, mask_blur_size)
+    
+    ### create movie
+    movie_base_name = time.strftime("%Y%m%d-%H%M%S")
+    movie_base_name = "merge_" + movie_base_name
+    
+    nosnd_path = os.path.join(project_dir , movie_base_name + get_ext(export_type))
+
+    merged_frames = sorted(glob.glob( os.path.join(tmp_dir, "[0-9]*.png"), recursive=False) )
+    start = int(os.path.splitext(os.path.basename(merged_frames[0]))[0])
+    end = int(os.path.splitext(os.path.basename(merged_frames[-1]))[0])
+
+    create_movie_from_frames(tmp_dir,start,end,5,fps,nosnd_path,export_type)
+
+    dbg.print("exported : " + nosnd_path)
+    
+    if export_type == "mp4":
+
+        with_snd_path = os.path.join(project_dir , movie_base_name + '_with_snd.mp4')
+
+        if trying_to_add_audio(original_movie_path, nosnd_path, with_snd_path, tmp_dir):
+            dbg.print("exported : " + with_snd_path)
+    
+    dbg.print("")
+    dbg.print("completed.")
+