diff --git a/README.md b/README.md
index 97eaa9c..1e03da4 100644
--- a/README.md
+++ b/README.md
@@ -5,12 +5,19 @@
#### This extension allows you to output edited videos using ebsynth.(AE is not required)
## Example
+- The following sample is raw output of this extension.
#### sample 1
#### sample 2
+#### sample 3 blend background
+- person : masterpiece, best quality, masterpiece, 1girl, masterpiece, best quality,anime screencap, anime style
+- background : cyberpunk, factory, room ,anime screencap, anime style
+It is also possible to blend with your favorite videos.
+
+
## Installation
- Install [ffmpeg](https://ffmpeg.org/) for your operating system
(https://www.geeksforgeeks.org/how-to-install-ffmpeg-on-windows/)
@@ -37,7 +44,9 @@ All frames of the video and mask images for all frames are generated.
#### Stage 2
In the implementation of this extension, the keyframe interval is chosen to be shorter where there is a lot of motion and longer where there is little motion.
+If the animation breaks up, increase the keyframe, if it flickers, decrease the keyframe.
First, generate one time with the default settings and go straight ahead without worrying about the result.
+
#### Stage 3
Select one of the keyframes, throw it to img2img, and run [Interrogate DeepBooru].
@@ -45,19 +54,21 @@ Delete unwanted words such as blur from the displayed prompt.
Fill in the rest of the settings as you would normally do for image generation.
Here is the settings I used.
-- Sampling method : DDIM
+- Sampling method : Euler a
- Sampling Steps : 50
- Width : 960
- Height : 512
- CFG Scale : 20
-- Denoising strength : 0.35
+- Denoising strength : 0.2
Here is the settings for extension.
-- Img2Img Repeat Count : 1 ( or 3 with Euler a / lower Denoising strength)
+- Mask Mode(Override img2img Mask mode) : Normal
+- Img2Img Repeat Count : 5
+- Add N to seed when repeating : 1
- use Face Crop img2img : True
- Face Detection Method : YuNet
- Max Crop Size : 1024
-- Face Denoising Strength : 0.35
+- Face Denoising Strength : 0.25
- Face Area Magnification : 1.5 (The larger the number, the closer to the model's painting style, but the more likely it is to shift when merged with the body.)
- Enable Face Prompt : False
diff --git a/ebsynth_utility.py b/ebsynth_utility.py
index 66d2da5..fb15977 100644
--- a/ebsynth_utility.py
+++ b/ebsynth_utility.py
@@ -5,10 +5,12 @@ from modules.ui import plaintext_to_html
import cv2
import glob
-from extensions.ebsynth_utility.stage1 import ebsynth_utility_stage1
+from extensions.ebsynth_utility.stage1 import ebsynth_utility_stage1,ebsynth_utility_stage1_invert
from extensions.ebsynth_utility.stage2 import ebsynth_utility_stage2
from extensions.ebsynth_utility.stage5 import ebsynth_utility_stage5
from extensions.ebsynth_utility.stage7 import ebsynth_utility_stage7
+from extensions.ebsynth_utility.stage8 import ebsynth_utility_stage8
+
def x_ceiling(value, step):
return -(-value // step) * step
@@ -21,11 +23,12 @@ def dump_dict(string, d:dict):
class debug_string:
txt = ""
def print(self, comment):
+ print(comment)
self.txt += comment + '\n'
def to_string(self):
return self.txt
-def ebsynth_utility_process(stage_index: int, project_dir:str, original_movie_path:str, key_min_gap:int, key_max_gap:int, key_th:float, key_add_last_frame:bool, blend_rate:float, export_type:str, no_mask_mode:bool):
+def ebsynth_utility_process(stage_index: int, project_dir:str, original_movie_path:str, tb_use_fast_mode:bool, tb_use_jit:bool, key_min_gap:int, key_max_gap:int, key_th:float, key_add_last_frame:bool, blend_rate:float, export_type:str, bg_src:str, bg_type:str, mask_blur_size:int, mask_mode:str):
args = locals()
info = ""
info = dump_dict(info, args)
@@ -43,22 +46,40 @@ def ebsynth_utility_process(stage_index: int, project_dir:str, original_movie_pa
dbg.print("original_movie_path not found")
return process_end( dbg, info )
+ is_invert_mask = False
+ if mask_mode == "Invert":
+ is_invert_mask = True
+
frame_path = os.path.join(project_dir , "video_frame")
frame_mask_path = os.path.join(project_dir, "video_mask")
- org_key_path = os.path.join(project_dir, "video_key")
- img2img_key_path = os.path.join(project_dir, "img2img_key")
- img2img_upscale_key_path = os.path.join(project_dir, "img2img_upscale_key")
- if no_mask_mode:
+ if is_invert_mask:
+ inv_path = os.path.join(project_dir, "inv")
+ os.makedirs(inv_path, exist_ok=True)
+
+ org_key_path = os.path.join(inv_path, "video_key")
+ img2img_key_path = os.path.join(inv_path, "img2img_key")
+ img2img_upscale_key_path = os.path.join(inv_path, "img2img_upscale_key")
+ else:
+ org_key_path = os.path.join(project_dir, "video_key")
+ img2img_key_path = os.path.join(project_dir, "img2img_key")
+ img2img_upscale_key_path = os.path.join(project_dir, "img2img_upscale_key")
+
+ if mask_mode == "None":
frame_mask_path = ""
+
project_args = [project_dir, original_movie_path, frame_path, frame_mask_path, org_key_path, img2img_key_path, img2img_upscale_key_path]
if stage_index == 0:
- ebsynth_utility_stage1(dbg, project_args)
+ ebsynth_utility_stage1(dbg, project_args, tb_use_fast_mode, tb_use_jit, is_invert_mask)
+ if is_invert_mask:
+ inv_mask_path = os.path.join(inv_path, "inv_video_mask")
+ ebsynth_utility_stage1_invert(dbg, frame_mask_path, inv_mask_path)
+
elif stage_index == 1:
- ebsynth_utility_stage2(dbg, project_args, key_min_gap, key_max_gap, key_th, key_add_last_frame)
+ ebsynth_utility_stage2(dbg, project_args, key_min_gap, key_max_gap, key_th, key_add_last_frame, is_invert_mask)
elif stage_index == 2:
sample_image = glob.glob( os.path.join(frame_path , "*.png" ) )[0]
@@ -78,18 +99,24 @@ def ebsynth_utility_process(stage_index: int, project_dir:str, original_movie_pa
dbg.print("1. Go to img2img tab")
dbg.print("2. Select [ebsynth utility] in the script combo box")
dbg.print("3. Fill in the \"Project directory\" field with [" + project_dir + "]" )
- dbg.print("4. I recommend to fill in the \"Width\" field with [" + str(img_width) + "]" )
- dbg.print("5. I recommend to fill in the \"Height\" field with [" + str(img_height) + "]" )
- dbg.print("6. I recommend to fill in the \"Denoising strength\" field with lower than 0.35" )
+ dbg.print("4. Select in the \"Mask Mode(Override img2img Mask mode)\" field with [" + ("Invert" if is_invert_mask else "Normal") + "]" )
+ dbg.print("5. I recommend to fill in the \"Width\" field with [" + str(img_width) + "]" )
+ dbg.print("6. I recommend to fill in the \"Height\" field with [" + str(img_height) + "]" )
+ dbg.print("7. I recommend to fill in the \"Denoising strength\" field with lower than 0.35" )
dbg.print(" (It's okay to put a Large value in \"Face Denoising Strength\")")
- dbg.print("7. Fill in the remaining configuration fields of img2img. No image and mask settings are required.")
- dbg.print("8. Generate")
+ dbg.print("8. Fill in the remaining configuration fields of img2img. No image and mask settings are required.")
+ dbg.print("9. Drop any image onto the img2img main screen. This is necessary to avoid errors, but does not affect the results of img2img.")
+ dbg.print("10. Generate")
dbg.print("(Images are output to [" + img2img_key_path + "])")
dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
return process_end( dbg, "" )
elif stage_index == 3:
sample_image = glob.glob( os.path.join(frame_path , "*.png" ) )[0]
img_height, img_width, _ = cv2.imread(sample_image).shape
+
+ if is_invert_mask:
+ project_dir = inv_path
+
dbg.print("stage 4")
dbg.print("")
dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
@@ -98,32 +125,44 @@ def ebsynth_utility_process(stage_index: int, project_dir:str, original_movie_pa
dbg.print(" Saving images/grids ->")
dbg.print(" Use original name for output filename during batch process in extras tab")
dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
- dbg.print("1. Go to Extras tab")
- dbg.print("2. Go to Batch from Directory tab")
- dbg.print("3. Fill in the \"Input directory\" field with [" + img2img_key_path + "]" )
- dbg.print("4. Fill in the \"Output directory\" field with [" + img2img_upscale_key_path + "]" )
- dbg.print("5. Go to Scale to tab")
- dbg.print("6. Fill in the \"Width\" field with [" + str(img_width) + "]" )
- dbg.print("7. Fill in the \"Height\" field with [" + str(img_height) + "]" )
- dbg.print("8. Fill in the remaining configuration fields of Upscaler.")
- dbg.print("9. Generate")
+ dbg.print("1. If \"img2img_upscale_key\" directory already exists in the %s, delete it manually before executing."%(project_dir))
+ dbg.print("2. Go to Extras tab")
+ dbg.print("3. Go to Batch from Directory tab")
+ dbg.print("4. Fill in the \"Input directory\" field with [" + img2img_key_path + "]" )
+ dbg.print("5. Fill in the \"Output directory\" field with [" + img2img_upscale_key_path + "]" )
+ dbg.print("6. Go to Scale to tab")
+ dbg.print("7. Fill in the \"Width\" field with [" + str(img_width) + "]" )
+ dbg.print("8. Fill in the \"Height\" field with [" + str(img_height) + "]" )
+ dbg.print("9. Fill in the remaining configuration fields of Upscaler.")
+ dbg.print("10. Generate")
dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
return process_end( dbg, "" )
elif stage_index == 4:
- ebsynth_utility_stage5(dbg, project_args)
+ ebsynth_utility_stage5(dbg, project_args, is_invert_mask)
elif stage_index == 5:
+
+ if is_invert_mask:
+ project_dir = inv_path
+
dbg.print("stage 6")
dbg.print("")
dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
dbg.print("Running ebsynth.(on your self)")
- dbg.print("Open the generated .ebs under project directory and press [Run All] button.")
- dbg.print("If ""out-*"" directory already exists in the Project directory, delete it manually before executing.")
+ dbg.print("Open the generated .ebs under %s and press [Run All] button."%(project_dir))
+ dbg.print("If ""out-*"" directory already exists in the %s, delete it manually before executing."%(project_dir))
dbg.print("If multiple .ebs files are generated, run them all.")
dbg.print("(I recommend associating the .ebs file with EbSynth.exe.)")
dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
return process_end( dbg, "" )
elif stage_index == 6:
- ebsynth_utility_stage7(dbg, project_args, blend_rate, export_type)
+ ebsynth_utility_stage7(dbg, project_args, blend_rate, export_type, is_invert_mask)
+ elif stage_index == 7:
+ if mask_mode != "Normal":
+ dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
+ dbg.print("Please reset [configuration]->[etc]->[Mask Mode] to Normal.")
+ dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
+ return process_end( dbg, "" )
+ ebsynth_utility_stage8(dbg, project_args, bg_src, bg_type, mask_blur_size, export_type)
else:
pass
diff --git a/imgs/sample6.mp4 b/imgs/sample6.mp4
new file mode 100644
index 0000000..02e0436
Binary files /dev/null and b/imgs/sample6.mp4 differ
diff --git a/scripts/custom_script.py b/scripts/custom_script.py
index 637637e..c3d6770 100644
--- a/scripts/custom_script.py
+++ b/scripts/custom_script.py
@@ -4,7 +4,7 @@ import os
import torch
import random
-from modules.processing import process_images
+from modules.processing import process_images,Processed
from modules.paths import models_path
from modules.textual_inversion import autocrop
import cv2
@@ -80,9 +80,10 @@ class Script(scripts.Script):
def ui(self, is_img2img):
project_dir = gr.Textbox(label='Project directory', lines=1)
+ mask_mode = gr.Dropdown(choices=["Normal","Invert","None","Don't Override"], value="Normal" ,label="Mask Mode(Override img2img Mask mode)")
- img2img_repeat_count = gr.Slider(minimum=1, maximum=10, step=1, value=1, label="Img2Img Repeat Count")
- inc_seed = gr.Slider(minimum=0, maximum=9999999, step=1, value=0, label="Add N to seed when repeating")
+ img2img_repeat_count = gr.Slider(minimum=1, maximum=10, step=1, value=1, label="Img2Img Repeat Count(Loop Back)")
+ inc_seed = gr.Slider(minimum=0, maximum=9999999, step=1, value=1, label="Add N to seed when repeating ")
with gr.Group():
is_facecrop = gr.Checkbox(False, label="use Face Crop img2img")
@@ -93,7 +94,7 @@ class Script(scripts.Script):
")
max_crop_size = gr.Slider(minimum=0, maximum=2048, step=1, value=1024, label="Max Crop Size")
face_denoising_strength = gr.Slider(minimum=0.00, maximum=1.00, step=0.01, value=0.5, label="Face Denoising Strength")
- face_area_magnification = gr.Slider(minimum=1.00, maximum=10.00, step=0.01, value=1.5, label="Face Area Magnification")
+ face_area_magnification = gr.Slider(minimum=1.00, maximum=10.00, step=0.01, value=1.5, label="Face Area Magnification ")
with gr.Column():
enable_face_prompt = gr.Checkbox(False, label="Enable Face Prompt")
@@ -102,7 +103,7 @@ class Script(scripts.Script):
value = "face close up,"
)
- return [project_dir, img2img_repeat_count, inc_seed, is_facecrop, face_detection_method, max_crop_size, face_denoising_strength, face_area_magnification, enable_face_prompt, face_prompt]
+ return [project_dir, mask_mode, img2img_repeat_count, inc_seed, is_facecrop, face_detection_method, max_crop_size, face_denoising_strength, face_area_magnification, enable_face_prompt, face_prompt]
def detect_face(self, img_array):
@@ -282,7 +283,8 @@ class Script(scripts.Script):
# to be used in processing. The return value should be a Processed object, which is
# what is returned by the process_images method.
- def run(self, p, project_dir, img2img_repeat_count, inc_seed, is_facecrop, face_detection_method, max_crop_size, face_denoising_strength, face_area_magnification, enable_face_prompt, face_prompt):
+ def run(self, p, project_dir, mask_mode, img2img_repeat_count, inc_seed, is_facecrop, face_detection_method, max_crop_size, face_denoising_strength, face_area_magnification, enable_face_prompt, face_prompt):
+ args = locals()
def detect_face(img, mask, face_detection_method, max_crop_size):
img_array = np.array(img)
@@ -325,15 +327,39 @@ class Script(scripts.Script):
if not os.path.isdir(project_dir):
print("project_dir not found")
- return process_images(p)
-
+ return Processed()
+
if p.seed == -1:
p.seed = int(random.randrange(4294967294))
-
- frame_mask_path = os.path.join(project_dir, "video_mask")
- org_key_path = os.path.join(project_dir, "video_key")
- img2img_key_path = os.path.join(project_dir, "img2img_key")
+ if mask_mode == "Normal":
+ p.inpainting_mask_invert = 0
+ elif mask_mode == "Invert":
+ p.inpainting_mask_invert = 1
+
+ is_invert_mask = False
+ if mask_mode == "Invert":
+ is_invert_mask = True
+
+ inv_path = os.path.join(project_dir, "inv")
+ if not os.path.isdir(inv_path):
+ print("project_dir/inv not found")
+ return Processed()
+
+ org_key_path = os.path.join(inv_path, "video_key")
+ img2img_key_path = os.path.join(inv_path, "img2img_key")
+ else:
+ org_key_path = os.path.join(project_dir, "video_key")
+ img2img_key_path = os.path.join(project_dir, "img2img_key")
+
+ frame_mask_path = os.path.join(project_dir, "video_mask")
+
+ if not os.path.isdir(org_key_path):
+ print(org_key_path + " not found")
+ print("Generate key frames first." if is_invert_mask == False else \
+ "Generate key frames first.(with [Ebsynth Utility] Tab -> [configuration] -> [etc]-> [Mask Mode] = Invert setting)")
+ return Processed()
+
remove_pngs_in_dir(img2img_key_path)
os.makedirs(img2img_key_path, exist_ok=True)
@@ -343,11 +369,13 @@ class Script(scripts.Script):
image = Image.open(img)
img_basename = os.path.basename(img)
- mask_path = os.path.join( frame_mask_path , img_basename )
mask = None
- if os.path.isfile( mask_path ):
- mask = Image.open(mask_path)
+
+ if mask_mode != "None":
+ mask_path = os.path.join( frame_mask_path , img_basename )
+ if os.path.isfile( mask_path ):
+ mask = Image.open(mask_path)
_p = copy.copy(p)
@@ -379,7 +407,6 @@ class Script(scripts.Script):
else:
proc = process_images(_p)
print(proc.seed)
-
repeat_count -= 1
@@ -394,4 +421,9 @@ class Script(scripts.Script):
proc.images[0].save( os.path.join( img2img_key_path , img_basename ) )
+ with open( os.path.join( project_dir if is_invert_mask == False else inv_path,"param.txt" ), "w") as f:
+ f.write(proc.info)
+ with open( os.path.join( project_dir if is_invert_mask == False else inv_path ,"args.txt" ), "w") as f:
+ f.write(str(args))
+
return proc
diff --git a/scripts/ui.py b/scripts/ui.py
index 335ba9a..278292e 100644
--- a/scripts/ui.py
+++ b/scripts/ui.py
@@ -17,18 +17,32 @@ def on_ui_tabs():
project_dir = gr.Textbox(label='Project directory', lines=1)
original_movie_path = gr.Textbox(label='Original Movie Path', lines=1)
with gr.TabItem('configuration', id='ebs_configuration'):
+ with gr.Accordion(label="stage 1"):
+ # https://pypi.org/project/transparent-background/
+ gr.HTML(value="\
+ configuration for \
+ [transparent-background]\
+
")
+ tb_use_fast_mode = gr.Checkbox(label="Use Fast Mode(It will be faster, but the quality of the mask will be lower.)", value=False)
+ tb_use_jit = gr.Checkbox(label="Use Jit", value=False)
+
with gr.Accordion(label="stage 2"):
key_min_gap = gr.Slider(minimum=0, maximum=500, step=1, label='Minimum keyframe gap', value=10)
key_max_gap = gr.Slider(minimum=0, maximum=1000, step=1, label='Maximum keyframe gap', value=300)
- key_th = gr.Slider(minimum=5.0, maximum=100.0, step=0.1, label='Threshold of delta frame edge', value=27.0)
+ key_th = gr.Slider(minimum=0.0, maximum=100.0, step=0.1, label='Threshold of delta frame edge', value=8.5)
key_add_last_frame = gr.Checkbox(label="Add last frame to keyframes", value=True)
with gr.Accordion(label="stage 7"):
blend_rate = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='Crossfade blend rate', value=1.0)
export_type = gr.Dropdown(choices=["mp4","webm","gif","rawvideo"], value="mp4" ,label="Export type")
+ with gr.Accordion(label="stage 8"):
+ bg_src = gr.Textbox(label='Background source(mp4 or directory containing images)', lines=1)
+ bg_type = gr.Dropdown(choices=["Fit video length","Loop"], value="Fit video length" ,label="Background type")
+ mask_blur_size = gr.Slider(minimum=0, maximum=150, step=1, label='Mask Blur Kernel Size', value=5)
+
with gr.Accordion(label="etc"):
- no_mask_mode = gr.Checkbox(label="No Mask Mode", value=False)
+ mask_mode = gr.Dropdown(choices=["Normal","Invert","None"], value="Normal" ,label="Mask Mode")
with gr.Column(variant='panel'):
with gr.Column(scale=1):
@@ -36,7 +50,7 @@ def on_ui_tabs():
debug_info = gr.HTML(elem_id="ebs_info_area", value=".")
with gr.Column(scale=2):
- stage_index = gr.Radio(label='Process Stage', choices=["stage 1","stage 2","stage 3","stage 4","stage 5","stage 6","stage 7"], value="stage 1", type="index")
+ stage_index = gr.Radio(label='Process Stage', choices=["stage 1","stage 2","stage 3","stage 4","stage 5","stage 6","stage 7","stage 8"], value="stage 1", type="index")
gr.HTML(value="\
The process of creating a video can be divided into the following stages.
\
(Stage 3, 4, and 6 only show a guide and do nothing actual processing.)
\
@@ -59,7 +73,13 @@ def on_ui_tabs():
If multiple .ebs files are generated, run them all.
\
stage 7
\
Concatenate each frame while crossfading.
\
- Composite audio files extracted from the original video onto the concatenated video.
\
+ Composite audio files extracted from the original video onto the concatenated video.
\
+ stage 8
\
+ This is an extra stage.
\
+ You can put any image or images or video you like in the background.
\
+ You can specify in this field -> [Ebsynth Utility]->[configuration]->[stage 8]->[Background source]
\
+ If you have already created a background video in Invert Mask Mode([Ebsynth Utility]->[configuration]->[etc]->[Mask Mode]),
\
+ You can specify \"path_to_project_dir/inv/crossfade_tmp\".
\
")
with gr.Row():
@@ -77,6 +97,9 @@ def on_ui_tabs():
project_dir,
original_movie_path,
+ tb_use_fast_mode,
+ tb_use_jit,
+
key_min_gap,
key_max_gap,
key_th,
@@ -85,7 +108,11 @@ def on_ui_tabs():
blend_rate,
export_type,
- no_mask_mode,
+ bg_src,
+ bg_type,
+ mask_blur_size,
+
+ mask_mode,
],
outputs=[
diff --git a/stage1.py b/stage1.py
index 5ad84ab..d30d763 100644
--- a/stage1.py
+++ b/stage1.py
@@ -12,12 +12,17 @@ def remove_pngs_in_dir(path):
for png in pngs:
os.remove(png)
-def ebsynth_utility_stage1(dbg, project_args):
+def ebsynth_utility_stage1(dbg, project_args, tb_use_fast_mode, tb_use_jit, is_invert_mask):
dbg.print("stage1")
dbg.print("")
_, original_movie_path, frame_path, frame_mask_path, _, _, _ = project_args
+ if is_invert_mask:
+ if os.path.isdir( frame_path ) and os.path.isdir( frame_mask_path ):
+ dbg.print("Skip as it appears that the frame and normal masks have already been generated.")
+ return
+
remove_pngs_in_dir(frame_path)
if frame_mask_path:
@@ -35,7 +40,9 @@ def ebsynth_utility_stage1(dbg, project_args):
dbg.print("frame extracted")
if frame_mask_path:
- subprocess.call("venv\\Scripts\\transparent-background --source " + frame_path + " --dest " + frame_mask_path + " --type map --fast", shell=True)
+ fast_str = " --fast" if tb_use_fast_mode else ""
+ jit_str = " --jit" if tb_use_jit else ""
+ subprocess.call("venv\\Scripts\\transparent-background --source " + frame_path + " --dest " + frame_mask_path + " --type map" + fast_str + jit_str, shell=True)
mask_imgs = glob.glob( os.path.join(frame_mask_path, "*.png") )
@@ -57,3 +64,26 @@ def ebsynth_utility_stage1(dbg, project_args):
dbg.print("completed.")
+def ebsynth_utility_stage1_invert(dbg, frame_mask_path, inv_mask_path):
+ dbg.print("stage 1 create_invert_mask")
+ dbg.print("")
+
+ if not os.path.isdir( frame_mask_path ):
+ dbg.print( frame_mask_path + " not found")
+ dbg.print("Normal masks must be generated previously.")
+ dbg.print("Do stage 1 with [Ebsynth Utility] Tab -> [configuration] -> [etc]-> [Mask Mode] = Normal setting first")
+ return
+
+ os.makedirs(inv_mask_path, exist_ok=True)
+
+ mask_imgs = glob.glob( os.path.join(frame_mask_path, "*.png") )
+
+ for m in mask_imgs:
+ img = cv2.imread(m)
+ inv = cv2.bitwise_not(img)
+
+ base_name = os.path.basename(m)
+ cv2.imwrite(os.path.join(inv_mask_path,base_name), inv)
+
+ dbg.print("")
+ dbg.print("completed.")
diff --git a/stage2.py b/stage2.py
index 2c53263..762affb 100644
--- a/stage2.py
+++ b/stage2.py
@@ -54,18 +54,32 @@ def _detect_edges(lum: np.ndarray) -> np.ndarray:
#---------------------------------
-def detect_edges(img_path):
- hue, sat, lum = cv2.split(cv2.cvtColor( cv2.imread(img_path) , cv2.COLOR_BGR2HSV))
+def detect_edges(img_path, mask_path, is_invert_mask):
+ im = cv2.imread(img_path)
+ if mask_path:
+ mask = cv2.imread(mask_path)[:,:,0]
+ mask = mask[:, :, np.newaxis]
+ im = im * ( (mask == 0) if is_invert_mask else (mask > 0) )
+# im = im * (mask/255)
+# im = im.astype(np.uint8)
+# cv2.imwrite( os.path.join( os.path.dirname(mask_path) , "tmp.png" ) , im)
+
+ hue, sat, lum = cv2.split(cv2.cvtColor( im , cv2.COLOR_BGR2HSV))
return _detect_edges(lum)
-def analyze_key_frames(png_dir, th, min_gap, max_gap, add_last_frame):
+def get_mask_path_of_img(img_path, mask_dir):
+ img_basename = os.path.basename(img_path)
+ mask_path = os.path.join( mask_dir , img_basename )
+ return mask_path if os.path.isfile( mask_path ) else None
+
+def analyze_key_frames(png_dir, mask_dir, th, min_gap, max_gap, add_last_frame, is_invert_mask):
keys = []
frames = sorted(glob.glob( os.path.join(png_dir, "[0-9]*.png") ))
key_frame = frames[0]
keys.append( int(os.path.splitext(os.path.basename(key_frame))[0]) )
- key_edges = detect_edges( key_frame )
+ key_edges = detect_edges( key_frame, get_mask_path_of_img( key_frame, mask_dir ), is_invert_mask )
gap = 0
for frame in frames:
@@ -73,7 +87,7 @@ def analyze_key_frames(png_dir, th, min_gap, max_gap, add_last_frame):
if gap < min_gap:
continue
- edges = detect_edges( frame )
+ edges = detect_edges( frame, get_mask_path_of_img( frame, mask_dir ), is_invert_mask )
delta = mean_pixel_distance( edges, key_edges )
@@ -102,11 +116,11 @@ def remove_pngs_in_dir(path):
for png in pngs:
os.remove(png)
-def ebsynth_utility_stage2(dbg, project_args, key_min_gap, key_max_gap, key_th, key_add_last_frame):
+def ebsynth_utility_stage2(dbg, project_args, key_min_gap, key_max_gap, key_th, key_add_last_frame, is_invert_mask):
dbg.print("stage2")
dbg.print("")
- _, original_movie_path, frame_path, _, org_key_path, _, _ = project_args
+ _, original_movie_path, frame_path, frame_mask_path, org_key_path, _, _ = project_args
remove_pngs_in_dir(org_key_path)
os.makedirs(org_key_path, exist_ok=True)
@@ -136,7 +150,7 @@ def ebsynth_utility_stage2(dbg, project_args, key_min_gap, key_max_gap, key_th,
dbg.print("key_max_gap: {}".format(key_max_gap))
dbg.print("key_th: {}".format(key_th))
- keys = analyze_key_frames(frame_path, key_th, key_min_gap, key_max_gap, key_add_last_frame)
+ keys = analyze_key_frames(frame_path, frame_mask_path, key_th, key_min_gap, key_max_gap, key_add_last_frame, is_invert_mask)
dbg.print("keys : " + str(keys))
@@ -147,6 +161,10 @@ def ebsynth_utility_stage2(dbg, project_args, key_min_gap, key_max_gap, key_th,
dbg.print("")
dbg.print("Keyframes are output to [" + org_key_path + "]")
+ dbg.print("")
+ dbg.print("[Ebsynth Utility]->[configuration]->[stage 2]->[Threshold of delta frame edge]")
+ dbg.print("The smaller this value, the narrower the keyframe spacing, and if set to 0, the keyframes will be equally spaced at the value of [Minimum keyframe gap].")
+ dbg.print("")
dbg.print("If you do not like the selection, you can modify it manually.")
dbg.print("(Delete keyframe, or Add keyframe from ["+frame_path+"])")
diff --git a/stage5.py b/stage5.py
index e5dcc3c..47afb3a 100644
--- a/stage5.py
+++ b/stage5.py
@@ -167,7 +167,7 @@ def rename_keys(key_dir):
dirname = os.path.dirname(img)
os.rename(img, os.path.join(dirname, f))
-def ebsynth_utility_stage5(dbg, project_args):
+def ebsynth_utility_stage5(dbg, project_args, is_invert_mask):
dbg.print("stage5")
dbg.print("")
@@ -212,13 +212,13 @@ def ebsynth_utility_stage5(dbg, project_args):
prev_key = key
project = {
- "proj_dir" : project_dir,
+ "proj_dir" : project_dir if is_invert_mask == False else os.path.join(project_dir, "inv"),
"file_name" : "/[" + "#" * number_of_digits + "].png",
"number_of_digits" : number_of_digits,
"key_dir" : "img2img_upscale_key",
- "video_dir" : "video_frame",
- "mask_dir" : "video_mask",
+ "video_dir" : "video_frame" if is_invert_mask == False else "../video_frame",
+ "mask_dir" : "video_mask" if is_invert_mask == False else "inv_video_mask",
"key_weight" : 1.0,
"video_weight" : 4.0,
"mask_weight" : 1.0,
@@ -234,6 +234,8 @@ def ebsynth_utility_stage5(dbg, project_args):
project["mask_dir"] = ""
proj_base_name = time.strftime("%Y%m%d-%H%M%S")
+ if is_invert_mask:
+ proj_base_name = "inv_" + proj_base_name
tmp=[]
proj_index = 0
diff --git a/stage7.py b/stage7.py
index 55fcb93..e74c590 100644
--- a/stage7.py
+++ b/stage7.py
@@ -91,8 +91,21 @@ def get_ext(export_type):
return "." + export_type
else:
return ".avi"
+
+def trying_to_add_audio(original_movie_path, no_snd_movie_path, output_path, tmp_dir ):
+ if os.path.isfile(original_movie_path):
+ sound_path = os.path.join(tmp_dir , 'sound.mp4')
+ subprocess.call("ffmpeg -i " + original_movie_path + " -vn -acodec copy " + sound_path, shell=True)
+
+ if os.path.isfile(sound_path):
+ # ffmpeg -i video.mp4 -i audio.wav -c:v copy -c:a aac -map 0:v:0 -map 1:a:0 output.mp4
+
+ subprocess.call("ffmpeg -i " + no_snd_movie_path + " -i " + sound_path + " -c:v copy -c:a aac -map 0:v:0 -map 1:a:0 " + output_path, shell=True)
+ return True
-def ebsynth_utility_stage7(dbg, project_args, blend_rate,export_type):
+ return False
+
+def ebsynth_utility_stage7(dbg, project_args, blend_rate,export_type,is_invert_mask):
dbg.print("stage7")
dbg.print("")
@@ -110,7 +123,11 @@ def ebsynth_utility_stage7(dbg, project_args, blend_rate,export_type):
dbg.print("export_type: {}".format(export_type))
dbg.print("fps: {}".format(fps))
+ if is_invert_mask:
+ project_dir = os.path.join( project_dir , "inv")
+
tmp_dir = os.path.join( project_dir , "crossfade_tmp")
+
if os.path.isdir(tmp_dir):
shutil.rmtree(tmp_dir)
@@ -191,9 +208,11 @@ def ebsynth_utility_stage7(dbg, project_args, blend_rate,export_type):
filename = str(i).zfill(number_of_digits) + ".png"
shutil.copy( os.path.join(out_dirs[cur_clip]['path'] , filename) , os.path.join(tmp_dir , filename) )
-
### create movie
movie_base_name = time.strftime("%Y%m%d-%H%M%S")
+ if is_invert_mask:
+ movie_base_name = "inv_" + movie_base_name
+
nosnd_path = os.path.join(project_dir , movie_base_name + get_ext(export_type))
start = out_dirs[0]['startframe']
@@ -204,18 +223,11 @@ def ebsynth_utility_stage7(dbg, project_args, blend_rate,export_type):
dbg.print("exported : " + nosnd_path)
if export_type == "mp4":
- if os.path.isfile(original_movie_path):
- sound_path = os.path.join(tmp_dir , 'sound.mp4')
- subprocess.call("ffmpeg -i " + original_movie_path + " -vn -acodec copy " + sound_path, shell=True)
-
- if os.path.isfile(sound_path):
- # ffmpeg -i video.mp4 -i audio.wav -c:v copy -c:a aac -map 0:v:0 -map 1:a:0 output.mp4
-
- with_snd_path = os.path.join(project_dir , movie_base_name + '_with_snd.mp4')
-
- subprocess.call("ffmpeg -i " + nosnd_path + " -i " + sound_path + " -c:v copy -c:a aac -map 0:v:0 -map 1:a:0 " + with_snd_path, shell=True)
- dbg.print("exported : " + with_snd_path)
+ with_snd_path = os.path.join(project_dir , movie_base_name + '_with_snd.mp4')
+
+ if trying_to_add_audio(original_movie_path, nosnd_path, with_snd_path, tmp_dir):
+ dbg.print("exported : " + with_snd_path)
dbg.print("")
dbg.print("completed.")
diff --git a/stage8.py b/stage8.py
new file mode 100644
index 0000000..92b6650
--- /dev/null
+++ b/stage8.py
@@ -0,0 +1,141 @@
+import os
+import re
+import subprocess
+import glob
+import shutil
+import time
+import cv2
+import numpy as np
+import itertools
+from extensions.ebsynth_utility.stage7 import create_movie_from_frames, get_ext, trying_to_add_audio
+
+def clamp(n, smallest, largest):
+ return sorted([smallest, n, largest])[1]
+
+def resize_img(img, w, h):
+ if img.shape[0] + img.shape[1] < h + w:
+ interpolation = interpolation=cv2.INTER_CUBIC
+ else:
+ interpolation = interpolation=cv2.INTER_AREA
+
+ return cv2.resize(img, (w, h), interpolation=interpolation)
+
+def merge_bg_src(base_frame_dir, bg_dir, frame_mask_path, tmp_dir, bg_type, mask_blur_size):
+
+ base_frames = sorted(glob.glob( os.path.join(base_frame_dir, "[0-9]*.png"), recursive=False) )
+
+ bg_frames = sorted(glob.glob( os.path.join(bg_dir, "*.png"), recursive=False) )
+
+ def bg_frame(total_frames):
+ bg_len = len(bg_frames)
+
+ if bg_type == "Loop":
+ itr = itertools.cycle(bg_frames)
+ while True:
+ yield next(itr)
+ else:
+ for i in range(total_frames):
+ yield bg_frames[ int(bg_len * (i/total_frames))]
+
+ bg_itr = bg_frame(len(base_frames))
+
+ for base_frame in base_frames:
+ im = cv2.imread(base_frame)
+ bg = cv2.imread( next(bg_itr) )
+ bg = resize_img(bg, im.shape[1], im.shape[0] )
+
+ basename = os.path.basename(base_frame)
+ mask_path = os.path.join(frame_mask_path, basename)
+ mask = cv2.imread(mask_path)[:,:,0]
+ if mask_blur_size > 0:
+ mask_blur_size = mask_blur_size//2 * 2 + 1
+ mask = cv2.GaussianBlur(mask, (mask_blur_size, mask_blur_size), 0)
+ mask = mask[:, :, np.newaxis]
+
+ im = im * (mask/255) + bg * (1- mask/255)
+ im = im.astype(np.uint8)
+ cv2.imwrite( os.path.join( tmp_dir , basename ) , im)
+
+def extract_frames(movie_path , output_dir, fps):
+ png_path = os.path.join(output_dir , "%05d.png")
+ # ffmpeg.exe -ss 00:00:00 -y -i %1 -qscale 0 -f image2 -c:v png "%05d.png"
+ subprocess.call("ffmpeg.exe -ss 00:00:00 -y -i " + movie_path + " -vf fps=" + str( round(fps, 2)) + " -qscale 0 -f image2 -c:v png " + png_path, shell=True)
+
+def ebsynth_utility_stage8(dbg, project_args, bg_src, bg_type, mask_blur_size, export_type):
+ dbg.print("stage8")
+ dbg.print("")
+
+ if not bg_src:
+ dbg.print("Fill [configuration] -> [stage 8] -> [Background source]")
+ return
+
+ project_dir, original_movie_path, _, frame_mask_path, _, _, _ = project_args
+
+ fps = 30
+ clip = cv2.VideoCapture(original_movie_path)
+ if clip:
+ fps = clip.get(cv2.CAP_PROP_FPS)
+ clip.release()
+
+ dbg.print("bg_src: {}".format(bg_src))
+ dbg.print("bg_type: {}".format(bg_type))
+ dbg.print("mask_blur_size: {}".format(mask_blur_size))
+ dbg.print("export_type: {}".format(export_type))
+ dbg.print("fps: {}".format(fps))
+
+ base_frame_dir = os.path.join( project_dir , "crossfade_tmp")
+
+ if not os.path.isdir(base_frame_dir):
+ dbg.print(base_frame_dir + " base frame not found")
+ return
+
+ tmp_dir = os.path.join( project_dir , "bg_merge_tmp")
+ if os.path.isdir(tmp_dir):
+ shutil.rmtree(tmp_dir)
+ os.mkdir(tmp_dir)
+
+ ### create frame imgs
+ if os.path.isfile(bg_src):
+ bg_ext = os.path.splitext(os.path.basename(bg_src))[1]
+ if bg_ext == ".mp4":
+ bg_tmp_dir = os.path.join( project_dir , "bg_extract_tmp")
+ if os.path.isdir(bg_tmp_dir):
+ shutil.rmtree(bg_tmp_dir)
+ os.mkdir(bg_tmp_dir)
+
+ extract_frames(bg_src, bg_tmp_dir, fps)
+
+ bg_src = bg_tmp_dir
+ else:
+ dbg.print(bg_src + " must be mp4 or directory")
+ return
+ elif not os.path.isdir(bg_src):
+ dbg.print(bg_src + " must be mp4 or directory")
+ return
+
+ merge_bg_src(base_frame_dir, bg_src, frame_mask_path, tmp_dir, bg_type, mask_blur_size)
+
+ ### create movie
+ movie_base_name = time.strftime("%Y%m%d-%H%M%S")
+ movie_base_name = "merge_" + movie_base_name
+
+ nosnd_path = os.path.join(project_dir , movie_base_name + get_ext(export_type))
+
+ merged_frames = sorted(glob.glob( os.path.join(tmp_dir, "[0-9]*.png"), recursive=False) )
+ start = int(os.path.splitext(os.path.basename(merged_frames[0]))[0])
+ end = int(os.path.splitext(os.path.basename(merged_frames[-1]))[0])
+
+ create_movie_from_frames(tmp_dir,start,end,5,fps,nosnd_path,export_type)
+
+ dbg.print("exported : " + nosnd_path)
+
+ if export_type == "mp4":
+
+ with_snd_path = os.path.join(project_dir , movie_base_name + '_with_snd.mp4')
+
+ if trying_to_add_audio(original_movie_path, nosnd_path, with_snd_path, tmp_dir):
+ dbg.print("exported : " + with_snd_path)
+
+ dbg.print("")
+ dbg.print("completed.")
+