From 07747ed844103e27421401d6343c59b5a048d5a5 Mon Sep 17 00:00:00 2001 From: alexbofa <58225118+alexbofa@users.noreply.github.com> Date: Sun, 22 Oct 2023 00:55:16 +0300 Subject: [PATCH] Main Repository --- README.md | 147 +++++- calculator.py | 237 +++++++++ ebsynth_utility.py | 361 +++++++------- install.py | 59 +-- scripts/custom_script.py | 1012 ++++++++++++++++++++++++++++++++++++++ scripts/ui.py | 367 +++++++------- stage3_5.py | 178 +++++++ stage5.py | 2 +- stage8.py | 292 +++++------ style.css | 82 +-- 10 files changed, 2156 insertions(+), 581 deletions(-) create mode 100644 calculator.py create mode 100644 scripts/custom_script.py create mode 100644 stage3_5.py diff --git a/README.md b/README.md index 8b856ab..841c529 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,58 @@ -# ebsynth_utility_lite -Fork was created to facilitate the creation of videos via img2img based on the original [ebsynth_utility](https://github.com/s9roll7/ebsynth_utility) -
-## TODO -- [x] Delete script for img2img -- [ ] Add configuration → stage 5 -- [ ] Stage 0 — changing the video size, for example from 1080x1920 to 512x904 -- [ ] Stage 2 — manually add **custom_gap** -- [ ] Change Stage 3 for create a grid (min 1x1 max 3x3) -- [ ] Change Stage 4 for disassemble the grid back -- [ ] Stage 0 — add Presets (with changes via .json) -- [ ] Stage 5 — automatisation with Ebsynth? (Is it possible?) -- [ ] Edit **Readme.md** +# ebsynth_utility -#### If you want to help, feel free to create the [PR](https://github.com/alexbofa/ebsynth_utility_lite/pulls) +## Overview +#### AUTOMATIC1111 UI extension for creating videos using img2img and ebsynth. +#### This extension allows you to output edited videos using ebsynth.(AE is not required) + + +##### With [Controlnet](https://github.com/Mikubill/sd-webui-controlnet) installed, I have confirmed that all features of this extension are working properly! +##### [Controlnet](https://github.com/Mikubill/sd-webui-controlnet) is a must for video editing, so I recommend installing it. +##### Multi ControlNet("canny" + "normal map") would be suitable for video editing. + +
+ +###### I modified animatediff-cli to create a txt2video tool that allows flexible prompt specification. You can use it if you like. +###### [animatediff-cli-prompt-travel](https://github.com/s9roll7/animatediff-cli-prompt-travel) +
+ + +
+ + +## Example +- The following sample is raw output of this extension. +#### sample 1 mask with [clipseg](https://github.com/timojl/clipseg) +- first from left : original +- second from left : masking "cat" exclude "finger" +- third from left : masking "cat head" +- right : color corrected with [color-matcher](https://github.com/hahnec/color-matcher) (see stage 3.5) +- Multiple targets can also be specified.(e.g. cat,dog,boy,girl) +
+ +#### sample 2 blend background +- person : masterpiece, best quality, masterpiece, 1girl, masterpiece, best quality,anime screencap, anime style +- background : cyberpunk, factory, room ,anime screencap, anime style +- It is also possible to blend with your favorite videos. +
+ +#### sample 3 auto tagging +- left : original +- center : apply the same prompts in all keyframes +- right : apply auto tagging by deepdanbooru in all keyframes +- This function improves the detailed changes in facial expressions, hand expressions, etc. + In the sample video, the "closed_eyes" and "hands_on_own_face" tags have been added to better represent eye blinks and hands brought in front of the face. +
+ +#### sample 4 auto tagging (apply lora dynamically) +- left : apply auto tagging by deepdanbooru in all keyframes +- right : apply auto tagging by deepdanbooru in all keyframes + apply "anyahehface" lora dynamically +- Added the function to dynamically apply TI, hypernet, Lora, and additional prompts according to automatically attached tags. + In the sample video, if the "smile" tag is given, the lora and lora trigger keywords are set to be added according to the strength of the "smile" tag. + Also, since automatically added tags are sometimes incorrect, unnecessary tags are listed in the blacklist. + [Here](sample/) is the actual configuration file used. placed in "Project directory" for use. +
+ +
## Installation - Install [ffmpeg](https://ffmpeg.org/) for your operating system @@ -20,9 +60,12 @@ Fork was created to facilitate the creation of videos via img2img based on the o - Install [Ebsynth](https://ebsynth.com/) - Use the Extensions tab of the webui to [Install from URL] +
+
+ ## Usage - Go to [Ebsynth Utility] tab. -- Create an empty directory somewhere, and fill in the «Project directory» field. +- Create an empty directory somewhere, and fill in the "Project directory" field. - Place the video you want to edit from somewhere, and fill in the "Original Movie Path" field. Use short videos of a few seconds at first. - Select stage 1 and Generate. @@ -32,6 +75,7 @@ Fork was created to facilitate the creation of videos via img2img based on the o (In the current latest webui, it seems to cause an error if you do not drop the image on the main screen of img2img. Please drop the image as it does not affect the result.) +

## Note 1 @@ -45,9 +89,51 @@ In the implementation of this extension, the keyframe interval is chosen to be s If the animation breaks up, increase the keyframe, if it flickers, decrease the keyframe. First, generate one time with the default settings and go straight ahead without worrying about the result. -#### Stage 3 (In development) + +#### Stage 3 +Select one of the keyframes, throw it to img2img, and run [Interrogate DeepBooru]. +Delete unwanted words such as blur from the displayed prompt. +Fill in the rest of the settings as you would normally do for image generation. + +Here is the settings I used. +- Sampling method : Euler a +- Sampling Steps : 50 +- Width : 960 +- Height : 512 +- CFG Scale : 20 +- Denoising strength : 0.2 + +Here is the settings for extension. +- Mask Mode(Override img2img Mask mode) : Normal +- Img2Img Repeat Count (Loop Back) : 5 +- Add N to seed when repeating : 1 +- use Face Crop img2img : True +- Face Detection Method : YuNet +- Max Crop Size : 1024 +- Face Denoising Strength : 0.25 +- Face Area Magnification : 1.5 (The larger the number, the closer to the model's painting style, but the more likely it is to shift when merged with the body.) +- Enable Face Prompt : False + +Trial and error in this process is the most time-consuming part. +Monitor the destination folder and if you do not like results, interrupt and change the settings. +[Prompt][Denoising strength] and [Face Denoising Strength] settings when using Face Crop img2img will greatly affect the result. +For more information on Face Crop img2img, check [here](https://github.com/s9roll7/face_crop_img2img) + +If you have lots of memory to spare, increasing the width and height values while maintaining the aspect ratio may greatly improve results. + +This extension may help with the adjustment. +https://github.com/s9roll7/img2img_for_all_method -#### Stage 4 (Will be changed in the future) +
+ +**The information above is from a time when there was no controlnet. +When controlnet are used together (especially multi-controlnets), +Even setting "Denoising strength" to a high value works well, and even setting it to 1.0 produces meaningful results. +If "Denoising strength" is set to a high value, "Loop Back" can be set to 1.** + +
+ +#### Stage 4 Scale it up or down and process it to exactly the same size as the original video. This process should only need to be done once. @@ -73,4 +159,29 @@ Finally, output the video. In my case, the entire process from 1 to 7 took about 30 minutes. - Crossfade blend rate : 1.0 -- Export type : mp4 +- Export type : mp4 + +
+
+ +## Note 2 : How to use multi-controlnet together +#### in webui setting +![controlnet_setting](imgs/controlnet_setting.png "controlnet_setting") +
+#### In controlnet settings in img2img tab(for controlnet 0) +![controlnet_0](imgs/controlnet_0.png "controlnet_0") +
+#### In controlnet settings in img2img tab(for controlnet 1) +![controlnet_1](imgs/controlnet_1.png "controlnet_1") +
+#### In ebsynth_utility settings in img2img tab +**Warning : "Weight" in the controlnet settings is overridden by the following values** +![controlnet_option_in_ebsynthutil](imgs/controlnet_option_in_ebsynthutil.png "controlnet_option_in_ebsynthutil") + +
+
+ +## Note 3 : How to use clipseg +![clipseg](imgs/clipseg.png "How to use clipseg") + + diff --git a/calculator.py b/calculator.py new file mode 100644 index 0000000..35d66bf --- /dev/null +++ b/calculator.py @@ -0,0 +1,237 @@ +# https://www.mycompiler.io/view/3TFZagC + +class ParseError(Exception): + def __init__(self, pos, msg, *args): + self.pos = pos + self.msg = msg + self.args = args + + def __str__(self): + return '%s at position %s' % (self.msg % self.args, self.pos) + +class Parser: + def __init__(self): + self.cache = {} + + def parse(self, text): + self.text = text + self.pos = -1 + self.len = len(text) - 1 + rv = self.start() + self.assert_end() + return rv + + def assert_end(self): + if self.pos < self.len: + raise ParseError( + self.pos + 1, + 'Expected end of string but got %s', + self.text[self.pos + 1] + ) + + def eat_whitespace(self): + while self.pos < self.len and self.text[self.pos + 1] in " \f\v\r\t\n": + self.pos += 1 + + def split_char_ranges(self, chars): + try: + return self.cache[chars] + except KeyError: + pass + + rv = [] + index = 0 + length = len(chars) + + while index < length: + if index + 2 < length and chars[index + 1] == '-': + if chars[index] >= chars[index + 2]: + raise ValueError('Bad character range') + + rv.append(chars[index:index + 3]) + index += 3 + else: + rv.append(chars[index]) + index += 1 + + self.cache[chars] = rv + return rv + + def char(self, chars=None): + if self.pos >= self.len: + raise ParseError( + self.pos + 1, + 'Expected %s but got end of string', + 'character' if chars is None else '[%s]' % chars + ) + + next_char = self.text[self.pos + 1] + if chars == None: + self.pos += 1 + return next_char + + for char_range in self.split_char_ranges(chars): + if len(char_range) == 1: + if next_char == char_range: + self.pos += 1 + return next_char + elif char_range[0] <= next_char <= char_range[2]: + self.pos += 1 + return next_char + + raise ParseError( + self.pos + 1, + 'Expected %s but got %s', + 'character' if chars is None else '[%s]' % chars, + next_char + ) + + def keyword(self, *keywords): + self.eat_whitespace() + if self.pos >= self.len: + raise ParseError( + self.pos + 1, + 'Expected %s but got end of string', + ','.join(keywords) + ) + + for keyword in keywords: + low = self.pos + 1 + high = low + len(keyword) + + if self.text[low:high] == keyword: + self.pos += len(keyword) + self.eat_whitespace() + return keyword + + raise ParseError( + self.pos + 1, + 'Expected %s but got %s', + ','.join(keywords), + self.text[self.pos + 1], + ) + + def match(self, *rules): + self.eat_whitespace() + last_error_pos = -1 + last_exception = None + last_error_rules = [] + + for rule in rules: + initial_pos = self.pos + try: + rv = getattr(self, rule)() + self.eat_whitespace() + return rv + except ParseError as e: + self.pos = initial_pos + + if e.pos > last_error_pos: + last_exception = e + last_error_pos = e.pos + last_error_rules.clear() + last_error_rules.append(rule) + elif e.pos == last_error_pos: + last_error_rules.append(rule) + + if len(last_error_rules) == 1: + raise last_exception + else: + raise ParseError( + last_error_pos, + 'Expected %s but got %s', + ','.join(last_error_rules), + self.text[last_error_pos] + ) + + def maybe_char(self, chars=None): + try: + return self.char(chars) + except ParseError: + return None + + def maybe_match(self, *rules): + try: + return self.match(*rules) + except ParseError: + return None + + def maybe_keyword(self, *keywords): + try: + return self.keyword(*keywords) + except ParseError: + return None + +class CalcParser(Parser): + def start(self): + return self.expression() + + def expression(self): + rv = self.match('term') + while True: + op = self.maybe_keyword('+', '-') + if op is None: + break + + term = self.match('term') + if op == '+': + rv += term + else: + rv -= term + + return rv + + def term(self): + rv = self.match('factor') + while True: + op = self.maybe_keyword('*', '/') + if op is None: + break + + term = self.match('factor') + if op == '*': + rv *= term + else: + rv /= term + + return rv + + def factor(self): + if self.maybe_keyword('('): + rv = self.match('expression') + self.keyword(')') + + return rv + + return self.match('number') + + def number(self): + chars = [] + + sign = self.maybe_keyword('+', '-') + if sign is not None: + chars.append(sign) + + chars.append(self.char('0-9')) + + while True: + char = self.maybe_char('0-9') + if char is None: + break + + chars.append(char) + + if self.maybe_char('.'): + chars.append('.') + chars.append(self.char('0-9')) + + while True: + char = self.maybe_char('0-9') + if char is None: + break + + chars.append(char) + + rv = float(''.join(chars)) + return rv + diff --git a/ebsynth_utility.py b/ebsynth_utility.py index d8cadc2..68c7466 100644 --- a/ebsynth_utility.py +++ b/ebsynth_utility.py @@ -1,176 +1,185 @@ -import os - -from modules.ui import plaintext_to_html - -import cv2 -import glob -from PIL import Image - -from extensions.ebsynth_utility_lite.stage1 import ebsynth_utility_stage1,ebsynth_utility_stage1_invert -from extensions.ebsynth_utility_lite.stage2 import ebsynth_utility_stage2 -from extensions.ebsynth_utility_lite.stage5 import ebsynth_utility_stage5 -from extensions.ebsynth_utility_lite.stage7 import ebsynth_utility_stage7 -from extensions.ebsynth_utility_lite.stage8 import ebsynth_utility_stage8 - - -def x_ceiling(value, step): - return -(-value // step) * step - -def dump_dict(string, d:dict): - for key in d.keys(): - string += ( key + " : " + str(d[key]) + "\n") - return string - -class debug_string: - txt = "" - def print(self, comment): - print(comment) - self.txt += comment + '\n' - def to_string(self): - return self.txt - -def ebsynth_utility_process(stage_index: int, project_dir:str, original_movie_path:str, frame_width:int, frame_height:int, st1_masking_method_index:int, st1_mask_threshold:float, tb_use_fast_mode:bool, tb_use_jit:bool, clipseg_mask_prompt:str, clipseg_exclude_prompt:str, clipseg_mask_threshold:int, clipseg_mask_blur_size:int, clipseg_mask_blur_size2:int, key_min_gap:int, key_max_gap:int, key_th:float, key_add_last_frame:bool, blend_rate:float, export_type:str, bg_src:str, bg_type:str, mask_blur_size:int, mask_threshold:float, fg_transparency:float, mask_mode:str): - args = locals() - info = "" - info = dump_dict(info, args) - dbg = debug_string() - - - def process_end(dbg, info): - return plaintext_to_html(dbg.to_string()), plaintext_to_html(info) - - - if not os.path.isdir(project_dir): - dbg.print("{0} project_dir not found".format(project_dir)) - return process_end( dbg, info ) - - if not os.path.isfile(original_movie_path): - dbg.print("{0} original_movie_path not found".format(original_movie_path)) - return process_end( dbg, info ) - - is_invert_mask = False - if mask_mode == "Invert": - is_invert_mask = True - - frame_path = os.path.join(project_dir , "video_frame") - frame_mask_path = os.path.join(project_dir, "video_mask") - - if is_invert_mask: - inv_path = os.path.join(project_dir, "inv") - os.makedirs(inv_path, exist_ok=True) - - org_key_path = os.path.join(inv_path, "video_key") - img2img_key_path = os.path.join(inv_path, "img2img_key") - img2img_upscale_key_path = os.path.join(inv_path, "img2img_upscale_key") - else: - org_key_path = os.path.join(project_dir, "video_key") - img2img_key_path = os.path.join(project_dir, "img2img_key") - img2img_upscale_key_path = os.path.join(project_dir, "img2img_upscale_key") - - if mask_mode == "None": - frame_mask_path = "" - - - project_args = [project_dir, original_movie_path, frame_path, frame_mask_path, org_key_path, img2img_key_path, img2img_upscale_key_path] - - - if stage_index == 0: - ebsynth_utility_stage1(dbg, project_args, frame_width, frame_height, st1_masking_method_index, st1_mask_threshold, tb_use_fast_mode, tb_use_jit, clipseg_mask_prompt, clipseg_exclude_prompt, clipseg_mask_threshold, clipseg_mask_blur_size, clipseg_mask_blur_size2, is_invert_mask) - if is_invert_mask: - inv_mask_path = os.path.join(inv_path, "inv_video_mask") - ebsynth_utility_stage1_invert(dbg, frame_mask_path, inv_mask_path) - - elif stage_index == 1: - ebsynth_utility_stage2(dbg, project_args, key_min_gap, key_max_gap, key_th, key_add_last_frame, is_invert_mask) - elif stage_index == 2: - - sample_image = glob.glob( os.path.join(frame_path , "*.png" ) )[0] - img_height, img_width, _ = cv2.imread(sample_image).shape - if img_width < img_height: - re_w = 512 - re_h = int(x_ceiling( (512 / img_width) * img_height , 64)) - else: - re_w = int(x_ceiling( (512 / img_height) * img_width , 64)) - re_h = 512 - img_width = re_w - img_height = re_h - - dbg.print("stage 3") - dbg.print("") - dbg.print("This is an information button, it does not do anything") - dbg.print("") - dbg.print("1. Go to img2img tab") - dbg.print("2. Generate") - dbg.print("(Images are output to " + img2img_key_path + ")") - dbg.print("") - dbg.print("This tab will be changed to create a GRID (min 1x1 — max 3x3)") - dbg.print("") - dbg.print("If you know how to do it and want to help, create the PR") - return process_end( dbg, "" ) - - elif stage_index == 4: - sample_image = glob.glob( os.path.join(frame_path , "*.png" ) )[0] - img_height, img_width, _ = cv2.imread(sample_image).shape - - sample_img2img_key = glob.glob( os.path.join(img2img_key_path , "*.png" ) )[0] - img_height_key, img_width_key, _ = cv2.imread(sample_img2img_key).shape - - if is_invert_mask: - project_dir = inv_path - - dbg.print("stage 4") - dbg.print("") - - if img_height == img_height_key and img_width == img_width_key: - dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") - dbg.print("!! The size of frame and img2img_key matched.") - dbg.print("!! You can skip this stage.") - - dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") - dbg.print("0. Enable the following item") - dbg.print("Settings ->") - dbg.print(" Saving images/grids ->") - dbg.print(" Use original name for output filename during batch process in extras tab") - dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") - dbg.print("1. If \"img2img_upscale_key\" directory already exists in the %s, delete it manually before executing."%(project_dir)) - dbg.print("2. Go to Extras tab") - dbg.print("3. Go to Batch from Directory tab") - dbg.print("4. Fill in the \"Input directory\" field with [" + img2img_key_path + "]" ) - dbg.print("5. Fill in the \"Output directory\" field with [" + img2img_upscale_key_path + "]" ) - dbg.print("6. Go to Scale to tab") - dbg.print("7. Fill in the \"Width\" field with [" + str(img_width) + "]" ) - dbg.print("8. Fill in the \"Height\" field with [" + str(img_height) + "]" ) - dbg.print("9. Fill in the remaining configuration fields of Upscaler.") - dbg.print("10. Generate") - dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") - return process_end( dbg, "" ) - elif stage_index == 5: - ebsynth_utility_stage5(dbg, project_args, is_invert_mask) - elif stage_index == 6: - - if is_invert_mask: - project_dir = inv_path - - dbg.print("stage 6") - dbg.print("") - dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") - dbg.print("Running ebsynth.(on your self)") - dbg.print("Open the generated .ebs under %s and press [Run All] button."%(project_dir)) - dbg.print("If ""out-*"" directory already exists in the %s, delete it manually before executing."%(project_dir)) - dbg.print("If multiple .ebs files are generated, run them all.") - dbg.print("(I recommend associating the .ebs file with EbSynth.exe.)") - dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") - return process_end( dbg, "" ) - elif stage_index == 7: - ebsynth_utility_stage7(dbg, project_args, blend_rate, export_type, is_invert_mask) - elif stage_index == 8: - if mask_mode != "Normal": - dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") - dbg.print("Please reset [configuration]->[etc]->[Mask Mode] to Normal.") - dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") - return process_end( dbg, "" ) - ebsynth_utility_stage8(dbg, project_args, bg_src, bg_type, mask_blur_size, mask_threshold, fg_transparency, export_type) - else: - pass - - return process_end( dbg, info ) +import os + +from modules.ui import plaintext_to_html + +import cv2 +import glob +from PIL import Image + +from extensions.ebsynth_utility.stage1 import ebsynth_utility_stage1,ebsynth_utility_stage1_invert +from extensions.ebsynth_utility.stage2 import ebsynth_utility_stage2 +from extensions.ebsynth_utility.stage5 import ebsynth_utility_stage5 +from extensions.ebsynth_utility.stage7 import ebsynth_utility_stage7 +from extensions.ebsynth_utility.stage8 import ebsynth_utility_stage8 +from extensions.ebsynth_utility.stage3_5 import ebsynth_utility_stage3_5 + + +def x_ceiling(value, step): + return -(-value // step) * step + +def dump_dict(string, d:dict): + for key in d.keys(): + string += ( key + " : " + str(d[key]) + "\n") + return string + +class debug_string: + txt = "" + def print(self, comment): + print(comment) + self.txt += comment + '\n' + def to_string(self): + return self.txt + +def ebsynth_utility_process(stage_index: int, project_dir:str, original_movie_path:str, frame_width:int, frame_height:int, st1_masking_method_index:int, st1_mask_threshold:float, tb_use_fast_mode:bool, tb_use_jit:bool, clipseg_mask_prompt:str, clipseg_exclude_prompt:str, clipseg_mask_threshold:int, clipseg_mask_blur_size:int, clipseg_mask_blur_size2:int, key_min_gap:int, key_max_gap:int, key_th:float, key_add_last_frame:bool, color_matcher_method:str, st3_5_use_mask:bool, st3_5_use_mask_ref:bool, st3_5_use_mask_org:bool, color_matcher_ref_type:int, color_matcher_ref_image:Image, blend_rate:float, export_type:str, bg_src:str, bg_type:str, mask_blur_size:int, mask_threshold:float, fg_transparency:float, mask_mode:str): + args = locals() + info = "" + info = dump_dict(info, args) + dbg = debug_string() + + + def process_end(dbg, info): + return plaintext_to_html(dbg.to_string()), plaintext_to_html(info) + + + if not os.path.isdir(project_dir): + dbg.print("{0} project_dir not found".format(project_dir)) + return process_end( dbg, info ) + + if not os.path.isfile(original_movie_path): + dbg.print("{0} original_movie_path not found".format(original_movie_path)) + return process_end( dbg, info ) + + is_invert_mask = False + if mask_mode == "Invert": + is_invert_mask = True + + frame_path = os.path.join(project_dir , "video_frame") + frame_mask_path = os.path.join(project_dir, "video_mask") + + if is_invert_mask: + inv_path = os.path.join(project_dir, "inv") + os.makedirs(inv_path, exist_ok=True) + + org_key_path = os.path.join(inv_path, "video_key") + img2img_key_path = os.path.join(inv_path, "img2img_key") + img2img_upscale_key_path = os.path.join(inv_path, "img2img_upscale_key") + else: + org_key_path = os.path.join(project_dir, "video_key") + img2img_key_path = os.path.join(project_dir, "img2img_key") + img2img_upscale_key_path = os.path.join(project_dir, "img2img_upscale_key") + + if mask_mode == "None": + frame_mask_path = "" + + + project_args = [project_dir, original_movie_path, frame_path, frame_mask_path, org_key_path, img2img_key_path, img2img_upscale_key_path] + + + if stage_index == 0: + ebsynth_utility_stage1(dbg, project_args, frame_width, frame_height, st1_masking_method_index, st1_mask_threshold, tb_use_fast_mode, tb_use_jit, clipseg_mask_prompt, clipseg_exclude_prompt, clipseg_mask_threshold, clipseg_mask_blur_size, clipseg_mask_blur_size2, is_invert_mask) + if is_invert_mask: + inv_mask_path = os.path.join(inv_path, "inv_video_mask") + ebsynth_utility_stage1_invert(dbg, frame_mask_path, inv_mask_path) + + elif stage_index == 1: + ebsynth_utility_stage2(dbg, project_args, key_min_gap, key_max_gap, key_th, key_add_last_frame, is_invert_mask) + elif stage_index == 2: + + sample_image = glob.glob( os.path.join(frame_path , "*.png" ) )[0] + img_height, img_width, _ = cv2.imread(sample_image).shape + if img_width < img_height: + re_w = 512 + re_h = int(x_ceiling( (512 / img_width) * img_height , 64)) + else: + re_w = int(x_ceiling( (512 / img_height) * img_width , 64)) + re_h = 512 + img_width = re_w + img_height = re_h + + dbg.print("stage 3") + dbg.print("") + dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + dbg.print("1. Go to img2img tab") + dbg.print("2. Select [ebsynth utility] in the script combo box") + dbg.print("3. Fill in the \"Project directory\" field with [" + project_dir + "]" ) + dbg.print("4. Select in the \"Mask Mode(Override img2img Mask mode)\" field with [" + ("Invert" if is_invert_mask else "Normal") + "]" ) + dbg.print("5. I recommend to fill in the \"Width\" field with [" + str(img_width) + "]" ) + dbg.print("6. I recommend to fill in the \"Height\" field with [" + str(img_height) + "]" ) + dbg.print("7. I recommend to fill in the \"Denoising strength\" field with lower than 0.35" ) + dbg.print(" (When using controlnet together, you can put in large values (even 1.0 is possible).)") + dbg.print("8. Fill in the remaining configuration fields of img2img. No image and mask settings are required.") + dbg.print("9. Drop any image onto the img2img main screen. This is necessary to avoid errors, but does not affect the results of img2img.") + dbg.print("10. Generate") + dbg.print("(Images are output to [" + img2img_key_path + "])") + dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + return process_end( dbg, "" ) + + elif stage_index == 3: + ebsynth_utility_stage3_5(dbg, project_args, color_matcher_method, st3_5_use_mask, st3_5_use_mask_ref, st3_5_use_mask_org, color_matcher_ref_type, color_matcher_ref_image) + + elif stage_index == 4: + sample_image = glob.glob( os.path.join(frame_path , "*.png" ) )[0] + img_height, img_width, _ = cv2.imread(sample_image).shape + + sample_img2img_key = glob.glob( os.path.join(img2img_key_path , "*.png" ) )[0] + img_height_key, img_width_key, _ = cv2.imread(sample_img2img_key).shape + + if is_invert_mask: + project_dir = inv_path + + dbg.print("stage 4") + dbg.print("") + + if img_height == img_height_key and img_width == img_width_key: + dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + dbg.print("!! The size of frame and img2img_key matched.") + dbg.print("!! You can skip this stage.") + + dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + dbg.print("0. Enable the following item") + dbg.print("Settings ->") + dbg.print(" Saving images/grids ->") + dbg.print(" Use original name for output filename during batch process in extras tab") + dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + dbg.print("1. If \"img2img_upscale_key\" directory already exists in the %s, delete it manually before executing."%(project_dir)) + dbg.print("2. Go to Extras tab") + dbg.print("3. Go to Batch from Directory tab") + dbg.print("4. Fill in the \"Input directory\" field with [" + img2img_key_path + "]" ) + dbg.print("5. Fill in the \"Output directory\" field with [" + img2img_upscale_key_path + "]" ) + dbg.print("6. Go to Scale to tab") + dbg.print("7. Fill in the \"Width\" field with [" + str(img_width) + "]" ) + dbg.print("8. Fill in the \"Height\" field with [" + str(img_height) + "]" ) + dbg.print("9. Fill in the remaining configuration fields of Upscaler.") + dbg.print("10. Generate") + dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + return process_end( dbg, "" ) + elif stage_index == 5: + ebsynth_utility_stage5(dbg, project_args, is_invert_mask) + elif stage_index == 6: + + if is_invert_mask: + project_dir = inv_path + + dbg.print("stage 6") + dbg.print("") + dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + dbg.print("Running ebsynth.(on your self)") + dbg.print("Open the generated .ebs under %s and press [Run All] button."%(project_dir)) + dbg.print("If ""out-*"" directory already exists in the %s, delete it manually before executing."%(project_dir)) + dbg.print("If multiple .ebs files are generated, run them all.") + dbg.print("(I recommend associating the .ebs file with EbSynth.exe.)") + dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + return process_end( dbg, "" ) + elif stage_index == 7: + ebsynth_utility_stage7(dbg, project_args, blend_rate, export_type, is_invert_mask) + elif stage_index == 8: + if mask_mode != "Normal": + dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + dbg.print("Please reset [configuration]->[etc]->[Mask Mode] to Normal.") + dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + return process_end( dbg, "" ) + ebsynth_utility_stage8(dbg, project_args, bg_src, bg_type, mask_blur_size, mask_threshold, fg_transparency, export_type) + else: + pass + + return process_end( dbg, info ) diff --git a/install.py b/install.py index 38155ba..5bba6ff 100644 --- a/install.py +++ b/install.py @@ -1,28 +1,31 @@ -import launch -import platform - -def update_transparent_background(): - from importlib.metadata import version as meta_version - from packaging import version - v = meta_version("transparent-background") - print("current transparent-background " + v) - if version.parse(v) < version.parse('1.2.3'): - launch.run_pip("install -U transparent-background", "update transparent-background version for Ebsynth Utility Lite") - -# Check if user is running an M1/M2 device and, if so, install pyvirtualcam, which is required for updating the transparent_background package -# Note that we have to directly install from source because the prebuilt PyPl wheel does not support ARM64 machines such as M1/M2 Macs -if platform.system() == "Darwin" and platform.machine() == "arm64": - if not launch.is_installed("pyvirtualcam"): - launch.run_pip("install git+https://github.com/letmaik/pyvirtualcam", "requirements for Ebsynth Utility Lite") - -if not launch.is_installed("transparent_background"): - launch.run_pip("install transparent-background", "requirements for Ebsynth Utility Lite") - -update_transparent_background() - -if not launch.is_installed("IPython"): - launch.run_pip("install ipython", "requirements for Ebsynth Utility Lite") - -if not launch.is_installed("seaborn"): - launch.run_pip("install ""seaborn>=0.11.0""", "requirements for Ebsynth Utility Lite") - +import launch +import platform + +def update_transparent_background(): + from importlib.metadata import version as meta_version + from packaging import version + v = meta_version("transparent-background") + print("current transparent-background " + v) + if version.parse(v) < version.parse('1.2.3'): + launch.run_pip("install -U transparent-background", "update transparent-background version for Ebsynth Utility") + +# Check if user is running an M1/M2 device and, if so, install pyvirtualcam, which is required for updating the transparent_background package +# Note that we have to directly install from source because the prebuilt PyPl wheel does not support ARM64 machines such as M1/M2 Macs +if platform.system() == "Darwin" and platform.machine() == "arm64": + if not launch.is_installed("pyvirtualcam"): + launch.run_pip("install git+https://github.com/letmaik/pyvirtualcam", "requirements for Ebsynth Utility") + +if not launch.is_installed("transparent_background"): + launch.run_pip("install transparent-background", "requirements for Ebsynth Utility") + +update_transparent_background() + +if not launch.is_installed("IPython"): + launch.run_pip("install ipython", "requirements for Ebsynth Utility") + +if not launch.is_installed("seaborn"): + launch.run_pip("install ""seaborn>=0.11.0""", "requirements for Ebsynth Utility") + +if not launch.is_installed("color_matcher"): + launch.run_pip("install color-matcher", "requirements for Ebsynth Utility") + diff --git a/scripts/custom_script.py b/scripts/custom_script.py new file mode 100644 index 0000000..661d7e5 --- /dev/null +++ b/scripts/custom_script.py @@ -0,0 +1,1012 @@ +import modules.scripts as scripts +import gradio as gr +import os +import torch +import random +import time +import pprint +import shutil + +from modules.processing import process_images,Processed +from modules.paths import models_path +from modules.textual_inversion import autocrop +import modules.images +from modules import shared,deepbooru,masking +import cv2 +import copy +import numpy as np +from PIL import Image,ImageOps +import glob +import requests +import json +import re +from extensions.ebsynth_utility.calculator import CalcParser,ParseError + +def get_my_dir(): + if os.path.isdir("extensions/ebsynth_utility"): + return "extensions/ebsynth_utility" + return scripts.basedir() + +def x_ceiling(value, step): + return -(-value // step) * step + +def remove_pngs_in_dir(path): + if not os.path.isdir(path): + return + pngs = glob.glob( os.path.join(path, "*.png") ) + for png in pngs: + os.remove(png) + +def resize_img(img, w, h): + if img.shape[0] + img.shape[1] < h + w: + interpolation = interpolation=cv2.INTER_CUBIC + else: + interpolation = interpolation=cv2.INTER_AREA + + return cv2.resize(img, (w, h), interpolation=interpolation) + +def download_and_cache_models(dirname): + download_url = 'https://github.com/zymk9/yolov5_anime/blob/8b50add22dbd8224904221be3173390f56046794/weights/yolov5s_anime.pt?raw=true' + model_file_name = 'yolov5s_anime.pt' + + if not os.path.exists(dirname): + os.makedirs(dirname) + + cache_file = os.path.join(dirname, model_file_name) + if not os.path.exists(cache_file): + print(f"downloading face detection model from '{download_url}' to '{cache_file}'") + response = requests.get(download_url) + with open(cache_file, "wb") as f: + f.write(response.content) + + if os.path.exists(cache_file): + return cache_file + return None + +class Script(scripts.Script): + anime_face_detector = None + face_detector = None + face_merge_mask_filename = "face_crop_img2img_mask.png" + face_merge_mask_image = None + prompts_dir = "" + calc_parser = None + is_invert_mask = False + controlnet_weight = 0.5 + controlnet_weight_for_face = 0.5 + add_tag_replace_underscore = False + + +# The title of the script. This is what will be displayed in the dropdown menu. + def title(self): + return "ebsynth utility" + +# Determines when the script should be shown in the dropdown menu via the +# returned value. As an example: +# is_img2img is True if the current tab is img2img, and False if it is txt2img. +# Thus, return is_img2img to only show the script on the img2img tab. + + def show(self, is_img2img): + return is_img2img + +# How the script's is displayed in the UI. See https://gradio.app/docs/#components +# for the different UI components you can use and how to create them. +# Most UI components can return a value, such as a boolean for a checkbox. +# The returned values are passed to the run method as parameters. + + def ui(self, is_img2img): + with gr.Column(variant='panel'): + with gr.Column(): + project_dir = gr.Textbox(label='Project directory', lines=1) + generation_test = gr.Checkbox(False, label="Generation TEST!!(Ignore Project directory and use the image and mask specified in the main UI)") + + with gr.Accordion("Mask option"): + mask_mode = gr.Dropdown(choices=["Normal","Invert","None","Don't Override"], value="Normal" ,label="Mask Mode(Override img2img Mask mode)") + inpaint_area = gr.Dropdown(choices=["Whole picture","Only masked","Don't Override"], type = "index", value="Only masked" ,label="Inpaint Area(Override img2img Inpaint area)") + use_depth = gr.Checkbox(True, label="Use Depth Map If exists in /video_key_depth") + gr.HTML(value="

\ + See \ + [here] for depth map.\ +

") + + with gr.Accordion("ControlNet option"): + controlnet_weight = gr.Slider(minimum=0.0, maximum=2.0, step=0.01, value=0.5, label="Control Net Weight") + controlnet_weight_for_face = gr.Slider(minimum=0.0, maximum=2.0, step=0.01, value=0.5, label="Control Net Weight For Face") + use_preprocess_img = gr.Checkbox(True, label="Use Preprocess image If exists in /controlnet_preprocess") + gr.HTML(value="

\ + Please enable the following settings to use controlnet from this script.
\ + \ + Settings->ControlNet->Allow other script to control this extension\ + \ +

") + + with gr.Accordion("Loopback option"): + img2img_repeat_count = gr.Slider(minimum=1, maximum=30, step=1, value=1, label="Img2Img Repeat Count (Loop Back)") + inc_seed = gr.Slider(minimum=0, maximum=9999999, step=1, value=1, label="Add N to seed when repeating ") + + with gr.Accordion("Auto Tagging option"): + auto_tag_mode = gr.Dropdown(choices=["None","DeepDanbooru","CLIP"], value="None" ,label="Auto Tagging") + add_tag_to_head = gr.Checkbox(False, label="Add additional prompts to the head") + add_tag_replace_underscore = gr.Checkbox(False, label="Replace '_' with ' '(Does not affect the function to add tokens using add_token.txt.)") + gr.HTML(value="

\ + The results are stored in timestamp_prompts.txt.
\ + If you want to use the same tagging results the next time you run img2img, rename the file to prompts.txt
\ + Recommend enabling the following settings.
\ + \ + Settings->Interrogate Option->Interrogate: include ranks of model tags matches in results\ + \ +

") + + with gr.Accordion("Face Crop option"): + is_facecrop = gr.Checkbox(False, label="use Face Crop img2img") + + with gr.Row(): + face_detection_method = gr.Dropdown(choices=["YuNet","Yolov5_anime"], value="YuNet" ,label="Face Detection Method") + gr.HTML(value="

\ + If loading of the Yolov5_anime model fails, check\ + [this] solution.\ +

") + face_crop_resolution = gr.Slider(minimum=128, maximum=2048, step=1, value=512, label="Face Crop Resolution") + max_crop_size = gr.Slider(minimum=0, maximum=2048, step=1, value=1024, label="Max Crop Size") + face_denoising_strength = gr.Slider(minimum=0.00, maximum=1.00, step=0.01, value=0.5, label="Face Denoising Strength") + face_area_magnification = gr.Slider(minimum=1.00, maximum=10.00, step=0.01, value=1.5, label="Face Area Magnification ") + disable_facecrop_lpbk_last_time = gr.Checkbox(False, label="Disable at the last loopback time") + + with gr.Column(): + enable_face_prompt = gr.Checkbox(False, label="Enable Face Prompt") + face_prompt = gr.Textbox(label="Face Prompt", show_label=False, lines=2, + placeholder="Prompt for Face", + value = "face close up," + ) + + return [project_dir, generation_test, mask_mode, inpaint_area, use_depth, img2img_repeat_count, inc_seed, auto_tag_mode, add_tag_to_head, add_tag_replace_underscore, is_facecrop, face_detection_method, face_crop_resolution, max_crop_size, face_denoising_strength, face_area_magnification, enable_face_prompt, face_prompt, controlnet_weight, controlnet_weight_for_face, disable_facecrop_lpbk_last_time,use_preprocess_img] + + + def detect_face_from_img(self, img_array): + if not self.face_detector: + dnn_model_path = autocrop.download_and_cache_models(os.path.join(models_path, "opencv")) + self.face_detector = cv2.FaceDetectorYN.create(dnn_model_path, "", (0, 0)) + + self.face_detector.setInputSize((img_array.shape[1], img_array.shape[0])) + _, result = self.face_detector.detect(img_array) + return result + + def detect_anime_face_from_img(self, img_array): + import sys + + if not self.anime_face_detector: + if 'models' in sys.modules: + del sys.modules['models'] + + anime_model_path = download_and_cache_models(os.path.join(models_path, "yolov5_anime")) + + if not os.path.isfile(anime_model_path): + print( "WARNING!! " + anime_model_path + " not found.") + print( "use YuNet instead.") + return self.detect_face_from_img(img_array) + + self.anime_face_detector = torch.hub.load('ultralytics/yolov5', 'custom', path=anime_model_path) + + # warmup + test = np.zeros([512,512,3],dtype=np.uint8) + _ = self.anime_face_detector(test) + + result = self.anime_face_detector(img_array) + #models.common.Detections + faces = [] + for x_c, y_c, w, h, _, _ in result.xywh[0].tolist(): + faces.append( [ x_c - w/2 , y_c - h/2, w, h ] ) + + return faces + + def detect_face(self, img, mask, face_detection_method, max_crop_size): + img_array = np.array(img) + + # image without alpha + if img_array.shape[2] == 4: + img_array = img_array[:,:,:3] + + if mask is not None: + if self.is_invert_mask: + mask = ImageOps.invert(mask) + mask_array = np.array(mask)/255 + if mask_array.ndim == 2: + mask_array = mask_array[:, :, np.newaxis] + + if mask_array.shape[2] == 4: + mask_array = mask_array[:,:,:3] + + img_array = mask_array * img_array + img_array = img_array.astype(np.uint8) + + if face_detection_method == "YuNet": + faces = self.detect_face_from_img(img_array) + elif face_detection_method == "Yolov5_anime": + faces = self.detect_anime_face_from_img(img_array) + else: + faces = self.detect_face_from_img(img_array) + + if faces is None or len(faces) == 0: + return [] + + face_coords = [] + for face in faces: + x = int(face[0]) + y = int(face[1]) + w = int(face[2]) + h = int(face[3]) + if max(w,h) > max_crop_size: + print("ignore big face") + continue + if w == 0 or h == 0: + print("ignore w,h = 0 face") + continue + + face_coords.append( [ x/img_array.shape[1],y/img_array.shape[0],w/img_array.shape[1],h/img_array.shape[0]] ) + + return face_coords + + def get_mask(self): + def create_mask( output, x_rate, y_rate, k_size ): + img = np.zeros((512, 512, 3)) + img = cv2.ellipse(img, ((256, 256), (int(512 * x_rate), int(512 * y_rate)), 0), (255, 255, 255), thickness=-1) + img = cv2.GaussianBlur(img, (k_size, k_size), 0) + cv2.imwrite(output, img) + + if self.face_merge_mask_image is None: + mask_file_path = os.path.join( get_my_dir() , self.face_merge_mask_filename) + if not os.path.isfile(mask_file_path): + create_mask( mask_file_path, 0.9, 0.9, 91) + + m = cv2.imread( mask_file_path )[:,:,0] + m = m[:, :, np.newaxis] + self.face_merge_mask_image = m / 255 + + return self.face_merge_mask_image + + def face_img_crop(self, img, face_coords,face_area_magnification): + img_array = np.array(img) + face_imgs =[] + new_coords = [] + + for face in face_coords: + x = int(face[0] * img_array.shape[1]) + y = int(face[1] * img_array.shape[0]) + w = int(face[2] * img_array.shape[1]) + h = int(face[3] * img_array.shape[0]) + print([x,y,w,h]) + + cx = x + int(w/2) + cy = y + int(h/2) + + x = cx - int(w*face_area_magnification / 2) + x = x if x > 0 else 0 + w = cx + int(w*face_area_magnification / 2) - x + w = w if x+w < img.width else img.width - x + + y = cy - int(h*face_area_magnification / 2) + y = y if y > 0 else 0 + h = cy + int(h*face_area_magnification / 2) - y + h = h if y+h < img.height else img.height - y + + print([x,y,w,h]) + + face_imgs.append( img_array[y: y+h, x: x+w] ) + new_coords.append( [x,y,w,h] ) + + resized = [] + for face_img in face_imgs: + if face_img.shape[1] < face_img.shape[0]: + re_w = self.face_crop_resolution + re_h = int(x_ceiling( (self.face_crop_resolution / face_img.shape[1]) * face_img.shape[0] , 64)) + else: + re_w = int(x_ceiling( (self.face_crop_resolution / face_img.shape[0]) * face_img.shape[1] , 64)) + re_h = self.face_crop_resolution + + face_img = resize_img(face_img, re_w, re_h) + resized.append( Image.fromarray(face_img)) + + return resized, new_coords + + def face_crop_img2img(self, p, face_coords, face_denoising_strength, face_area_magnification, enable_face_prompt, face_prompt, controlnet_input_img, controlnet_input_face_imgs, preprocess_img_exist): + + def merge_face(img, face_img, face_coord, base_img_size, mask): + x_rate = img.width / base_img_size[0] + y_rate = img.height / base_img_size[1] + + img_array = np.array(img) + x = int(face_coord[0] * x_rate) + y = int(face_coord[1] * y_rate) + w = int(face_coord[2] * x_rate) + h = int(face_coord[3] * y_rate) + + face_array = np.array(face_img) + face_array = resize_img(face_array, w, h) + mask = resize_img(mask, w, h) + if mask.ndim == 2: + mask = mask[:, :, np.newaxis] + + bg = img_array[y: y+h, x: x+w] + img_array[y: y+h, x: x+w] = mask * face_array + (1-mask)*bg + + return Image.fromarray(img_array) + + base_img = p.init_images[0] + + base_img_size = (base_img.width, base_img.height) + + if face_coords is None or len(face_coords) == 0: + print("no face detected") + return process_images(p) + + print(face_coords) + face_imgs, new_coords = self.face_img_crop(base_img, face_coords, face_area_magnification) + + if not face_imgs: + return process_images(p) + + face_p = copy.copy(p) + + ### img2img base img + proc = self.process_images(p, controlnet_input_img, self.controlnet_weight, preprocess_img_exist) + print(proc.seed) + + ### img2img for each face + face_img2img_results = [] + + for face, coord, controlnet_input_face in zip(face_imgs, new_coords, controlnet_input_face_imgs): + # cv2.imwrite("scripts/face.png", np.array(face)[:, :, ::-1]) + face_p.init_images = [face] + face_p.width = face.width + face_p.height = face.height + face_p.denoising_strength = face_denoising_strength + + if enable_face_prompt: + face_p.prompt = face_prompt + else: + face_p.prompt = "close-up face ," + face_p.prompt + + if p.image_mask is not None: + x,y,w,h = coord + cropped_face_mask = Image.fromarray(np.array(p.image_mask)[y: y+h, x: x+w]) + face_p.image_mask = modules.images.resize_image(0, cropped_face_mask, face.width, face.height) + + face_proc = self.process_images(face_p, controlnet_input_face, self.controlnet_weight_for_face, preprocess_img_exist) + print(face_proc.seed) + + face_img2img_results.append((face_proc.images[0], coord)) + + ### merge faces + bg = proc.images[0] + mask = self.get_mask() + + for face_img, coord in face_img2img_results: + bg = merge_face(bg, face_img, coord, base_img_size, mask) + + proc.images[0] = bg + + return proc + + def get_depth_map(self, mask, depth_path ,img_basename, is_invert_mask): + depth_img_path = os.path.join( depth_path , img_basename ) + + depth = None + + if os.path.isfile( depth_img_path ): + depth = Image.open(depth_img_path) + else: + # try 00001-0000.png + os.path.splitext(img_basename)[0] + depth_img_path = os.path.join( depth_path , os.path.splitext(img_basename)[0] + "-0000.png" ) + if os.path.isfile( depth_img_path ): + depth = Image.open(depth_img_path) + + if depth: + if mask: + mask_array = np.array(mask) + depth_array = np.array(depth) + + if is_invert_mask == False: + depth_array[mask_array[:,:,0] == 0] = 0 + else: + depth_array[mask_array[:,:,0] != 0] = 0 + + depth = Image.fromarray(depth_array) + + tmp_path = os.path.join( depth_path , "tmp" ) + os.makedirs(tmp_path, exist_ok=True) + tmp_path = os.path.join( tmp_path , img_basename ) + depth_array = depth_array.astype(np.uint16) + cv2.imwrite(tmp_path, depth_array) + + mask = depth + + return depth!=None, mask + +### auto tagging + debug_count = 0 + + def get_masked_image(self, image, mask_image): + + if mask_image == None: + return image.convert("RGB") + + mask = mask_image.convert('L') + if self.is_invert_mask: + mask = ImageOps.invert(mask) + crop_region = masking.get_crop_region(np.array(mask), 0) +# crop_region = masking.expand_crop_region(crop_region, self.width, self.height, mask.width, mask.height) +# x1, y1, x2, y2 = crop_region + image = image.crop(crop_region).convert("RGB") + mask = mask.crop(crop_region) + + base_img = Image.new("RGB", image.size, (255, 190, 200)) + + image = Image.composite( image, base_img, mask ) + +# image.save("scripts/get_masked_image_test_"+ str(self.debug_count) + ".png") +# self.debug_count += 1 + + return image + + def interrogate_deepdanbooru(self, imgs, masks): + prompts_dict = {} + cause_err = False + + try: + deepbooru.model.start() + + for img,mask in zip(imgs,masks): + key = os.path.basename(img) + print(key + " interrogate deepdanbooru") + + image = Image.open(img) + mask_image = Image.open(mask) if mask else None + image = self.get_masked_image(image, mask_image) + + prompt = deepbooru.model.tag_multi(image) + + prompts_dict[key] = prompt + except Exception as e: + import traceback + traceback.print_exc() + print(e) + cause_err = True + finally: + deepbooru.model.stop() + if cause_err: + print("Exception occurred during auto-tagging(deepdanbooru)") + return Processed() + + return prompts_dict + + + def interrogate_clip(self, imgs, masks): + from modules import devices, shared, lowvram, paths + import importlib + import models + + caption_list = [] + prompts_dict = {} + cause_err = False + + try: + if shared.cmd_opts.lowvram or shared.cmd_opts.medvram: + lowvram.send_everything_to_cpu() + devices.torch_gc() + + with paths.Prioritize("BLIP"): + importlib.reload(models) + shared.interrogator.load() + + for img,mask in zip(imgs,masks): + key = os.path.basename(img) + print(key + " generate caption") + + image = Image.open(img) + mask_image = Image.open(mask) if mask else None + image = self.get_masked_image(image, mask_image) + + caption = shared.interrogator.generate_caption(image) + caption_list.append(caption) + + shared.interrogator.send_blip_to_ram() + devices.torch_gc() + + for img,mask,caption in zip(imgs,masks,caption_list): + key = os.path.basename(img) + print(key + " interrogate clip") + + image = Image.open(img) + mask_image = Image.open(mask) if mask else None + image = self.get_masked_image(image, mask_image) + + clip_image = shared.interrogator.clip_preprocess(image).unsqueeze(0).type(shared.interrogator.dtype).to(devices.device_interrogate) + + res = "" + + with torch.no_grad(), devices.autocast(): + image_features = shared.interrogator.clip_model.encode_image(clip_image).type(shared.interrogator.dtype) + image_features /= image_features.norm(dim=-1, keepdim=True) + + for name, topn, items in shared.interrogator.categories(): + matches = shared.interrogator.rank(image_features, items, top_count=topn) + for match, score in matches: + if shared.opts.interrogate_return_ranks: + res += f", ({match}:{score/100:.3f})" + else: + res += ", " + match + + prompts_dict[key] = (caption + res) + + except Exception as e: + import traceback + traceback.print_exc() + print(e) + cause_err = True + finally: + shared.interrogator.unload() + if cause_err: + print("Exception occurred during auto-tagging(blip/clip)") + return Processed() + + return prompts_dict + + + def remove_reserved_token(self, token_list): + reserved_list = ["pink_background","simple_background","pink","pink_theme"] + + result_list = [] + + head_token = token_list[0] + + if head_token[2] == "normal": + head_token_str = head_token[0].replace('pink background', '') + token_list[0] = (head_token_str, head_token[1], head_token[2]) + + for token in token_list: + if token[0] in reserved_list: + continue + result_list.append(token) + + return result_list + + def remove_blacklisted_token(self, token_list): + black_list_path = os.path.join(self.prompts_dir, "blacklist.txt") + if not os.path.isfile(black_list_path): + print(black_list_path + " not found.") + return token_list + + with open(black_list_path) as f: + black_list = [s.strip() for s in f.readlines()] + + result_list = [] + + for token in token_list: + if token[0] in black_list: + continue + result_list.append(token) + + token_list = result_list + + return token_list + + def add_token(self, token_list): + add_list_path = os.path.join(self.prompts_dir, "add_token.txt") + if not os.path.isfile(add_list_path): + print(add_list_path + " not found.") + + if self.add_tag_replace_underscore: + token_list = [ (x[0].replace("_"," "), x[1], x[2]) for x in token_list ] + + return token_list + + if not self.calc_parser: + self.calc_parser = CalcParser() + + with open(add_list_path) as f: + add_list = json.load(f) + ''' + [ + { + "target":"test_token", + "min_score":0.8, + "token": ["lora_name_A", "0.5"], + "type":"lora" + }, + { + "target":"test_token", + "min_score":0.5, + "token": ["bbbb", "score - 0.1"], + "type":"normal" + }, + { + "target":"test_token2", + "min_score":0.8, + "token": ["hypernet_name_A", "score"], + "type":"hypernet" + }, + { + "target":"test_token3", + "min_score":0.0, + "token": ["dddd", "score"], + "type":"normal" + } + ] + ''' + result_list = [] + + for token in token_list: + for add_item in add_list: + if token[0] == add_item["target"]: + if token[1] > add_item["min_score"]: + # hit + formula = str(add_item["token"][1]) + formula = formula.replace("score",str(token[1])) + print('Input: %s' % str(add_item["token"][1])) + + try: + score = self.calc_parser.parse(formula) + score = round(score, 3) + except (ParseError, ZeroDivisionError) as e: + print('Input: %s' % str(add_item["token"][1])) + print('Error: %s' % e) + print("ignore this token") + continue + + print("score = " + str(score)) + result_list.append( ( add_item["token"][0], score, add_item["type"] ) ) + + if self.add_tag_replace_underscore: + token_list = [ (x[0].replace("_"," "), x[1], x[2]) for x in token_list ] + + token_list = token_list + result_list + + return token_list + + def create_prompts_dict(self, imgs, masks, auto_tag_mode): + prompts_dict = {} + + if auto_tag_mode == "DeepDanbooru": + raw_dict = self.interrogate_deepdanbooru(imgs, masks) + elif auto_tag_mode == "CLIP": + raw_dict = self.interrogate_clip(imgs, masks) + + repatter = re.compile(r'\((.+)\:([0-9\.]+)\)') + + for key, value_str in raw_dict.items(): + value_list = [x.strip() for x in value_str.split(',')] + + value = [] + for v in value_list: + m = repatter.fullmatch(v) + if m: + value.append((m.group(1), float(m.group(2)), "normal")) + else: + value.append((v, 1, "no_score")) + +# print(value) + value = self.remove_reserved_token(value) +# print(value) + value = self.remove_blacklisted_token(value) +# print(value) + value = self.add_token(value) +# print(value) + + def create_token_str(x): + print(x) + if x[2] == "no_score": + return x[0] + elif x[2] == "lora": + return "" + elif x[2] == "hypernet": + return "" + else: + return "(" + x[0] + ":" + str(x[1]) + ")" + + value_list = [create_token_str(x) for x in value] + value = ",".join(value_list) + + prompts_dict[key] = value + + return prompts_dict + + def load_prompts_dict(self, imgs, default_token): + prompts_path = os.path.join(self.prompts_dir, "prompts.txt") + if not os.path.isfile(prompts_path): + print(prompts_path + " not found.") + return {} + + prompts_dict = {} + + print(prompts_path + " found!!") + print("skip auto tagging.") + + with open(prompts_path) as f: + raw_dict = json.load(f) + prev_value = default_token + for img in imgs: + key = os.path.basename(img) + + if key in raw_dict: + prompts_dict[key] = raw_dict[key] + prev_value = raw_dict[key] + else: + prompts_dict[key] = prev_value + + return prompts_dict + + def process_images(self, p, input_img, controlnet_weight, input_img_is_preprocessed): + p.control_net_input_image = input_img + p.control_net_weight = controlnet_weight + if input_img_is_preprocessed: + p.control_net_module = "none" + return process_images(p) + +# This is where the additional processing is implemented. The parameters include +# self, the model object "p" (a StableDiffusionProcessing class, see +# processing.py), and the parameters returned by the ui method. +# Custom functions can be defined here, and additional libraries can be imported +# to be used in processing. The return value should be a Processed object, which is +# what is returned by the process_images method. + def run(self, p, project_dir, generation_test, mask_mode, inpaint_area, use_depth, img2img_repeat_count, inc_seed, auto_tag_mode, add_tag_to_head, add_tag_replace_underscore, is_facecrop, face_detection_method, face_crop_resolution, max_crop_size, face_denoising_strength, face_area_magnification, enable_face_prompt, face_prompt, controlnet_weight, controlnet_weight_for_face, disable_facecrop_lpbk_last_time, use_preprocess_img): + args = locals() + + if generation_test: + print("generation_test") + test_proj_dir = os.path.join( get_my_dir() , "generation_test_proj") + os.makedirs(test_proj_dir, exist_ok=True) + test_video_key_path = os.path.join( test_proj_dir , "video_key") + os.makedirs(test_video_key_path, exist_ok=True) + test_video_mask_path = os.path.join( test_proj_dir , "video_mask") + os.makedirs(test_video_mask_path, exist_ok=True) + + controlnet_input_path = os.path.join(test_proj_dir, "controlnet_input") + if os.path.isdir(controlnet_input_path): + shutil.rmtree(controlnet_input_path) + + remove_pngs_in_dir(test_video_key_path) + remove_pngs_in_dir(test_video_mask_path) + + test_base_img = p.init_images[0] + test_mask = p.image_mask + + if test_base_img: + test_base_img.save( os.path.join( test_video_key_path , "00001.png") ) + if test_mask: + test_mask.save( os.path.join( test_video_mask_path , "00001.png") ) + + project_dir = test_proj_dir + else: + if not os.path.isdir(project_dir): + print("project_dir not found") + return Processed() + + self.controlnet_weight = controlnet_weight + self.controlnet_weight_for_face = controlnet_weight_for_face + + self.add_tag_replace_underscore = add_tag_replace_underscore + self.face_crop_resolution = face_crop_resolution + + if p.seed == -1: + p.seed = int(random.randrange(4294967294)) + + if mask_mode == "Normal": + p.inpainting_mask_invert = 0 + elif mask_mode == "Invert": + p.inpainting_mask_invert = 1 + + if inpaint_area in (0,1): #"Whole picture","Only masked" + p.inpaint_full_res = inpaint_area + + is_invert_mask = False + if mask_mode == "Invert": + is_invert_mask = True + + inv_path = os.path.join(project_dir, "inv") + if not os.path.isdir(inv_path): + print("project_dir/inv not found") + return Processed() + + org_key_path = os.path.join(inv_path, "video_key") + img2img_key_path = os.path.join(inv_path, "img2img_key") + depth_path = os.path.join(inv_path, "video_key_depth") + + preprocess_path = os.path.join(inv_path, "controlnet_preprocess") + + controlnet_input_path = os.path.join(inv_path, "controlnet_input") + + self.prompts_dir = inv_path + self.is_invert_mask = True + else: + org_key_path = os.path.join(project_dir, "video_key") + img2img_key_path = os.path.join(project_dir, "img2img_key") + depth_path = os.path.join(project_dir, "video_key_depth") + + preprocess_path = os.path.join(project_dir, "controlnet_preprocess") + + controlnet_input_path = os.path.join(project_dir, "controlnet_input") + + self.prompts_dir = project_dir + self.is_invert_mask = False + + frame_mask_path = os.path.join(project_dir, "video_mask") + + if not use_depth: + depth_path = None + + if not os.path.isdir(org_key_path): + print(org_key_path + " not found") + print("Generate key frames first." if is_invert_mask == False else \ + "Generate key frames first.(with [Ebsynth Utility] Tab -> [configuration] -> [etc]-> [Mask Mode] = Invert setting)") + return Processed() + + if not os.path.isdir(controlnet_input_path): + print(controlnet_input_path + " not found") + print("copy {0} -> {1}".format(org_key_path,controlnet_input_path)) + + os.makedirs(controlnet_input_path, exist_ok=True) + + imgs = glob.glob( os.path.join(org_key_path ,"*.png") ) + for img in imgs: + img_basename = os.path.basename(img) + shutil.copy( img , os.path.join(controlnet_input_path, img_basename) ) + + remove_pngs_in_dir(img2img_key_path) + os.makedirs(img2img_key_path, exist_ok=True) + + + def get_mask_of_img(img): + img_basename = os.path.basename(img) + + if mask_mode != "None": + mask_path = os.path.join( frame_mask_path , img_basename ) + if os.path.isfile( mask_path ): + return mask_path + return "" + + def get_pair_of_img(img, target_dir): + img_basename = os.path.basename(img) + + pair_path = os.path.join( target_dir , img_basename ) + if os.path.isfile( pair_path ): + return pair_path + print("!!! pair of "+ img + " not in " + target_dir) + return "" + + def get_controlnet_input_img(img): + pair_img = get_pair_of_img(img, controlnet_input_path) + if not pair_img: + pair_img = get_pair_of_img(img, org_key_path) + return pair_img + + imgs = glob.glob( os.path.join(org_key_path ,"*.png") ) + masks = [ get_mask_of_img(i) for i in imgs ] + controlnet_input_imgs = [ get_controlnet_input_img(i) for i in imgs ] + + for mask in masks: + m = cv2.imread(mask) if mask else None + if m is not None: + if m.max() == 0: + print("{0} blank mask found".format(mask)) + if m.ndim == 2: + m[0,0] = 255 + else: + m = m[:,:,:3] + m[0,0,0:3] = 255 + cv2.imwrite(mask, m) + + ###################### + # face crop + face_coords_dict={} + for img,mask in zip(imgs,masks): + face_detected = False + if is_facecrop: + image = Image.open(img) + mask_image = Image.open(mask) if mask else None + face_coords = self.detect_face(image, mask_image, face_detection_method, max_crop_size) + if face_coords is None or len(face_coords) == 0: + print("no face detected") + else: + print("face detected") + face_detected = True + + key = os.path.basename(img) + face_coords_dict[key] = face_coords if face_detected else [] + + with open( os.path.join( project_dir if is_invert_mask == False else inv_path,"faces.txt" ), "w") as f: + f.write(json.dumps(face_coords_dict,indent=4)) + + ###################### + # prompts + prompts_dict = self.load_prompts_dict(imgs, p.prompt) + + if not prompts_dict: + if auto_tag_mode != "None": + prompts_dict = self.create_prompts_dict(imgs, masks, auto_tag_mode) + + for key, value in prompts_dict.items(): + prompts_dict[key] = (value + "," + p.prompt) if add_tag_to_head else (p.prompt + "," + value) + + else: + for img in imgs: + key = os.path.basename(img) + prompts_dict[key] = p.prompt + + with open( os.path.join( project_dir if is_invert_mask == False else inv_path, time.strftime("%Y%m%d-%H%M%S_") + "prompts.txt" ), "w") as f: + f.write(json.dumps(prompts_dict,indent=4)) + + + ###################### + # img2img + for img, mask, controlnet_input_img, face_coords, prompts in zip(imgs, masks, controlnet_input_imgs, face_coords_dict.values(), prompts_dict.values()): + + # Generation cancelled. + if shared.state.interrupted: + print("Generation cancelled.") + break + + image = Image.open(img) + mask_image = Image.open(mask) if mask else None + + img_basename = os.path.basename(img) + + _p = copy.copy(p) + + _p.init_images=[image] + _p.image_mask = mask_image + _p.prompt = prompts + resized_mask = None + + repeat_count = img2img_repeat_count + + if mask_mode != "None" or use_depth: + if use_depth: + depth_found, _p.image_mask = self.get_depth_map( mask_image, depth_path ,img_basename, is_invert_mask ) + mask_image = _p.image_mask + if depth_found: + _p.inpainting_mask_invert = 0 + + preprocess_img_exist = False + controlnet_input_base_img = Image.open(controlnet_input_img) if controlnet_input_img else None + + if use_preprocess_img: + preprocess_img = os.path.join(preprocess_path, img_basename) + if os.path.isfile( preprocess_img ): + controlnet_input_base_img = Image.open(preprocess_img) + preprocess_img_exist = True + + if face_coords: + controlnet_input_face_imgs, _ = self.face_img_crop(controlnet_input_base_img, face_coords, face_area_magnification) + + while repeat_count > 0: + + if disable_facecrop_lpbk_last_time: + if img2img_repeat_count > 1: + if repeat_count == 1: + face_coords = None + + if face_coords: + proc = self.face_crop_img2img(_p, face_coords, face_denoising_strength, face_area_magnification, enable_face_prompt, face_prompt, controlnet_input_base_img, controlnet_input_face_imgs, preprocess_img_exist) + else: + proc = self.process_images(_p, controlnet_input_base_img, self.controlnet_weight, preprocess_img_exist) + print(proc.seed) + + repeat_count -= 1 + + if repeat_count > 0: + _p.init_images=[proc.images[0]] + + if mask_image is not None and resized_mask is None: + resized_mask = resize_img(np.array(mask_image) , proc.images[0].width, proc.images[0].height) + resized_mask = Image.fromarray(resized_mask) + _p.image_mask = resized_mask + _p.seed += inc_seed + + proc.images[0].save( os.path.join( img2img_key_path , img_basename ) ) + + with open( os.path.join( project_dir if is_invert_mask == False else inv_path,"param.txt" ), "w") as f: + f.write(pprint.pformat(proc.info)) + with open( os.path.join( project_dir if is_invert_mask == False else inv_path ,"args.txt" ), "w") as f: + f.write(pprint.pformat(args)) + + return proc diff --git a/scripts/ui.py b/scripts/ui.py index 85de1b4..887f48d 100644 --- a/scripts/ui.py +++ b/scripts/ui.py @@ -1,171 +1,196 @@ - -import gradio as gr - -from ebsynth_utility import ebsynth_utility_process -from modules import script_callbacks -from modules.call_queue import wrap_gradio_gpu_call - -def on_ui_tabs(): - - with gr.Blocks(analytics_enabled=False) as ebs_interface: - with gr.Row().style(equal_height=False): - with gr.Column(variant='panel'): - - with gr.Row(): - with gr.Tabs(elem_id="ebs_settings"): - with gr.TabItem('project setting', elem_id='ebs_project_setting'): - project_dir = gr.Textbox(label='Project directory', lines=1) - original_movie_path = gr.Textbox(label='Original Movie Path', lines=1) - - org_video = gr.Video(interactive=True, mirror_webcam=False) - def fn_upload_org_video(video): - return video - org_video.upload(fn_upload_org_video, org_video, original_movie_path) - gr.HTML(value="

\ - If you have trouble entering the video path manually, you can also use drag and drop. \ -

") - - with gr.TabItem('configuration', elem_id='ebs_configuration'): - with gr.Tabs(elem_id="ebs_configuration_tab"): - with gr.TabItem(label="stage 1",elem_id='ebs_configuration_tab1'): - with gr.Row(): - frame_width = gr.Number(value=-1, label="Frame Width", precision=0, interactive=True) - frame_height = gr.Number(value=-1, label="Frame Height", precision=0, interactive=True) - gr.HTML(value="

\ - -1 means that it is calculated automatically. If both are -1, the size will be the same as the source size. \ -

") - - st1_masking_method_index = gr.Radio(label='Masking Method', choices=["transparent-background","clipseg","transparent-background AND clipseg"], value="transparent-background", type="index") - - with gr.Accordion(label="transparent-background options"): - st1_mask_threshold = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='Mask Threshold', value=0.0) - - # https://pypi.org/project/transparent-background/ - gr.HTML(value="

\ - configuration for \ - [transparent-background]\ -

") - tb_use_fast_mode = gr.Checkbox(label="Use Fast Mode(It will be faster, but the quality of the mask will be lower.)", value=False) - tb_use_jit = gr.Checkbox(label="Use Jit", value=False) - - with gr.Accordion(label="clipseg options"): - clipseg_mask_prompt = gr.Textbox(label='Mask Target (e.g., girl, cats)', lines=1) - clipseg_exclude_prompt = gr.Textbox(label='Exclude Target (e.g., finger, book)', lines=1) - clipseg_mask_threshold = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='Mask Threshold', value=0.4) - clipseg_mask_blur_size = gr.Slider(minimum=0, maximum=150, step=1, label='Mask Blur Kernel Size(MedianBlur)', value=11) - clipseg_mask_blur_size2 = gr.Slider(minimum=0, maximum=150, step=1, label='Mask Blur Kernel Size(GaussianBlur)', value=11) - - with gr.TabItem(label="stage 2", elem_id='ebs_configuration_tab2'): - key_min_gap = gr.Slider(minimum=0, maximum=500, step=1, label='Minimum keyframe gap', value=10) - key_max_gap = gr.Slider(minimum=0, maximum=1000, step=1, label='Maximum keyframe gap', value=300) - key_th = gr.Slider(minimum=0.0, maximum=100.0, step=0.1, label='Threshold of delta frame edge', value=8.5) - key_add_last_frame = gr.Checkbox(label="Add last frame to keyframes", value=True) - - with gr.TabItem(label="stage 7", elem_id='ebs_configuration_tab7'): - blend_rate = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='Crossfade blend rate', value=1.0) - export_type = gr.Dropdown(choices=["mp4","webm","gif","rawvideo"], value="mp4" ,label="Export type") - - with gr.TabItem(label="stage 8", elem_id='ebs_configuration_tab8'): - bg_src = gr.Textbox(label='Background source(mp4 or directory containing images)', lines=1) - bg_type = gr.Dropdown(choices=["Fit video length","Loop"], value="Fit video length" ,label="Background type") - mask_blur_size = gr.Slider(minimum=0, maximum=150, step=1, label='Mask Blur Kernel Size', value=5) - mask_threshold = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='Mask Threshold', value=0.0) - #is_transparent = gr.Checkbox(label="Is Transparent", value=True, visible = False) - fg_transparency = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='Foreground Transparency', value=0.0) - - with gr.TabItem(label="etc", elem_id='ebs_configuration_tab_etc'): - mask_mode = gr.Dropdown(choices=["Normal","Invert","None"], value="Normal" ,label="Mask Mode") - with gr.TabItem('info', elem_id='ebs_info'): - gr.HTML(value="

\ - The process of creating a video can be divided into the following stages.
\ - (Stage 3, 4, and 6 only show a guide and do nothing actual processing.)

\ - stage 1
\ - Extract frames from the original video.
\ - Generate a mask image.

\ - stage 2
\ - Select keyframes to be given to ebsynth.

\ - stage 3
\ - img2img keyframes.

\ - stage 4
\ - and upscale to the size of the original video.

\ - stage 5
\ - Rename keyframes.
\ - Generate .ebs file.(ebsynth project file)

\ - stage 6
\ - Running ebsynth.(on your self)
\ - Open the generated .ebs under project directory and press [Run All] button.
\ - If ""out-*"" directory already exists in the Project directory, delete it manually before executing.
\ - If multiple .ebs files are generated, run them all.

\ - stage 7
\ - Concatenate each frame while crossfading.
\ - Composite audio files extracted from the original video onto the concatenated video.

\ - stage 8
\ - This is an extra stage.
\ - You can put any image or images or video you like in the background.
\ - You can specify in this field -> [Ebsynth Utility]->[configuration]->[stage 8]->[Background source]
\ - If you have already created a background video in Invert Mask Mode([Ebsynth Utility]->[configuration]->[etc]->[Mask Mode]),
\ - You can specify \"path_to_project_dir/inv/crossfade_tmp\".
\ -

") - - with gr.Column(variant='panel'): - with gr.Column(scale=1): - with gr.Row(): - stage_index = gr.Radio(label='Process Stage', choices=["stage 1","stage 2","stage 3","stage 4","stage 5","stage 6","stage 7","stage 8"], value="stage 1", type="index", elem_id='ebs_stages') - - with gr.Row(): - generate_btn = gr.Button('Generate', elem_id="ebs_generate_btn", variant='primary') - - with gr.Group(): - debug_info = gr.HTML(elem_id="ebs_info_area", value=".") - - with gr.Column(scale=2): - html_info = gr.HTML() - - ebs_args = dict( - fn=wrap_gradio_gpu_call(ebsynth_utility_process), - inputs=[ - stage_index, - - project_dir, - original_movie_path, - - frame_width, - frame_height, - st1_masking_method_index, - st1_mask_threshold, - tb_use_fast_mode, - tb_use_jit, - clipseg_mask_prompt, - clipseg_exclude_prompt, - clipseg_mask_threshold, - clipseg_mask_blur_size, - clipseg_mask_blur_size2, - - key_min_gap, - key_max_gap, - key_th, - key_add_last_frame, - - blend_rate, - export_type, - - bg_src, - bg_type, - mask_blur_size, - mask_threshold, - fg_transparency, - - mask_mode, - - ], - outputs=[ - debug_info, - html_info, - ], - show_progress=False, - ) - generate_btn.click(**ebs_args) - - return (ebs_interface, "Ebsynth Utility Lite", "ebs_interface"), - -script_callbacks.on_ui_tabs(on_ui_tabs) + +import gradio as gr + +from ebsynth_utility import ebsynth_utility_process +from modules import script_callbacks +from modules.call_queue import wrap_gradio_gpu_call + +def on_ui_tabs(): + + with gr.Blocks(analytics_enabled=False) as ebs_interface: + with gr.Row(equal_height=False): + with gr.Column(variant='panel'): + + with gr.Row(): + with gr.Tabs(elem_id="ebs_settings"): + with gr.TabItem('project setting', elem_id='ebs_project_setting'): + project_dir = gr.Textbox(label='Project directory', lines=1) + original_movie_path = gr.Textbox(label='Original Movie Path', lines=1) + + org_video = gr.Video(interactive=True, mirror_webcam=False) + def fn_upload_org_video(video): + return video + org_video.upload(fn_upload_org_video, org_video, original_movie_path) + gr.HTML(value="

\ + If you have trouble entering the video path manually, you can also use drag and drop.For large videos, please enter the path manually. \ +

") + + with gr.TabItem('configuration', elem_id='ebs_configuration'): + with gr.Tabs(elem_id="ebs_configuration_tab"): + with gr.TabItem(label="stage 1",elem_id='ebs_configuration_tab1'): + with gr.Row(): + frame_width = gr.Number(value=-1, label="Frame Width", precision=0, interactive=True) + frame_height = gr.Number(value=-1, label="Frame Height", precision=0, interactive=True) + gr.HTML(value="

\ + -1 means that it is calculated automatically. If both are -1, the size will be the same as the source size. \ +

") + + st1_masking_method_index = gr.Radio(label='Masking Method', choices=["transparent-background","clipseg","transparent-background AND clipseg"], value="transparent-background", type="index") + + with gr.Accordion(label="transparent-background options"): + st1_mask_threshold = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='Mask Threshold', value=0.0) + + # https://pypi.org/project/transparent-background/ + gr.HTML(value="

\ + configuration for \ + [transparent-background]\ +

") + tb_use_fast_mode = gr.Checkbox(label="Use Fast Mode(It will be faster, but the quality of the mask will be lower.)", value=False) + tb_use_jit = gr.Checkbox(label="Use Jit", value=False) + + with gr.Accordion(label="clipseg options"): + clipseg_mask_prompt = gr.Textbox(label='Mask Target (e.g., girl, cats)', lines=1) + clipseg_exclude_prompt = gr.Textbox(label='Exclude Target (e.g., finger, book)', lines=1) + clipseg_mask_threshold = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='Mask Threshold', value=0.4) + clipseg_mask_blur_size = gr.Slider(minimum=0, maximum=150, step=1, label='Mask Blur Kernel Size(MedianBlur)', value=11) + clipseg_mask_blur_size2 = gr.Slider(minimum=0, maximum=150, step=1, label='Mask Blur Kernel Size(GaussianBlur)', value=11) + + with gr.TabItem(label="stage 2", elem_id='ebs_configuration_tab2'): + key_min_gap = gr.Slider(minimum=0, maximum=500, step=1, label='Minimum keyframe gap', value=10) + key_max_gap = gr.Slider(minimum=0, maximum=1000, step=1, label='Maximum keyframe gap', value=300) + key_th = gr.Slider(minimum=0.0, maximum=100.0, step=0.1, label='Threshold of delta frame edge', value=8.5) + key_add_last_frame = gr.Checkbox(label="Add last frame to keyframes", value=True) + + with gr.TabItem(label="stage 3.5", elem_id='ebs_configuration_tab3_5'): + gr.HTML(value="

\ + [color-matcher]\ +

") + + color_matcher_method = gr.Radio(label='Color Transfer Method', choices=['default', 'hm', 'reinhard', 'mvgd', 'mkl', 'hm-mvgd-hm', 'hm-mkl-hm'], value="hm-mkl-hm", type="value") + color_matcher_ref_type = gr.Radio(label='Color Matcher Ref Image Type', choices=['original video frame', 'first frame of img2img result'], value="original video frame", type="index") + gr.HTML(value="

\ + If an image is specified below, it will be used with highest priority.\ +

") + color_matcher_ref_image = gr.Image(label="Color Matcher Ref Image", source='upload', mirror_webcam=False, type='pil') + st3_5_use_mask = gr.Checkbox(label="Apply mask to the result", value=True) + st3_5_use_mask_ref = gr.Checkbox(label="Apply mask to the Ref Image", value=False) + st3_5_use_mask_org = gr.Checkbox(label="Apply mask to original image", value=False) + #st3_5_number_of_itr = gr.Slider(minimum=1, maximum=10, step=1, label='Number of iterations', value=1) + + with gr.TabItem(label="stage 7", elem_id='ebs_configuration_tab7'): + blend_rate = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='Crossfade blend rate', value=1.0) + export_type = gr.Dropdown(choices=["mp4","webm","gif","rawvideo"], value="mp4" ,label="Export type") + + with gr.TabItem(label="stage 8", elem_id='ebs_configuration_tab8'): + bg_src = gr.Textbox(label='Background source(mp4 or directory containing images)', lines=1) + bg_type = gr.Dropdown(choices=["Fit video length","Loop"], value="Fit video length" ,label="Background type") + mask_blur_size = gr.Slider(minimum=0, maximum=150, step=1, label='Mask Blur Kernel Size', value=5) + mask_threshold = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='Mask Threshold', value=0.0) + #is_transparent = gr.Checkbox(label="Is Transparent", value=True, visible = False) + fg_transparency = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='Foreground Transparency', value=0.0) + + with gr.TabItem(label="etc", elem_id='ebs_configuration_tab_etc'): + mask_mode = gr.Dropdown(choices=["Normal","Invert","None"], value="Normal" ,label="Mask Mode") + with gr.TabItem('info', elem_id='ebs_info'): + gr.HTML(value="

\ + The process of creating a video can be divided into the following stages.
\ + (Stage 3, 4, and 6 only show a guide and do nothing actual processing.)

\ + stage 1
\ + Extract frames from the original video.
\ + Generate a mask image.

\ + stage 2
\ + Select keyframes to be given to ebsynth.

\ + stage 3
\ + img2img keyframes.

\ + stage 3.5
\ + (this is optional. Perform color correction on the img2img results and expect flickering to decrease. Or, you can simply change the color tone from the generated result.)

\ + stage 4
\ + and upscale to the size of the original video.

\ + stage 5
\ + Rename keyframes.
\ + Generate .ebs file.(ebsynth project file)

\ + stage 6
\ + Running ebsynth.(on your self)
\ + Open the generated .ebs under project directory and press [Run All] button.
\ + If ""out-*"" directory already exists in the Project directory, delete it manually before executing.
\ + If multiple .ebs files are generated, run them all.

\ + stage 7
\ + Concatenate each frame while crossfading.
\ + Composite audio files extracted from the original video onto the concatenated video.

\ + stage 8
\ + This is an extra stage.
\ + You can put any image or images or video you like in the background.
\ + You can specify in this field -> [Ebsynth Utility]->[configuration]->[stage 8]->[Background source]
\ + If you have already created a background video in Invert Mask Mode([Ebsynth Utility]->[configuration]->[etc]->[Mask Mode]),
\ + You can specify \"path_to_project_dir/inv/crossfade_tmp\".
\ +

") + + with gr.Column(variant='panel'): + with gr.Column(scale=1): + with gr.Row(): + stage_index = gr.Radio(label='Process Stage', choices=["stage 1","stage 2","stage 3","stage 3.5","stage 4","stage 5","stage 6","stage 7","stage 8"], value="stage 1", type="index", elem_id='ebs_stages') + + with gr.Row(): + generate_btn = gr.Button('Generate', elem_id="ebs_generate_btn", variant='primary') + + with gr.Group(): + debug_info = gr.HTML(elem_id="ebs_info_area", value=".") + + with gr.Column(scale=2): + html_info = gr.HTML() + + ebs_args = dict( + fn=wrap_gradio_gpu_call(ebsynth_utility_process), + inputs=[ + stage_index, + + project_dir, + original_movie_path, + + frame_width, + frame_height, + st1_masking_method_index, + st1_mask_threshold, + tb_use_fast_mode, + tb_use_jit, + clipseg_mask_prompt, + clipseg_exclude_prompt, + clipseg_mask_threshold, + clipseg_mask_blur_size, + clipseg_mask_blur_size2, + + key_min_gap, + key_max_gap, + key_th, + key_add_last_frame, + + color_matcher_method, + st3_5_use_mask, + st3_5_use_mask_ref, + st3_5_use_mask_org, + color_matcher_ref_type, + color_matcher_ref_image, + + blend_rate, + export_type, + + bg_src, + bg_type, + mask_blur_size, + mask_threshold, + fg_transparency, + + mask_mode, + + ], + outputs=[ + debug_info, + html_info, + ], + show_progress=False, + ) + generate_btn.click(**ebs_args) + + return (ebs_interface, "Ebsynth Utility", "ebs_interface"), + +script_callbacks.on_ui_tabs(on_ui_tabs) diff --git a/stage3_5.py b/stage3_5.py new file mode 100644 index 0000000..26607a7 --- /dev/null +++ b/stage3_5.py @@ -0,0 +1,178 @@ +import cv2 +import os +import glob +import shutil +import numpy as np +from PIL import Image + +from color_matcher import ColorMatcher +from color_matcher.normalizer import Normalizer + +def resize_img(img, w, h): + if img.shape[0] + img.shape[1] < h + w: + interpolation = interpolation=cv2.INTER_CUBIC + else: + interpolation = interpolation=cv2.INTER_AREA + + return cv2.resize(img, (w, h), interpolation=interpolation) + +def get_pair_of_img(img_path, target_dir): + img_basename = os.path.basename(img_path) + target_path = os.path.join( target_dir , img_basename ) + return target_path if os.path.isfile( target_path ) else None + +def remove_pngs_in_dir(path): + if not os.path.isdir(path): + return + + pngs = glob.glob( os.path.join(path, "*.png") ) + for png in pngs: + os.remove(png) + +def get_pair_of_img(img, target_dir): + img_basename = os.path.basename(img) + + pair_path = os.path.join( target_dir , img_basename ) + if os.path.isfile( pair_path ): + return pair_path + print("!!! pair of "+ img + " not in " + target_dir) + return "" + +def get_mask_array(mask_path): + if not mask_path: + return None + mask_array = np.asarray(Image.open( mask_path )) + if mask_array.ndim == 2: + mask_array = mask_array[:, :, np.newaxis] + mask_array = mask_array[:,:,:1] + mask_array = mask_array/255 + return mask_array + +def color_match(imgs, ref_image, color_matcher_method, dst_path): + cm = ColorMatcher(method=color_matcher_method) + + i = 0 + total = len(imgs) + + for fname in imgs: + + img_src = Image.open(fname) + img_src = Normalizer(np.asarray(img_src)).type_norm() + + img_src = cm.transfer(src=img_src, ref=ref_image, method=color_matcher_method) + + img_src = Normalizer(img_src).uint8_norm() + Image.fromarray(img_src).save(os.path.join(dst_path, os.path.basename(fname))) + + i += 1 + print("{0}/{1}".format(i, total)) + + imgs = sorted( glob.glob( os.path.join(dst_path, "*.png") ) ) + + +def ebsynth_utility_stage3_5(dbg, project_args, color_matcher_method, st3_5_use_mask, st3_5_use_mask_ref, st3_5_use_mask_org, color_matcher_ref_type, color_matcher_ref_image): + dbg.print("stage3.5") + dbg.print("") + + _, _, frame_path, frame_mask_path, org_key_path, img2img_key_path, _ = project_args + + backup_path = os.path.join( os.path.join( img2img_key_path, "..") , "st3_5_backup_img2img_key") + backup_path = os.path.normpath(backup_path) + + if not os.path.isdir( backup_path ): + dbg.print("{0} not found -> create backup.".format(backup_path)) + os.makedirs(backup_path, exist_ok=True) + + imgs = glob.glob( os.path.join(img2img_key_path, "*.png") ) + + for img in imgs: + img_basename = os.path.basename(img) + pair_path = os.path.join( backup_path , img_basename ) + shutil.copy( img , pair_path) + + else: + dbg.print("{0} found -> Treat the images here as originals.".format(backup_path)) + + org_imgs = sorted( glob.glob( os.path.join(backup_path, "*.png") ) ) + head_of_keyframe = org_imgs[0] + + # open ref img + ref_image = color_matcher_ref_image + if not ref_image: + dbg.print("color_matcher_ref_image not set") + + if color_matcher_ref_type == 0: + #'original video frame' + dbg.print("select -> original video frame") + ref_image = Image.open( get_pair_of_img(head_of_keyframe, frame_path) ) + else: + #'first frame of img2img result' + dbg.print("select -> first frame of img2img result") + ref_image = Image.open( get_pair_of_img(head_of_keyframe, backup_path) ) + + ref_image = np.asarray(ref_image) + + if st3_5_use_mask_ref: + mask = get_pair_of_img(head_of_keyframe, frame_mask_path) + if mask: + mask_array = get_mask_array( mask ) + ref_image = ref_image * mask_array + ref_image = ref_image.astype(np.uint8) + + else: + dbg.print("select -> color_matcher_ref_image") + ref_image = np.asarray(ref_image) + + + if color_matcher_method in ('mvgd', 'hm-mvgd-hm'): + sample_img = Image.open(head_of_keyframe) + ref_image = resize_img( ref_image, sample_img.width, sample_img.height ) + + ref_image = Normalizer(ref_image).type_norm() + + + if st3_5_use_mask_org: + tmp_path = os.path.join( os.path.join( img2img_key_path, "..") , "st3_5_tmp") + tmp_path = os.path.normpath(tmp_path) + dbg.print("create {0} for masked original image".format(tmp_path)) + + remove_pngs_in_dir(tmp_path) + os.makedirs(tmp_path, exist_ok=True) + + for org_img in org_imgs: + image_basename = os.path.basename(org_img) + + org_image = np.asarray(Image.open(org_img)) + + mask = get_pair_of_img(org_img, frame_mask_path) + if mask: + mask_array = get_mask_array( mask ) + org_image = org_image * mask_array + org_image = org_image.astype(np.uint8) + + Image.fromarray(org_image).save( os.path.join( tmp_path, image_basename ) ) + + org_imgs = sorted( glob.glob( os.path.join(tmp_path, "*.png") ) ) + + + color_match(org_imgs, ref_image, color_matcher_method, img2img_key_path) + + + if st3_5_use_mask or st3_5_use_mask_org: + imgs = sorted( glob.glob( os.path.join(img2img_key_path, "*.png") ) ) + for img in imgs: + mask = get_pair_of_img(img, frame_mask_path) + if mask: + mask_array = get_mask_array( mask ) + bg = get_pair_of_img(img, frame_path) + bg_image = np.asarray(Image.open( bg )) + fg_image = np.asarray(Image.open( img )) + + final_img = fg_image * mask_array + bg_image * (1-mask_array) + final_img = final_img.astype(np.uint8) + + Image.fromarray(final_img).save(img) + + dbg.print("") + dbg.print("completed.") + diff --git a/stage5.py b/stage5.py index 14991e2..c70b358 100644 --- a/stage5.py +++ b/stage5.py @@ -8,7 +8,7 @@ from sys import byteorder import binascii import numpy as np -SYNTHS_PER_PROJECT = 25 +SYNTHS_PER_PROJECT = 15 def to_float_bytes(f): if byteorder == 'little': diff --git a/stage8.py b/stage8.py index 2ea070f..8f0f112 100644 --- a/stage8.py +++ b/stage8.py @@ -1,146 +1,146 @@ -import os -import re -import subprocess -import glob -import shutil -import time -import cv2 -import numpy as np -import itertools -from extensions.ebsynth_utility_lite.stage7 import create_movie_from_frames, get_ext, trying_to_add_audio - -def clamp(n, smallest, largest): - return sorted([smallest, n, largest])[1] - -def resize_img(img, w, h): - if img.shape[0] + img.shape[1] < h + w: - interpolation = interpolation=cv2.INTER_CUBIC - else: - interpolation = interpolation=cv2.INTER_AREA - - return cv2.resize(img, (w, h), interpolation=interpolation) - -def merge_bg_src(base_frame_dir, bg_dir, frame_mask_path, tmp_dir, bg_type, mask_blur_size, mask_threshold, fg_transparency): - - base_frames = sorted(glob.glob( os.path.join(base_frame_dir, "[0-9]*.png"), recursive=False) ) - - bg_frames = sorted(glob.glob( os.path.join(bg_dir, "*.png"), recursive=False) ) - - def bg_frame(total_frames): - bg_len = len(bg_frames) - - if bg_type == "Loop": - itr = itertools.cycle(bg_frames) - while True: - yield next(itr) - else: - for i in range(total_frames): - yield bg_frames[ int(bg_len * (i/total_frames))] - - bg_itr = bg_frame(len(base_frames)) - - for base_frame in base_frames: - im = cv2.imread(base_frame) - bg = cv2.imread( next(bg_itr) ) - bg = resize_img(bg, im.shape[1], im.shape[0] ) - - basename = os.path.basename(base_frame) - mask_path = os.path.join(frame_mask_path, basename) - mask = cv2.imread(mask_path)[:,:,0] - - mask[mask < int( 255 * mask_threshold )] = 0 - - if mask_blur_size > 0: - mask_blur_size = mask_blur_size//2 * 2 + 1 - mask = cv2.GaussianBlur(mask, (mask_blur_size, mask_blur_size), 0) - mask = mask[:, :, np.newaxis] - - fore_rate = (mask/255) * (1 - fg_transparency) - - im = im * fore_rate + bg * (1- fore_rate) - im = im.astype(np.uint8) - cv2.imwrite( os.path.join( tmp_dir , basename ) , im) - -def extract_frames(movie_path , output_dir, fps): - png_path = os.path.join(output_dir , "%05d.png") - # ffmpeg.exe -ss 00:00:00 -y -i %1 -qscale 0 -f image2 -c:v png "%05d.png" - subprocess.call("ffmpeg -ss 00:00:00 -y -i " + movie_path + " -vf fps=" + str( round(fps, 2)) + " -qscale 0 -f image2 -c:v png " + png_path, shell=True) - -def ebsynth_utility_stage8(dbg, project_args, bg_src, bg_type, mask_blur_size, mask_threshold, fg_transparency, export_type): - dbg.print("stage8") - dbg.print("") - - if not bg_src: - dbg.print("Fill [configuration] -> [stage 8] -> [Background source]") - return - - project_dir, original_movie_path, _, frame_mask_path, _, _, _ = project_args - - fps = 30 - clip = cv2.VideoCapture(original_movie_path) - if clip: - fps = clip.get(cv2.CAP_PROP_FPS) - clip.release() - - dbg.print("bg_src: {}".format(bg_src)) - dbg.print("bg_type: {}".format(bg_type)) - dbg.print("mask_blur_size: {}".format(mask_blur_size)) - dbg.print("export_type: {}".format(export_type)) - dbg.print("fps: {}".format(fps)) - - base_frame_dir = os.path.join( project_dir , "crossfade_tmp") - - if not os.path.isdir(base_frame_dir): - dbg.print(base_frame_dir + " base frame not found") - return - - tmp_dir = os.path.join( project_dir , "bg_merge_tmp") - if os.path.isdir(tmp_dir): - shutil.rmtree(tmp_dir) - os.mkdir(tmp_dir) - - ### create frame imgs - if os.path.isfile(bg_src): - bg_ext = os.path.splitext(os.path.basename(bg_src))[1] - if bg_ext == ".mp4": - bg_tmp_dir = os.path.join( project_dir , "bg_extract_tmp") - if os.path.isdir(bg_tmp_dir): - shutil.rmtree(bg_tmp_dir) - os.mkdir(bg_tmp_dir) - - extract_frames(bg_src, bg_tmp_dir, fps) - - bg_src = bg_tmp_dir - else: - dbg.print(bg_src + " must be mp4 or directory") - return - elif not os.path.isdir(bg_src): - dbg.print(bg_src + " must be mp4 or directory") - return - - merge_bg_src(base_frame_dir, bg_src, frame_mask_path, tmp_dir, bg_type, mask_blur_size, mask_threshold, fg_transparency) - - ### create movie - movie_base_name = time.strftime("%Y%m%d-%H%M%S") - movie_base_name = "merge_" + movie_base_name - - nosnd_path = os.path.join(project_dir , movie_base_name + get_ext(export_type)) - - merged_frames = sorted(glob.glob( os.path.join(tmp_dir, "[0-9]*.png"), recursive=False) ) - start = int(os.path.splitext(os.path.basename(merged_frames[0]))[0]) - end = int(os.path.splitext(os.path.basename(merged_frames[-1]))[0]) - - create_movie_from_frames(tmp_dir,start,end,5,fps,nosnd_path,export_type) - - dbg.print("exported : " + nosnd_path) - - if export_type == "mp4": - - with_snd_path = os.path.join(project_dir , movie_base_name + '_with_snd.mp4') - - if trying_to_add_audio(original_movie_path, nosnd_path, with_snd_path, tmp_dir): - dbg.print("exported : " + with_snd_path) - - dbg.print("") - dbg.print("completed.") - +import os +import re +import subprocess +import glob +import shutil +import time +import cv2 +import numpy as np +import itertools +from extensions.ebsynth_utility.stage7 import create_movie_from_frames, get_ext, trying_to_add_audio + +def clamp(n, smallest, largest): + return sorted([smallest, n, largest])[1] + +def resize_img(img, w, h): + if img.shape[0] + img.shape[1] < h + w: + interpolation = interpolation=cv2.INTER_CUBIC + else: + interpolation = interpolation=cv2.INTER_AREA + + return cv2.resize(img, (w, h), interpolation=interpolation) + +def merge_bg_src(base_frame_dir, bg_dir, frame_mask_path, tmp_dir, bg_type, mask_blur_size, mask_threshold, fg_transparency): + + base_frames = sorted(glob.glob( os.path.join(base_frame_dir, "[0-9]*.png"), recursive=False) ) + + bg_frames = sorted(glob.glob( os.path.join(bg_dir, "*.png"), recursive=False) ) + + def bg_frame(total_frames): + bg_len = len(bg_frames) + + if bg_type == "Loop": + itr = itertools.cycle(bg_frames) + while True: + yield next(itr) + else: + for i in range(total_frames): + yield bg_frames[ int(bg_len * (i/total_frames))] + + bg_itr = bg_frame(len(base_frames)) + + for base_frame in base_frames: + im = cv2.imread(base_frame) + bg = cv2.imread( next(bg_itr) ) + bg = resize_img(bg, im.shape[1], im.shape[0] ) + + basename = os.path.basename(base_frame) + mask_path = os.path.join(frame_mask_path, basename) + mask = cv2.imread(mask_path)[:,:,0] + + mask[mask < int( 255 * mask_threshold )] = 0 + + if mask_blur_size > 0: + mask_blur_size = mask_blur_size//2 * 2 + 1 + mask = cv2.GaussianBlur(mask, (mask_blur_size, mask_blur_size), 0) + mask = mask[:, :, np.newaxis] + + fore_rate = (mask/255) * (1 - fg_transparency) + + im = im * fore_rate + bg * (1- fore_rate) + im = im.astype(np.uint8) + cv2.imwrite( os.path.join( tmp_dir , basename ) , im) + +def extract_frames(movie_path , output_dir, fps): + png_path = os.path.join(output_dir , "%05d.png") + # ffmpeg.exe -ss 00:00:00 -y -i %1 -qscale 0 -f image2 -c:v png "%05d.png" + subprocess.call("ffmpeg -ss 00:00:00 -y -i " + movie_path + " -vf fps=" + str( round(fps, 2)) + " -qscale 0 -f image2 -c:v png " + png_path, shell=True) + +def ebsynth_utility_stage8(dbg, project_args, bg_src, bg_type, mask_blur_size, mask_threshold, fg_transparency, export_type): + dbg.print("stage8") + dbg.print("") + + if not bg_src: + dbg.print("Fill [configuration] -> [stage 8] -> [Background source]") + return + + project_dir, original_movie_path, _, frame_mask_path, _, _, _ = project_args + + fps = 30 + clip = cv2.VideoCapture(original_movie_path) + if clip: + fps = clip.get(cv2.CAP_PROP_FPS) + clip.release() + + dbg.print("bg_src: {}".format(bg_src)) + dbg.print("bg_type: {}".format(bg_type)) + dbg.print("mask_blur_size: {}".format(mask_blur_size)) + dbg.print("export_type: {}".format(export_type)) + dbg.print("fps: {}".format(fps)) + + base_frame_dir = os.path.join( project_dir , "crossfade_tmp") + + if not os.path.isdir(base_frame_dir): + dbg.print(base_frame_dir + " base frame not found") + return + + tmp_dir = os.path.join( project_dir , "bg_merge_tmp") + if os.path.isdir(tmp_dir): + shutil.rmtree(tmp_dir) + os.mkdir(tmp_dir) + + ### create frame imgs + if os.path.isfile(bg_src): + bg_ext = os.path.splitext(os.path.basename(bg_src))[1] + if bg_ext == ".mp4": + bg_tmp_dir = os.path.join( project_dir , "bg_extract_tmp") + if os.path.isdir(bg_tmp_dir): + shutil.rmtree(bg_tmp_dir) + os.mkdir(bg_tmp_dir) + + extract_frames(bg_src, bg_tmp_dir, fps) + + bg_src = bg_tmp_dir + else: + dbg.print(bg_src + " must be mp4 or directory") + return + elif not os.path.isdir(bg_src): + dbg.print(bg_src + " must be mp4 or directory") + return + + merge_bg_src(base_frame_dir, bg_src, frame_mask_path, tmp_dir, bg_type, mask_blur_size, mask_threshold, fg_transparency) + + ### create movie + movie_base_name = time.strftime("%Y%m%d-%H%M%S") + movie_base_name = "merge_" + movie_base_name + + nosnd_path = os.path.join(project_dir , movie_base_name + get_ext(export_type)) + + merged_frames = sorted(glob.glob( os.path.join(tmp_dir, "[0-9]*.png"), recursive=False) ) + start = int(os.path.splitext(os.path.basename(merged_frames[0]))[0]) + end = int(os.path.splitext(os.path.basename(merged_frames[-1]))[0]) + + create_movie_from_frames(tmp_dir,start,end,5,fps,nosnd_path,export_type) + + dbg.print("exported : " + nosnd_path) + + if export_type == "mp4": + + with_snd_path = os.path.join(project_dir , movie_base_name + '_with_snd.mp4') + + if trying_to_add_audio(original_movie_path, nosnd_path, with_snd_path, tmp_dir): + dbg.print("exported : " + with_snd_path) + + dbg.print("") + dbg.print("completed.") + diff --git a/style.css b/style.css index e638cdb..6d06f20 100644 --- a/style.css +++ b/style.css @@ -1,41 +1,41 @@ -#ebs_info_area { - border: #374151 2px solid; - border-radius: 5px; - font-size: 15px; - margin: 10px; - padding: 10px; -} - -#ebs_configuration_tab1>div{ - margin: 5px; - padding: 5px; -} - -#ebs_configuration_tab2>div{ - margin: 5px; - padding: 5px; -} - -#ebs_configuration_tab3_5>div{ - margin: 5px; - padding: 5px; -} - -#ebs_configuration_tab7>div{ - margin: 5px; - padding: 5px; -} - -#ebs_configuration_tab8>div{ - margin: 5px; - padding: 5px; -} - -#ebs_configuration_tab_etc>div{ - margin: 5px; - padding: 5px; -} - -video.svelte-w5wajl { - max-height: 500px; -} +#ebs_info_area { + border: #0B0F19 2px solid; + border-radius: 5px; + font-size: 15px; + margin: 10px; + padding: 10px; +} + +#ebs_configuration_tab1>div{ + margin: 5px; + padding: 5px; +} + +#ebs_configuration_tab2>div{ + margin: 5px; + padding: 5px; +} + +#ebs_configuration_tab3_5>div{ + margin: 5px; + padding: 5px; +} + +#ebs_configuration_tab7>div{ + margin: 5px; + padding: 5px; +} + +#ebs_configuration_tab8>div{ + margin: 5px; + padding: 5px; +} + +#ebs_configuration_tab_etc>div{ + margin: 5px; + padding: 5px; +} + +video.svelte-w5wajl { + max-height: 500px; +}