1580158
/
automatic
mirror of https://github.com/vladmandic/automatic
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add video processing

pull/2631/head
Vladimir Mandic 2023-12-08 10:24:01 -05:00
parent efba57a1ca
commit 72a167c43a
10 changed files with 119 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -16,6 +16,9 @@
- **Schedulers**
- add timesteps range, changing it will make scheduler to be over-complete or under-complete
- add rescale betas with zero SNR option (applicable to Euler and DDIM, allows for higher dynamic range)
- **Process**
- create videos from batch or folder processing
supports GIF, PNG and MP4 with full interpolation, scene change detection, etc.
- **General**
- **LoRA** add support for block weights, thanks @AI-Casanova
example `<lora:SDXL_LCM_LoRA:1.0:in=0:mid=1:out=0>`
@ -25,6 +28,7 @@
- disable google fonts check on server startup
- fix torchvision/basicsr compatibility
- add hdr settings to metadata
- update built-in log monitor in ui, thanks @midcoastal
## Update for 2023-12-04

18
javascript/logMonitor.js
View File

@ -11,14 +11,13 @@ async function logMonitor() {
logMonitorStatus = true;
if (!logMonitorEl) {
logMonitorEl = document.getElementById('logMonitorData');
logMonitorEl.onscrollend = function() {
const at_bottom = logMonitorEl.scrollHeight <= (logMonitorEl.scrollTop + logMonitorEl.clientHeight)
if (at_bottom)
logMonitorEl.parentElement.style = '';
}
logMonitorEl.onscrollend = () => {
const at_bottom = logMonitorEl.scrollHeight <= (logMonitorEl.scrollTop + logMonitorEl.clientHeight);
if (at_bottom) logMonitorEl.parentElement.style = '';
};
}
if (!logMonitorEl) return;
const at_bottom = logMonitorEl.scrollHeight <= (logMonitorEl.scrollTop + logMonitorEl.clientHeight)
const at_bottom = logMonitorEl.scrollHeight <= (logMonitorEl.scrollTop + logMonitorEl.clientHeight);
const lines = await res.json();
if (logMonitorEl && lines?.length > 0) logMonitorEl.parentElement.parentElement.style.display = opts.logmonitor_show ? 'block' : 'none';
for (const line of lines) {
@ -31,11 +30,8 @@ async function logMonitor() {
} catch {}
}
while (logMonitorEl.childElementCount > 100) logMonitorEl.removeChild(logMonitorEl.firstChild);
if (at_bottom) {
logMonitorEl.scrollTop = logMonitorEl.scrollHeight;
} else if (lines?.length > 0) {
logMonitorEl.parentElement.style = 'border-bottom: 2px solid yellow';
}
if (at_bottom) logMonitorEl.scrollTop = logMonitorEl.scrollHeight;
else if (lines?.length > 0) logMonitorEl.parentElement.style = 'border-bottom: 2px solid var(--highlight-color);';
}
}

36
modules/images.py
View File

@ -322,6 +322,8 @@ class FilenameGenerator:
default_time_format = '%Y%m%d%H%M%S'
def __init__(self, p, seed, prompt, image, grid=False):
if getattr(self, 'p', None) is None:
return
self.p = p
self.seed = seed
self.prompt = prompt
@ -335,7 +337,7 @@ class FilenameGenerator:
def hasprompt(self, *args):
lower = self.prompt.lower()
if self.p is None or self.prompt is None:
if getattr(self, 'p', None) is None or getattr(self, 'prompt', None) is None:
return None
outres = ""
for arg in args:
@ -350,7 +352,7 @@ class FilenameGenerator:
return outres
def image_hash(self):
if self.image is None:
if getattr(self, 'image', None) is None:
return None
import base64
from io import BytesIO
@ -364,7 +366,7 @@ class FilenameGenerator:
return self.prompt_sanitize(self.prompt)
def prompt_words(self):
if self.prompt is None:
if getattr(self, 'prompt', None) is None:
return ''
no_attention = re_attention.sub(r'\1', self.prompt)
no_network = re_network.sub(r'\1', no_attention)
@ -374,7 +376,7 @@ class FilenameGenerator:
return self.prompt_sanitize(prompt)
def prompt_no_style(self):
if self.p is None or self.prompt is None:
if getattr(self, 'p', None) is None or getattr(self, 'prompt', None) is None:
return None
prompt_no_style = self.prompt
for style in shared.prompt_styles.get_style_prompts(self.p.styles):
@ -455,7 +457,7 @@ class FilenameGenerator:
break
pattern, arg = m.groups()
pattern_args.insert(0, arg)
fun = self.replacements.get(pattern.lower())
fun = self.replacements.get(pattern.lower(), None)
if fun is not None:
try:
replacement = fun(self, *pattern_args)
@ -525,7 +527,7 @@ def atomically_save_image():
try:
image.save(fn, format=image_format, compress_level=6, pnginfo=pnginfo_data if shared.opts.image_metadata else None)
except Exception as e:
shared.log.warning(f'Image save failed: {fn} {e}')
shared.log.error(f'Image save failed: file="{fn}" {e}')
elif image_format == 'JPEG':
if image.mode == 'RGBA':
shared.log.warning('Saving RGBA image as JPEG: Alpha channel will be lost')
@ -536,7 +538,7 @@ def atomically_save_image():
try:
image.save(fn, format=image_format, optimize=True, quality=shared.opts.jpeg_quality, exif=exif_bytes)
except Exception as e:
shared.log.warning(f'Image save failed: {fn} {e}')
shared.log.error(f'Image save failed: file="{fn}" {e}')
elif image_format == 'WEBP':
if image.mode == 'I;16':
image = image.point(lambda p: p * 0.0038910505836576).convert("RGB")
@ -544,13 +546,13 @@ def atomically_save_image():
try:
image.save(fn, format=image_format, quality=shared.opts.jpeg_quality, lossless=shared.opts.webp_lossless, exif=exif_bytes)
except Exception as e:
shared.log.warning(f'Image save failed: {fn} {e}')
shared.log.error(f'Image save failed: file="{fn}" {e}')
else:
# shared.log.warning(f'Unrecognized image format: {extension} attempting save as {image_format}')
try:
image.save(fn, format=image_format, quality=shared.opts.jpeg_quality)
except Exception as e:
shared.log.warning(f'Image save failed: {fn} {e}')
shared.log.error(f'Image save failed: file="{fn}" {e}')
with open(os.path.join(paths.data_path, "params.txt"), "w", encoding="utf8") as file:
file.write(exifinfo)
if shared.opts.save_log_fn != '' and len(exifinfo) > 0:
@ -668,12 +670,20 @@ def save_video(p, images, filename = None, video_type: str = 'none', duration: f
if images is None or len(images) < 2 or video_type is None or video_type.lower() == 'none':
return
image = images[0]
namegen = FilenameGenerator(p, seed=p.all_seeds[0], prompt=p.all_prompts[0], image=image)
if filename is None:
if p is not None:
namegen = FilenameGenerator(p, seed=p.all_seeds[0], prompt=p.all_prompts[0], image=image)
else:
namegen = FilenameGenerator(None, seed=0, prompt='', image=image)
if filename is None and p is not None:
filename = namegen.apply(shared.opts.samples_filename_pattern if shared.opts.samples_filename_pattern and len(shared.opts.samples_filename_pattern) > 0 else "[seq]-[prompt_words]")
filename = namegen.sanitize(os.path.join(shared.opts.outdir_video, filename))
filename = os.path.join(shared.opts.outdir_video, filename)
filename = namegen.sequence(filename, shared.opts.outdir_video, '')
filename = f'{filename}.{video_type.lower()}'
else:
if not os.pathsep in filename:
filename = os.path.join(shared.opts.outdir_video, filename)
if not filename.lower().endswith(video_type.lower()):
filename += f'.{video_type.lower()}'
filename = namegen.sanitize(filename)
threading.Thread(target=save_video_atomic, args=(images, filename, video_type, duration, loop, interpolate, scale, pad, change)).start()

20
modules/postprocessing.py
View File

@ -13,24 +13,29 @@ def run_postprocessing(extras_mode, image, image_folder: List[tempfile.NamedTemp
shared.state.begin('extras')
image_data = []
image_names = []
image_fullnames = []
image_ext = []
outputs = []
params = {}
if extras_mode == 1:
shared.log.debug(f'process: mode=batch folder={image_folder}')
for img in image_folder:
if isinstance(img, Image.Image):
image = img
fn = ''
ext = None
else:
image = Image.open(os.path.abspath(img.name))
try:
image = Image.open(os.path.abspath(img.name))
except Exception as e:
shared.log.error(f'Failed to open image: file="{img.name}" {e}')
continue
fn, ext = os.path.splitext(img.orig_name)
image_fullnames.append(img.name)
image_data.append(image)
image_names.append(fn)
image_ext.append(ext)
shared.log.debug(f'Process: mode=batch inputs={len(image_folder)} images={len(image_data)}')
elif extras_mode == 2:
shared.log.debug(f'process: mode=folder folder={input_dir}')
assert not shared.cmd_opts.hide_ui_dir_config, '--hide-ui-dir-config option must be disabled'
assert input_dir, 'input directory not selected'
image_list = shared.listfiles(input_dir)
@ -38,11 +43,13 @@ def run_postprocessing(extras_mode, image, image_folder: List[tempfile.NamedTemp
try:
image = Image.open(filename)
except Exception as e:
shared.log.error(f'Failed to open image: {filename} {e}')
shared.log.error(f'Failed to open image: file="{filename}" {e}')
continue
image_fullnames.append(filename)
image_data.append(image)
image_names.append(filename)
image_ext.append(None)
shared.log.debug(f'Process: mode=folder inputs={input_dir} files={len(image_list)} images={len(image_data)}')
else:
image_data.append(image)
image_names.append(None)
@ -51,8 +58,9 @@ def run_postprocessing(extras_mode, image, image_folder: List[tempfile.NamedTemp
outpath = output_dir
else:
outpath = opts.outdir_samples or opts.outdir_extras_samples
processed_images = []
for image, name, ext in zip(image_data, image_names, image_ext): # pylint: disable=redefined-argument-from-local
shared.log.debug(f'process: image={image} {args}')
shared.log.debug(f'Process: image={image} {args}')
infotext = ''
if shared.state.interrupted:
shared.log.debug('Postprocess interrupted')
@ -74,11 +82,13 @@ def run_postprocessing(extras_mode, image, image_folder: List[tempfile.NamedTemp
infotext = items['parameters'] + ', '
infotext = infotext + ", ".join([k if k == v else f'{k}: {generation_parameters_copypaste.quote(v)}' for k, v in pp.info.items() if v is not None])
pp.image.info["postprocessing"] = infotext
processed_images.append(pp.image)
if save_output:
images.save_image(pp.image, path=outpath, basename=basename, seed=None, prompt=None, extension=ext or opts.samples_format, info=infotext, short_filename=True, no_prompt=True, grid=False, pnginfo_section_name="extras", existing_info=pp.image.info, forced_filename=None)
if extras_mode != 2 or show_extras_results:
outputs.append(pp.image)
image.close()
scripts.scripts_postproc.postprocess(processed_images, args)
devices.torch_gc()
return outputs, infotext, params

6
modules/rife/__init__.py
View File

@ -11,12 +11,12 @@ from PIL import Image
from torch.nn import functional as F
from tqdm.rich import tqdm
from modules.rife.ssim import ssim_matlab
from modules.rife.model_rife import Model
from modules.rife.model_rife import RifeModel
from modules import devices, shared
model_url = 'https://github.com/vladmandic/rife/raw/main/model/flownet-v46.pkl'
model = None
model: RifeModel = None
def load(model_path: str = 'rife/flownet-v46.pkl'):
@ -26,7 +26,7 @@ def load(model_path: str = 'rife/flownet-v46.pkl'):
model_dir = os.path.join(shared.models_path, 'RIFE')
model_path = modelloader.load_file_from_url(url=model_url, model_dir=model_dir, file_name='flownet-v46.pkl')
shared.log.debug(f'RIFE load model: file="{model_path}"')
model = Model()
model = RifeModel()
model.load_model(model_path, -1)
model.eval()
model.device()

2
modules/rife/model_rife.py
View File

@ -6,7 +6,7 @@ from modules.rife.loss import EPE, SOBEL
from modules import devices
class Model:
class RifeModel:
def __init__(self, local_rank=-1):
self.flownet = IFNet()
self.device()

16
modules/scripts_postprocessing.py
View File

@ -101,7 +101,7 @@ class ScriptPostprocessingRunner:
process_args = {}
for (name, _component), value in zip(script.controls.items(), script_args):
process_args[name] = value
shared.log.debug(f'postprocess: script={script.name} args={process_args}')
shared.log.debug(f'Process: script={script.name} args={process_args}')
script.process(pp, **process_args)
def create_args_for_run(self, scripts_args):
@ -110,15 +110,25 @@ class ScriptPostprocessingRunner:
self.setup_ui()
scripts = self.scripts_in_preferred_order()
args = [None] * max([x.args_to for x in scripts])
for script in scripts:
script_args_dict = scripts_args.get(script.name, None)
if script_args_dict is not None:
for i, name in enumerate(script.controls):
args[script.args_from + i] = script_args_dict.get(name, None)
return args
def image_changed(self):
for script in self.scripts_in_preferred_order():
script.image_changed()
def postprocess(self, filenames, args):
for script in self.scripts_in_preferred_order():
if not hasattr(script, 'postprocess'):
continue
shared.state.job = script.name
script_args = args[script.args_from:script.args_to]
process_args = {}
for (name, _component), value in zip(script.controls.items(), script_args):
process_args[name] = value
shared.log.debug(f'Postprocess: script={script.name} args={process_args}')
script.postprocess(filenames, **process_args)

7
modules/ui_postprocessing.py
View File

@ -12,8 +12,8 @@ def wrap_pnginfo(image):
return infotext_to_html(geninfo), info, geninfo
def submit_click(tab_index, extras_image, image_batch, extras_batch_input_dir, extras_batch_output_dir, show_extras_results, *script_inputs):
result_images, geninfo, js_info = postprocessing.run_postprocessing(tab_index, extras_image, image_batch, extras_batch_input_dir, extras_batch_output_dir, show_extras_results, *script_inputs)
def submit_click(tab_index, extras_image, image_batch, extras_batch_input_dir, extras_batch_output_dir, show_extras_results, save_output, *script_inputs):
result_images, geninfo, js_info = postprocessing.run_postprocessing(tab_index, extras_image, image_batch, extras_batch_input_dir, extras_batch_output_dir, show_extras_results, *script_inputs, save_output=save_output)
return result_images, geninfo, json.dumps(js_info), ''
@ -32,6 +32,8 @@ def create_ui():
show_extras_results = gr.Checkbox(label='Show result images', value=True, elem_id="extras_show_extras_results")
with gr.Row():
buttons = parameters_copypaste.create_buttons(["txt2img", "img2img", "inpaint"])
with gr.Row():
save_output = gr.Checkbox(label='Save output', value=True, elem_id="extras_save_output")
script_inputs = scripts.scripts_postproc.setup_ui()
with gr.Column():
id_part = 'extras'
@ -66,6 +68,7 @@ def create_ui():
extras_batch_input_dir,
extras_batch_output_dir,
show_extras_results,
save_output,
*script_inputs,
],
outputs=[

1
requirements.txt
View File

@ -42,6 +42,7 @@ scikit-image
basicsr
fasteners
dctorch
pymatting
httpx==0.24.1
compel==2.0.2
torchsde==0.2.6

47
scripts/postprocessing_video.py Normal file
View File

@ -0,0 +1,47 @@
import gradio as gr
import modules.images
from modules import scripts_postprocessing
class ScriptPostprocessingUpscale(scripts_postprocessing.ScriptPostprocessing):
name = "Video"
def ui(self):
def video_type_change(video_type):
return [
gr.update(visible=video_type != 'None'),
gr.update(visible=video_type == 'GIF' or video_type == 'PNG'),
gr.update(visible=video_type == 'MP4'),
gr.update(visible=video_type == 'MP4'),
gr.update(visible=video_type == 'MP4'),
gr.update(visible=video_type == 'MP4'),
]
with gr.Row():
video_type = gr.Dropdown(label='Video file', choices=['None', 'GIF', 'PNG', 'MP4'], value='None')
duration = gr.Slider(label='Duration', minimum=0.25, maximum=10, step=0.25, value=2, visible=False)
with gr.Row():
loop = gr.Checkbox(label='Loop', value=True, visible=False)
pad = gr.Slider(label='Pad frames', minimum=0, maximum=24, step=1, value=1, visible=False)
interpolate = gr.Slider(label='Interpolate frames', minimum=0, maximum=24, step=1, value=0, visible=False)
scale = gr.Slider(label='Rescale', minimum=0.5, maximum=2, step=0.05, value=1, visible=False)
change = gr.Slider(label='Frame change sensitivity', minimum=0, maximum=1, step=0.05, value=0.3, visible=False)
with gr.Row():
filename = gr.Textbox(label='Filename', placeholder='enter filename', lines=1)
video_type.change(fn=video_type_change, inputs=[video_type], outputs=[duration, loop, pad, interpolate, scale, change])
return {
"filename": filename,
"video_type": video_type,
"duration": duration,
"loop": loop,
"pad": pad,
"interpolate": interpolate,
"scale": scale,
"change": change,
}
def postprocess(self, images, filename, video_type, duration, loop, pad, interpolate, scale, change): # pylint: disable=arguments-differ
filename = filename.strip()
if video_type == 'None' or len(filename) == 0 or images is None or len(images) < 2:
return
modules.images.save_video(p=None, filename=filename, images=images, video_type=video_type, duration=duration, loop=loop, pad=pad, interpolate=interpolate, scale=scale, change=change)