240 lines
7.8 KiB
Python
240 lines
7.8 KiB
Python
import os
|
|
import cv2
|
|
from PIL import Image
|
|
import numpy as np
|
|
|
|
from modules import shared, modelloader
|
|
from modules.sd_models import model_hash
|
|
from modules.paths import models_path
|
|
|
|
dd_models_path = os.path.join(models_path, "mmdet")
|
|
|
|
|
|
def get_person_mask(image):
|
|
results_a = inference_mmdet_segm(image, 'segm\mmdet_dd-person_mask2former.pth [1c8dbe8d]', 30/100.0, 'A')
|
|
masks_a = create_segmasks(results_a)
|
|
masks_a = dilate_masks(masks_a, 4, 1)
|
|
masks_a = offset_masks(masks_a, 0, 0)
|
|
return masks_a[0]
|
|
|
|
def list_models(model_path):
|
|
model_list = modelloader.load_models(model_path=model_path, ext_filter=[".pth"])
|
|
|
|
def modeltitle(path, shorthash):
|
|
abspath = os.path.abspath(path)
|
|
|
|
if abspath.startswith(model_path):
|
|
name = abspath.replace(model_path, '')
|
|
else:
|
|
name = os.path.basename(path)
|
|
|
|
if name.startswith("\\") or name.startswith("/"):
|
|
name = name[1:]
|
|
|
|
shortname = os.path.splitext(name.replace("/", "_").replace("\\", "_"))[0]
|
|
|
|
return f'{name} [{shorthash}]', shortname
|
|
|
|
models = []
|
|
for filename in model_list:
|
|
h = model_hash(filename)
|
|
title, short_model_name = modeltitle(filename, h)
|
|
models.append(title)
|
|
|
|
return models
|
|
|
|
def modeldataset(model_shortname):
|
|
path = modelpath(model_shortname)
|
|
if ("mmdet" in path and "segm" in path):
|
|
dataset = 'coco'
|
|
else:
|
|
dataset = 'bbox'
|
|
return dataset
|
|
|
|
def modelpath(model_shortname):
|
|
model_list = modelloader.load_models(model_path=dd_models_path, ext_filter=[".pth"])
|
|
model_h = model_shortname.split("[")[-1].split("]")[0]
|
|
for path in model_list:
|
|
if ( model_hash(path) == model_h):
|
|
return path
|
|
|
|
def update_result_masks(results, masks):
|
|
for i in range(len(masks)):
|
|
boolmask = np.array(masks[i], dtype=bool)
|
|
results[2][i] = boolmask
|
|
return results
|
|
|
|
def create_segmask_preview(results, image):
|
|
labels = results[0]
|
|
bboxes = results[1]
|
|
segms = results[2]
|
|
|
|
cv2_image = np.array(image)
|
|
cv2_image = cv2_image[:, :, ::-1].copy()
|
|
|
|
for i in range(len(segms)):
|
|
color = np.full_like(cv2_image, np.random.randint(100, 256, (1, 3), dtype=np.uint8))
|
|
alpha = 0.2
|
|
color_image = cv2.addWeighted(cv2_image, alpha, color, 1-alpha, 0)
|
|
cv2_mask = segms[i].astype(np.uint8) * 255
|
|
cv2_mask_bool = np.array(segms[i], dtype=bool)
|
|
centroid = np.mean(np.argwhere(cv2_mask_bool),axis=0)
|
|
centroid_x, centroid_y = int(centroid[1]), int(centroid[0])
|
|
|
|
cv2_mask_rgb = cv2.merge((cv2_mask, cv2_mask, cv2_mask))
|
|
cv2_image = np.where(cv2_mask_rgb == 255, color_image, cv2_image)
|
|
text_color = tuple([int(x) for x in ( color[0][0] - 100 )])
|
|
name = labels[i]
|
|
score = bboxes[i][4]
|
|
score = str(score)[:4]
|
|
text = name + ":" + score
|
|
cv2.putText(cv2_image, text, (centroid_x - 30, centroid_y), cv2.FONT_HERSHEY_DUPLEX, 0.4, text_color, 1, cv2.LINE_AA)
|
|
|
|
if ( len(segms) > 0):
|
|
preview_image = Image.fromarray(cv2.cvtColor(cv2_image, cv2.COLOR_BGR2RGB))
|
|
else:
|
|
preview_image = image
|
|
|
|
return preview_image
|
|
|
|
def is_allblack(mask):
|
|
cv2_mask = np.array(mask)
|
|
return cv2.countNonZero(cv2_mask) == 0
|
|
|
|
def bitwise_and_masks(mask1, mask2):
|
|
cv2_mask1 = np.array(mask1)
|
|
cv2_mask2 = np.array(mask2)
|
|
cv2_mask = cv2.bitwise_and(cv2_mask1, cv2_mask2)
|
|
mask = Image.fromarray(cv2_mask)
|
|
return mask
|
|
|
|
def subtract_masks(mask1, mask2):
|
|
cv2_mask1 = np.array(mask1)
|
|
cv2_mask2 = np.array(mask2)
|
|
cv2_mask = cv2.subtract(cv2_mask1, cv2_mask2)
|
|
mask = Image.fromarray(cv2_mask)
|
|
return mask
|
|
|
|
def dilate_masks(masks, dilation_factor, iter=1):
|
|
if dilation_factor == 0:
|
|
return masks
|
|
dilated_masks = []
|
|
kernel = np.ones((dilation_factor,dilation_factor), np.uint8)
|
|
for i in range(len(masks)):
|
|
cv2_mask = np.array(masks[i])
|
|
dilated_mask = cv2.dilate(cv2_mask, kernel, iter)
|
|
dilated_masks.append(Image.fromarray(dilated_mask))
|
|
return dilated_masks
|
|
|
|
def offset_masks(masks, offset_x, offset_y):
|
|
if (offset_x == 0 and offset_y == 0):
|
|
return masks
|
|
offset_masks = []
|
|
for i in range(len(masks)):
|
|
cv2_mask = np.array(masks[i])
|
|
offset_mask = cv2_mask.copy()
|
|
offset_mask = np.roll(offset_mask, -offset_y, axis=0)
|
|
offset_mask = np.roll(offset_mask, offset_x, axis=1)
|
|
|
|
offset_masks.append(Image.fromarray(offset_mask))
|
|
return offset_masks
|
|
|
|
def combine_masks(masks):
|
|
initial_cv2_mask = np.array(masks[0])
|
|
combined_cv2_mask = initial_cv2_mask
|
|
for i in range(1, len(masks)):
|
|
cv2_mask = np.array(masks[i])
|
|
combined_cv2_mask = cv2.bitwise_or(combined_cv2_mask, cv2_mask)
|
|
|
|
combined_mask = Image.fromarray(combined_cv2_mask)
|
|
return combined_mask
|
|
|
|
def create_segmasks(results):
|
|
segms = results[2]
|
|
segmasks = []
|
|
for i in range(len(segms)):
|
|
cv2_mask = segms[i].astype(np.uint8) * 255
|
|
mask = Image.fromarray(cv2_mask)
|
|
segmasks.append(mask)
|
|
|
|
return segmasks
|
|
|
|
import mmcv
|
|
from mmdet.core import get_classes
|
|
from mmdet.apis import (inference_detector,
|
|
init_detector)
|
|
|
|
def get_device():
|
|
device_id = shared.cmd_opts.device_id
|
|
if device_id is not None:
|
|
cuda_device = f"cuda:{device_id}"
|
|
else:
|
|
cuda_device = "cpu"
|
|
return cuda_device
|
|
|
|
def inference(image, modelname, conf_thres, label):
|
|
path = modelpath(modelname)
|
|
if ( "mmdet" in path and "bbox" in path ):
|
|
results = inference_mmdet_bbox(image, modelname, conf_thres, label)
|
|
elif ( "mmdet" in path and "segm" in path):
|
|
results = inference_mmdet_segm(image, modelname, conf_thres, label)
|
|
return results
|
|
|
|
def inference_mmdet_segm(image, modelname, conf_thres, label):
|
|
model_checkpoint = modelpath(modelname)
|
|
model_config = os.path.splitext(model_checkpoint)[0] + ".py"
|
|
model_device = get_device()
|
|
model = init_detector(model_config, model_checkpoint, device=model_device)
|
|
mmdet_results = inference_detector(model, np.array(image))
|
|
bbox_results, segm_results = mmdet_results
|
|
dataset = modeldataset(modelname)
|
|
classes = get_classes(dataset)
|
|
labels = [
|
|
np.full(bbox.shape[0], i, dtype=np.int32)
|
|
for i, bbox in enumerate(bbox_results)
|
|
]
|
|
n,m = bbox_results[0].shape
|
|
if (n == 0):
|
|
return [[],[],[]]
|
|
labels = np.concatenate(labels)
|
|
bboxes = np.vstack(bbox_results)
|
|
segms = mmcv.concat_list(segm_results)
|
|
filter_inds = np.where(bboxes[:,-1] > conf_thres)[0]
|
|
results = [[],[],[]]
|
|
for i in filter_inds:
|
|
results[0].append(label + "-" + classes[labels[i]])
|
|
results[1].append(bboxes[i])
|
|
results[2].append(segms[i])
|
|
|
|
return results
|
|
|
|
def inference_mmdet_bbox(image, modelname, conf_thres, label):
|
|
model_checkpoint = modelpath(modelname)
|
|
model_config = os.path.splitext(model_checkpoint)[0] + ".py"
|
|
model_device = get_device()
|
|
model = init_detector(model_config, model_checkpoint, device=model_device)
|
|
results = inference_detector(model, np.array(image))
|
|
cv2_image = np.array(image)
|
|
cv2_image = cv2_image[:, :, ::-1].copy()
|
|
cv2_gray = cv2.cvtColor(cv2_image, cv2.COLOR_BGR2GRAY)
|
|
|
|
segms = []
|
|
for (x0, y0, x1, y1, conf) in results[0]:
|
|
cv2_mask = np.zeros((cv2_gray.shape), np.uint8)
|
|
cv2.rectangle(cv2_mask, (int(x0), int(y0)), (int(x1), int(y1)), 255, -1)
|
|
cv2_mask_bool = cv2_mask.astype(bool)
|
|
segms.append(cv2_mask_bool)
|
|
|
|
n,m = results[0].shape
|
|
if (n == 0):
|
|
return [[],[],[]]
|
|
bboxes = np.vstack(results[0])
|
|
filter_inds = np.where(bboxes[:,-1] > conf_thres)[0]
|
|
results = [[],[],[]]
|
|
for i in filter_inds:
|
|
results[0].append(label)
|
|
results[1].append(bboxes[i])
|
|
results[2].append(segms[i])
|
|
|
|
return results
|