import os import cv2 from PIL import Image import numpy as np from modules import shared, modelloader from modules.sd_models import model_hash from modules.paths import models_path dd_models_path = os.path.join(models_path, "mmdet") def get_person_mask(image): results_a = inference_mmdet_segm(image, 'segm\mmdet_dd-person_mask2former.pth [1c8dbe8d]', 30/100.0, 'A') masks_a = create_segmasks(results_a) masks_a = dilate_masks(masks_a, 4, 1) masks_a = offset_masks(masks_a, 0, 0) return masks_a[0] def list_models(model_path): model_list = modelloader.load_models(model_path=model_path, ext_filter=[".pth"]) def modeltitle(path, shorthash): abspath = os.path.abspath(path) if abspath.startswith(model_path): name = abspath.replace(model_path, '') else: name = os.path.basename(path) if name.startswith("\\") or name.startswith("/"): name = name[1:] shortname = os.path.splitext(name.replace("/", "_").replace("\\", "_"))[0] return f'{name} [{shorthash}]', shortname models = [] for filename in model_list: h = model_hash(filename) title, short_model_name = modeltitle(filename, h) models.append(title) return models def modeldataset(model_shortname): path = modelpath(model_shortname) if ("mmdet" in path and "segm" in path): dataset = 'coco' else: dataset = 'bbox' return dataset def modelpath(model_shortname): model_list = modelloader.load_models(model_path=dd_models_path, ext_filter=[".pth"]) model_h = model_shortname.split("[")[-1].split("]")[0] for path in model_list: if ( model_hash(path) == model_h): return path def update_result_masks(results, masks): for i in range(len(masks)): boolmask = np.array(masks[i], dtype=bool) results[2][i] = boolmask return results def create_segmask_preview(results, image): labels = results[0] bboxes = results[1] segms = results[2] cv2_image = np.array(image) cv2_image = cv2_image[:, :, ::-1].copy() for i in range(len(segms)): color = np.full_like(cv2_image, np.random.randint(100, 256, (1, 3), dtype=np.uint8)) alpha = 0.2 color_image = cv2.addWeighted(cv2_image, alpha, color, 1-alpha, 0) cv2_mask = segms[i].astype(np.uint8) * 255 cv2_mask_bool = np.array(segms[i], dtype=bool) centroid = np.mean(np.argwhere(cv2_mask_bool),axis=0) centroid_x, centroid_y = int(centroid[1]), int(centroid[0]) cv2_mask_rgb = cv2.merge((cv2_mask, cv2_mask, cv2_mask)) cv2_image = np.where(cv2_mask_rgb == 255, color_image, cv2_image) text_color = tuple([int(x) for x in ( color[0][0] - 100 )]) name = labels[i] score = bboxes[i][4] score = str(score)[:4] text = name + ":" + score cv2.putText(cv2_image, text, (centroid_x - 30, centroid_y), cv2.FONT_HERSHEY_DUPLEX, 0.4, text_color, 1, cv2.LINE_AA) if ( len(segms) > 0): preview_image = Image.fromarray(cv2.cvtColor(cv2_image, cv2.COLOR_BGR2RGB)) else: preview_image = image return preview_image def is_allblack(mask): cv2_mask = np.array(mask) return cv2.countNonZero(cv2_mask) == 0 def bitwise_and_masks(mask1, mask2): cv2_mask1 = np.array(mask1) cv2_mask2 = np.array(mask2) cv2_mask = cv2.bitwise_and(cv2_mask1, cv2_mask2) mask = Image.fromarray(cv2_mask) return mask def subtract_masks(mask1, mask2): cv2_mask1 = np.array(mask1) cv2_mask2 = np.array(mask2) cv2_mask = cv2.subtract(cv2_mask1, cv2_mask2) mask = Image.fromarray(cv2_mask) return mask def dilate_masks(masks, dilation_factor, iter=1): if dilation_factor == 0: return masks dilated_masks = [] kernel = np.ones((dilation_factor,dilation_factor), np.uint8) for i in range(len(masks)): cv2_mask = np.array(masks[i]) dilated_mask = cv2.dilate(cv2_mask, kernel, iter) dilated_masks.append(Image.fromarray(dilated_mask)) return dilated_masks def offset_masks(masks, offset_x, offset_y): if (offset_x == 0 and offset_y == 0): return masks offset_masks = [] for i in range(len(masks)): cv2_mask = np.array(masks[i]) offset_mask = cv2_mask.copy() offset_mask = np.roll(offset_mask, -offset_y, axis=0) offset_mask = np.roll(offset_mask, offset_x, axis=1) offset_masks.append(Image.fromarray(offset_mask)) return offset_masks def combine_masks(masks): initial_cv2_mask = np.array(masks[0]) combined_cv2_mask = initial_cv2_mask for i in range(1, len(masks)): cv2_mask = np.array(masks[i]) combined_cv2_mask = cv2.bitwise_or(combined_cv2_mask, cv2_mask) combined_mask = Image.fromarray(combined_cv2_mask) return combined_mask def create_segmasks(results): segms = results[2] segmasks = [] for i in range(len(segms)): cv2_mask = segms[i].astype(np.uint8) * 255 mask = Image.fromarray(cv2_mask) segmasks.append(mask) return segmasks import mmcv from mmdet.core import get_classes from mmdet.apis import (inference_detector, init_detector) def get_device(): device_id = shared.cmd_opts.device_id if device_id is not None: cuda_device = f"cuda:{device_id}" else: cuda_device = "cpu" return cuda_device def inference(image, modelname, conf_thres, label): path = modelpath(modelname) if ( "mmdet" in path and "bbox" in path ): results = inference_mmdet_bbox(image, modelname, conf_thres, label) elif ( "mmdet" in path and "segm" in path): results = inference_mmdet_segm(image, modelname, conf_thres, label) return results def inference_mmdet_segm(image, modelname, conf_thres, label): model_checkpoint = modelpath(modelname) model_config = os.path.splitext(model_checkpoint)[0] + ".py" model_device = get_device() model = init_detector(model_config, model_checkpoint, device=model_device) mmdet_results = inference_detector(model, np.array(image)) bbox_results, segm_results = mmdet_results dataset = modeldataset(modelname) classes = get_classes(dataset) labels = [ np.full(bbox.shape[0], i, dtype=np.int32) for i, bbox in enumerate(bbox_results) ] n,m = bbox_results[0].shape if (n == 0): return [[],[],[]] labels = np.concatenate(labels) bboxes = np.vstack(bbox_results) segms = mmcv.concat_list(segm_results) filter_inds = np.where(bboxes[:,-1] > conf_thres)[0] results = [[],[],[]] for i in filter_inds: results[0].append(label + "-" + classes[labels[i]]) results[1].append(bboxes[i]) results[2].append(segms[i]) return results def inference_mmdet_bbox(image, modelname, conf_thres, label): model_checkpoint = modelpath(modelname) model_config = os.path.splitext(model_checkpoint)[0] + ".py" model_device = get_device() model = init_detector(model_config, model_checkpoint, device=model_device) results = inference_detector(model, np.array(image)) cv2_image = np.array(image) cv2_image = cv2_image[:, :, ::-1].copy() cv2_gray = cv2.cvtColor(cv2_image, cv2.COLOR_BGR2GRAY) segms = [] for (x0, y0, x1, y1, conf) in results[0]: cv2_mask = np.zeros((cv2_gray.shape), np.uint8) cv2.rectangle(cv2_mask, (int(x0), int(y0)), (int(x1), int(y1)), 255, -1) cv2_mask_bool = cv2_mask.astype(bool) segms.append(cv2_mask_bool) n,m = results[0].shape if (n == 0): return [[],[],[]] bboxes = np.vstack(results[0]) filter_inds = np.where(bboxes[:,-1] > conf_thres)[0] results = [[],[],[]] for i in filter_inds: results[0].append(label) results[1].append(bboxes[i]) results[2].append(segms[i]) return results