mrq/stable-diffusion-webui
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

improve debug markers, fix algo weighting

Browse Source
This commit is contained in:
captin411 2022-10-25 13:10:58 -07:00
parent 1be5933ba2
commit 3e6c2420c1
1 changed files with 127 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

203
modules/textual_inversion/autocrop.py
View File

@ -1,4 +1,5 @@
import cv2 import cv2
import os
from collections import defaultdict from collections import defaultdict
from math import log, sqrt from math import log, sqrt
import numpy as np import numpy as np
@ -26,19 +27,9 @@ def crop_image(im, settings):
scale_by = settings.crop_height / im.height scale_by = settings.crop_height / im.height
im = im.resize((int(im.width * scale_by), int(im.height * scale_by))) im = im.resize((int(im.width * scale_by), int(im.height * scale_by)))
im_debug = im.copy()
if im.width == settings.crop_width and im.height == settings.crop_height: focus = focal_point(im_debug, settings)
if settings.annotate_image:
d = ImageDraw.Draw(im)
rect = [0, 0, im.width, im.height]
rect[2] -= 1
rect[3] -= 1
d.rectangle(rect, outline=GREEN)
if settings.destop_view_image:
im.show()
return im
focus = focal_point(im, settings)
# take the focal point and turn it into crop coordinates that try to center over the focal # take the focal point and turn it into crop coordinates that try to center over the focal
# point but then get adjusted back into the frame # point but then get adjusted back into the frame
@ -62,89 +53,143 @@ def crop_image(im, settings):
crop = [x1, y1, x2, y2] crop = [x1, y1, x2, y2]
results = []
results.append(im.crop(tuple(crop)))
if settings.annotate_image: if settings.annotate_image:
d = ImageDraw.Draw(im) d = ImageDraw.Draw(im_debug)
rect = list(crop) rect = list(crop)
rect[2] -= 1 rect[2] -= 1
rect[3] -= 1 rect[3] -= 1
d.rectangle(rect, outline=GREEN) d.rectangle(rect, outline=GREEN)
results.append(im_debug)
if settings.destop_view_image: if settings.destop_view_image:
im.show() im_debug.show()
return im.crop(tuple(crop)) return results
def focal_point(im, settings): def focal_point(im, settings):
corner_points = image_corner_points(im, settings) corner_points = image_corner_points(im, settings)
entropy_points = image_entropy_points(im, settings) entropy_points = image_entropy_points(im, settings)
face_points = image_face_points(im, settings) face_points = image_face_points(im, settings)
total_points = len(corner_points) + len(entropy_points) + len(face_points)
corner_weight = settings.corner_points_weight
entropy_weight = settings.entropy_points_weight
face_weight = settings.face_points_weight
weight_pref_total = corner_weight + entropy_weight + face_weight
# weight things
pois = [] pois = []
if weight_pref_total == 0 or total_points == 0:
return pois
pois.extend( weight_pref_total = 0
[ PointOfInterest( p.x, p.y, weight=p.weight * ( (corner_weight/weight_pref_total) / (len(corner_points)/total_points) )) for p in corner_points ] if len(corner_points) > 0:
) weight_pref_total += settings.corner_points_weight
pois.extend( if len(entropy_points) > 0:
[ PointOfInterest( p.x, p.y, weight=p.weight * ( (entropy_weight/weight_pref_total) / (len(entropy_points)/total_points) )) for p in entropy_points ] weight_pref_total += settings.entropy_points_weight
) if len(face_points) > 0:
pois.extend( weight_pref_total += settings.face_points_weight
[ PointOfInterest( p.x, p.y, weight=p.weight * ( (face_weight/weight_pref_total) / (len(face_points)/total_points) )) for p in face_points ]
) corner_centroid = None
if len(corner_points) > 0:
corner_centroid = centroid(corner_points)
corner_centroid.weight = settings.corner_points_weight / weight_pref_total
pois.append(corner_centroid)
entropy_centroid = None
if len(entropy_points) > 0:
entropy_centroid = centroid(entropy_points)
entropy_centroid.weight = settings.entropy_points_weight / weight_pref_total
pois.append(entropy_centroid)
face_centroid = None
if len(face_points) > 0:
face_centroid = centroid(face_points)
face_centroid.weight = settings.face_points_weight / weight_pref_total
pois.append(face_centroid)
average_point = poi_average(pois, settings) average_point = poi_average(pois, settings)
if settings.annotate_image: if settings.annotate_image:
d = ImageDraw.Draw(im) d = ImageDraw.Draw(im)
for f in face_points: max_size = min(im.width, im.height) * 0.07
d.rectangle(f.bounding(f.size), outline=RED) if corner_centroid is not None:
for f in entropy_points: color = BLUE
d.rectangle(f.bounding(30), outline=BLUE) box = corner_centroid.bounding(max_size * corner_centroid.weight)
for poi in pois: d.text((box[0], box[1]-15), "Edge: %.02f" % corner_centroid.weight, fill=color)
w = max(4, 4 * 0.5 * sqrt(poi.weight)) d.ellipse(box, outline=color)
d.ellipse(poi.bounding(w), fill=BLUE) if len(corner_points) > 1:
d.ellipse(average_point.bounding(25), outline=GREEN) for f in corner_points:
d.rectangle(f.bounding(4), outline=color)
if entropy_centroid is not None:
color = "#ff0"
box = entropy_centroid.bounding(max_size * entropy_centroid.weight)
d.text((box[0], box[1]-15), "Entropy: %.02f" % entropy_centroid.weight, fill=color)
d.ellipse(box, outline=color)
if len(entropy_points) > 1:
for f in entropy_points:
d.rectangle(f.bounding(4), outline=color)
if face_centroid is not None:
color = RED
box = face_centroid.bounding(max_size * face_centroid.weight)
d.text((box[0], box[1]-15), "Face: %.02f" % face_centroid.weight, fill=color)
d.ellipse(box, outline=color)
if len(face_points) > 1:
for f in face_points:
d.rectangle(f.bounding(4), outline=color)
d.ellipse(average_point.bounding(max_size), outline=GREEN)
return average_point return average_point
def image_face_points(im, settings): def image_face_points(im, settings):
np_im = np.array(im) if settings.dnn_model_path is not None:
gray = cv2.cvtColor(np_im, cv2.COLOR_BGR2GRAY) detector = cv2.FaceDetectorYN.create(
settings.dnn_model_path,
"",
(im.width, im.height),
0.8, # score threshold
0.3, # nms threshold
5000 # keep top k before nms
)
faces = detector.detect(np.array(im))
results = []
if faces[1] is not None:
for face in faces[1]:
x = face[0]
y = face[1]
w = face[2]
h = face[3]
results.append(
PointOfInterest(
int(x + (w * 0.5)), # face focus left/right is center
int(y + (h * 0)), # face focus up/down is close to the top of the head
size = w,
weight = 1/len(faces[1])
)
)
return results
else:
np_im = np.array(im)
gray = cv2.cvtColor(np_im, cv2.COLOR_BGR2GRAY)
tries = [ tries = [
[ f'{cv2.data.haarcascades}haarcascade_eye.xml', 0.01 ], [ f'{cv2.data.haarcascades}haarcascade_eye.xml', 0.01 ],
[ f'{cv2.data.haarcascades}haarcascade_frontalface_default.xml', 0.05 ], [ f'{cv2.data.haarcascades}haarcascade_frontalface_default.xml', 0.05 ],
[ f'{cv2.data.haarcascades}haarcascade_profileface.xml', 0.05 ], [ f'{cv2.data.haarcascades}haarcascade_profileface.xml', 0.05 ],
[ f'{cv2.data.haarcascades}haarcascade_frontalface_alt.xml', 0.05 ], [ f'{cv2.data.haarcascades}haarcascade_frontalface_alt.xml', 0.05 ],
[ f'{cv2.data.haarcascades}haarcascade_frontalface_alt2.xml', 0.05 ], [ f'{cv2.data.haarcascades}haarcascade_frontalface_alt2.xml', 0.05 ],
[ f'{cv2.data.haarcascades}haarcascade_frontalface_alt_tree.xml', 0.05 ], [ f'{cv2.data.haarcascades}haarcascade_frontalface_alt_tree.xml', 0.05 ],
[ f'{cv2.data.haarcascades}haarcascade_eye_tree_eyeglasses.xml', 0.05 ], [ f'{cv2.data.haarcascades}haarcascade_eye_tree_eyeglasses.xml', 0.05 ],
[ f'{cv2.data.haarcascades}haarcascade_upperbody.xml', 0.05 ] [ f'{cv2.data.haarcascades}haarcascade_upperbody.xml', 0.05 ]
] ]
for t in tries:
classifier = cv2.CascadeClassifier(t[0])
minsize = int(min(im.width, im.height) * t[1]) # at least N percent of the smallest side
try:
faces = classifier.detectMultiScale(gray, scaleFactor=1.1,
minNeighbors=7, minSize=(minsize, minsize), flags=cv2.CASCADE_SCALE_IMAGE)
except:
continue
for t in tries: if len(faces) > 0:
# print(t[0]) rects = [[f[0], f[1], f[0] + f[2], f[1] + f[3]] for f in faces]
classifier = cv2.CascadeClassifier(t[0]) return [PointOfInterest((r[0] +r[2]) // 2, (r[1] + r[3]) // 2, size=abs(r[0]-r[2]), weight=1/len(rects)) for r in rects]
minsize = int(min(im.width, im.height) * t[1]) # at least N percent of the smallest side
try:
faces = classifier.detectMultiScale(gray, scaleFactor=1.1,
minNeighbors=7, minSize=(minsize, minsize), flags=cv2.CASCADE_SCALE_IMAGE)
except:
continue
if len(faces) > 0:
rects = [[f[0], f[1], f[0] + f[2], f[1] + f[3]] for f in faces]
return [PointOfInterest((r[0] +r[2]) // 2, (r[1] + r[3]) // 2, size=abs(r[0]-r[2])) for r in rects]
return [] return []
@ -161,7 +206,7 @@ def image_corner_points(im, settings):
np_im, np_im,
maxCorners=100, maxCorners=100,
qualityLevel=0.04, qualityLevel=0.04,
minDistance=min(grayscale.width, grayscale.height)*0.07, minDistance=min(grayscale.width, grayscale.height)*0.03,
useHarrisDetector=False, useHarrisDetector=False,
) )
@ -171,7 +216,7 @@ def image_corner_points(im, settings):
focal_points = [] focal_points = []
for point in points: for point in points:
x, y = point.ravel() x, y = point.ravel()
focal_points.append(PointOfInterest(x, y, size=4)) focal_points.append(PointOfInterest(x, y, size=4, weight=1/len(points)))
return focal_points return focal_points
@ -205,17 +250,22 @@ def image_entropy_points(im, settings):
x_mid = int(crop_best[0] + settings.crop_width/2) x_mid = int(crop_best[0] + settings.crop_width/2)
y_mid = int(crop_best[1] + settings.crop_height/2) y_mid = int(crop_best[1] + settings.crop_height/2)
return [PointOfInterest(x_mid, y_mid, size=25)] return [PointOfInterest(x_mid, y_mid, size=25, weight=1.0)]
def image_entropy(im): def image_entropy(im):
# greyscale image entropy # greyscale image entropy
# band = np.asarray(im.convert("L")) band = np.asarray(im.convert("L"))
band = np.asarray(im.convert("1"), dtype=np.uint8) # band = np.asarray(im.convert("1"), dtype=np.uint8)
hist, _ = np.histogram(band, bins=range(0, 256)) hist, _ = np.histogram(band, bins=range(0, 256))
hist = hist[hist > 0] hist = hist[hist > 0]
return -np.log2(hist / hist.sum()).sum() return -np.log2(hist / hist.sum()).sum()
def centroid(pois):
x = [poi.x for poi in pois]
y = [poi.y for poi in pois]
return PointOfInterest(sum(x)/len(pois), sum(y)/len(pois))
def poi_average(pois, settings): def poi_average(pois, settings):
weight = 0.0 weight = 0.0
@ -260,11 +310,12 @@ class PointOfInterest:
class Settings: class Settings:
def __init__(self, crop_width=512, crop_height=512, corner_points_weight=0.5, entropy_points_weight=0.5, face_points_weight=0.5, annotate_image=False): def __init__(self, crop_width=512, crop_height=512, corner_points_weight=0.5, entropy_points_weight=0.5, face_points_weight=0.5, annotate_image=False, dnn_model_path=None):
self.crop_width = crop_width self.crop_width = crop_width
self.crop_height = crop_height self.crop_height = crop_height
self.corner_points_weight = corner_points_weight self.corner_points_weight = corner_points_weight
self.entropy_points_weight = entropy_points_weight self.entropy_points_weight = entropy_points_weight
self.face_points_weight = entropy_points_weight self.face_points_weight = face_points_weight
self.annotate_image = annotate_image self.annotate_image = annotate_image
self.destop_view_image = False self.destop_view_image = False
self.dnn_model_path = dnn_model_path