new resize with original images

handler.py

@@ -45,19 +45,19 @@ class EndpointHandler():
         face_adapter = f"./checkpoints/ip-adapter.bin"
         controlnet_path = f"./checkpoints/ControlNetModel"
 
-        transform = Compose([
-            Resize(
-                width=512,
-                height=512,
-                resize_target=False,
-                keep_aspect_ratio=True,
-                ensure_multiple_of=14,
-                resize_method='lower_bound',
-                image_interpolation_method=cv2.INTER_CUBIC,
-            ),
-            NormalizeImage(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
-            PrepareForNet(),
-        ])
+        # transform = Compose([
+        #     Resize(
+        #         width=512,
+        #         height=512,
+        #         resize_target=False,
+        #         keep_aspect_ratio=True,
+        #         ensure_multiple_of=14,
+        #         resize_method='lower_bound',
+        #         image_interpolation_method=cv2.INTER_CUBIC,
+        #     ),
+        #     NormalizeImage(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+        #     PrepareForNet(),
+        # ])
 
         self.controlnet_identitynet = ControlNetModel.from_pretrained(
             controlnet_path, torch_dtype=dtype
@@ -73,7 +73,6 @@ class EndpointHandler():
         feature_extractor=None,
         ).to(device)
 
-
         self.pipe.scheduler = diffusers.EulerDiscreteScheduler.from_config(
             self.pipe.scheduler.config
         )
@@ -89,10 +88,9 @@ class EndpointHandler():
 
         self.pipe.scheduler = diffusers.LCMScheduler.from_config(self.pipe.scheduler.config)
 
-
         controlnet_pose_model = "thibaud/controlnet-openpose-sdxl-1.0"
         controlnet_canny_model = "diffusers/controlnet-canny-sdxl-1.0"
-        controlnet_depth_model = "diffusers/controlnet-depth-sdxl-1.0-small"
+        # controlnet_depth_model = "diffusers/controlnet-depth-sdxl-1.0-small"
 
         controlnet_pose = ControlNetModel.from_pretrained(
             controlnet_pose_model, torch_dtype=dtype
@@ -100,49 +98,49 @@ class EndpointHandler():
         controlnet_canny = ControlNetModel.from_pretrained(
             controlnet_canny_model, torch_dtype=dtype
         ).to(device)
-        controlnet_depth = ControlNetModel.from_pretrained(
-            controlnet_depth_model, torch_dtype=dtype
-        ).to(device)
+        # controlnet_depth = ControlNetModel.from_pretrained(
+        #     controlnet_depth_model, torch_dtype=dtype
+        # ).to(device)
 
         openpose = OpenposeDetector.from_pretrained("lllyasviel/ControlNet")
-        depth_anything = DepthAnything.from_pretrained('LiheYoung/depth_anything_vitl14').to(device).eval()
+        # depth_anything = DepthAnything.from_pretrained('LiheYoung/depth_anything_vitl14').to(device).eval()
 
         def get_canny_image(image, t1=100, t2=200):
             image = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
             edges = cv2.Canny(image, t1, t2)
             return Image.fromarray(edges, "L")
         
-        def get_depth_map(image):
+        # def get_depth_map(image):
         
-            image = np.array(image) / 255.0
+        #     image = np.array(image) / 255.0
 
-            h, w = image.shape[:2]
+        #     h, w = image.shape[:2]
 
-            image = transform({'image': image})['image']
-            image = torch.from_numpy(image).unsqueeze(0).to("cuda")
+        #     image = transform({'image': image})['image']
+        #     image = torch.from_numpy(image).unsqueeze(0).to("cuda")
 
-            with torch.no_grad():
-                depth = depth_anything(image)
+        #     with torch.no_grad():
+        #         depth = depth_anything(image)
 
-            depth = F.interpolate(depth[None], (h, w), mode='bilinear', align_corners=False)[0, 0]
-            depth = (depth - depth.min()) / (depth.max() - depth.min()) * 255.0
+        #     depth = F.interpolate(depth[None], (h, w), mode='bilinear', align_corners=False)[0, 0]
+        #     depth = (depth - depth.min()) / (depth.max() - depth.min()) * 255.0
 
-            depth = depth.cpu().numpy().astype(np.uint8)
+        #     depth = depth.cpu().numpy().astype(np.uint8)
 
-            depth_image = Image.fromarray(depth)
+        #     depth_image = Image.fromarray(depth)
 
-            return depth_image
+        #     return depth_image
         
         self.controlnet_map = {
             "pose": controlnet_pose,
             "canny": controlnet_canny,
-            "depth": controlnet_depth,
+            # "depth": controlnet_depth,
         }
 
         self.controlnet_map_fn = {
             "pose": openpose,
             "canny": get_canny_image,
-            "depth": get_depth_map,
+            # "depth": get_depth_map,
         }
         
         self.app = FaceAnalysis(name="buffalo_l", root="./", providers=["CPUExecutionProvider"])
@@ -153,13 +151,14 @@ class EndpointHandler():
         identitynet_strength_ratio = 0.8
         pose_strength = 0.4
         canny_strength = 0.3
-        depth_strength = 0.5
-        self.my_controlnet_selection = ["pose", "canny", "depth"]
+        # depth_strength = 0.5
+        self.my_controlnet_selection = ["pose", "canny"]
+        # self.my_controlnet_selection = ["pose", "canny", "depth"]
 
         controlnet_scales = {
             "pose": pose_strength,
             "canny": canny_strength,
-            "depth": depth_strength,
+            # "depth": depth_strength,
         }
         
         self.pipe.controlnet = MultiControlNetModel(
@@ -176,19 +175,23 @@ class EndpointHandler():
         default_negative_prompt = "(lowres, low quality, worst quality:1.2), (text:1.2), watermark, anime, photorealistic, 35mm film, deformed, glitch, low contrast, noisy"
 
         # hyperparamters
-        prompt_input = data.pop("inputs", "a man")
+        # prompt_input = data.pop("inputs", "a man")
+        prompt_input = "a man"
         prompt=default_prompt.replace("{prompt}", prompt_input)
-        num_inference_steps = data.pop("num_inference_steps", 5)
-        guidance_scale = data.pop("guidance_scale", 1.5)
+        num_inference_steps = data.pop("num_inference_steps", 20)
+        guidance_scale = data.pop("guidance_scale", 5.0)
         negative_prompt = data.pop("negative_prompt", default_negative_prompt)
 
         # 1024px
         # face_image_path = data.pop("face_image_path", "https://i.ibb.co/SKg69dD/kaifu-resize.png")
         # pose_image_path = data.pop("pose_image_path", "https://i.ibb.co/ZSrQ8ZJ/pose.jpg")
         # 512px
-        face_image_path = "https://i.ibb.co/5Rsrd2d/kaifu-resize-1.png"
-        pose_image_path = "https://i.ibb.co/9bP9tMb/pose-2-1.jpg"
+        # face_image_path = "https://i.ibb.co/5Rsrd2d/kaifu-resize-1.png"
+        # pose_image_path = "https://i.ibb.co/9bP9tMb/pose-2-1.jpg"
         
+        # original
+        face_image_path = "https://i.ibb.co/GQzm527/examples-musk-resize.jpg"
+        pose_image_path = "https://i.ibb.co/ZSrQ8ZJ/pose.jpg"
         
         adapter_strength_ratio = 0.8
         
@@ -198,6 +201,44 @@ class EndpointHandler():
         def convert_from_image_to_cv2(img: Image) -> np.ndarray:
             return cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)
 
+        def resize_img(
+            input_image,
+            max_side=1280,
+            min_side=1024,
+            size=None,
+            pad_to_max_side=False,
+            mode=Image.BILINEAR,
+            base_pixel_number=64,
+        ):
+            if size is not None:
+                w_resize_new, h_resize_new = size
+            else:
+                w, h = input_image.size
+                # Calcular el redimensionamiento con un solo paso
+                ratio_min = min_side / min(w, h)
+                w_min, h_min = round(ratio_min * w), round(ratio_min * h)
+                ratio_max = max_side / max(w_min, h_min)
+                # Aplicar la menor de las dos ratios para asegurar que cumple ambas condiciones
+                final_ratio = min(ratio_min, ratio_max)
+                w_final, h_final = round(final_ratio * w), round(final_ratio * h)
+
+                # Ajustar al número base de píxeles más cercano
+                w_resize_new = (w_final // base_pixel_number) * base_pixel_number
+                h_resize_new = (h_final // base_pixel_number) * base_pixel_number
+
+            # Redimensionar una sola vez
+            input_image = input_image.resize([w_resize_new, h_resize_new], mode)
+
+            if pad_to_max_side:
+                # Optimizar la creación del fondo
+                res = Image.new("RGB", (max_side, max_side), (255, 255, 255))
+                offset_x = (max_side - w_resize_new) // 2
+                offset_y = (max_side - h_resize_new) // 2
+                res.paste(input_image, (offset_x, offset_y))
+                return res
+
+            return input_image
+
         # check if the input is valid
         # if face_image_path is None:
         #     raise gr.Error(
@@ -210,6 +251,7 @@ class EndpointHandler():
         # prompt, negative_prompt = apply_style(style_name, prompt, negative_prompt)
 
         face_image = load_image(face_image_path)
+        face_image = resize_img(face_image, max_side=1024)
         face_image_cv2 = convert_from_image_to_cv2(face_image)
         height, width, _ = face_image_cv2.shape
 
@@ -233,6 +275,7 @@ class EndpointHandler():
         img_controlnet = face_image
         if pose_image_path is not None:
             pose_image = load_image(pose_image_path)
+            pose_image = resize_img(pose_image, max_side=1024)
             img_controlnet = pose_image
             pose_image_cv2 = convert_from_image_to_cv2(pose_image)
 

pipeline_stable_diffusion_xl_instantid_full.py

@@ -61,10 +61,9 @@ EXAMPLE_DOC_STRING = """
         >>> from PIL import Image
         
         >>> from insightface.app import FaceAnalysis
-        
         >>> from pipeline_stable_diffusion_xl_instantid import StableDiffusionXLInstantIDPipeline, draw_kps
         >>> # download 'antelopev2' under ./models
-=        >>> app = FaceAnalysis(name='antelopev2')
+        >>> app = FaceAnalysis(name='antelopev2', root='./', providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
         >>> app.prepare(ctx_id=0, det_size=(640, 640))
         
         >>> # download models under ./checkpoints
@@ -469,33 +468,34 @@ class LongPromptWeight(object):
         prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0)
         return prompt_embeds
 
-    def draw_kps_optimized(image_pil, kps, color_list=[(255, 0, 0), (0, 255, 0), (0, 0, 255), (255, 255, 0), (255, 0, 255)]):
-        stickwidth = 4
-        limbSeq = np.array([[0, 2], [1, 2], [3, 2], [4, 2]])
-        kps = np.array(kps)
-
-        w, h = image_pil.size
-        out_img = np.zeros([h, w, 3], dtype=np.uint8) 
-
-        for i, (start, end) in enumerate(limbSeq):
-            color = color_list[i % len(color_list)] 
-
-            x = kps[[start, end], 0]
-            y = kps[[start, end], 1]
-
-            center = tuple(np.round(np.mean([x, y], axis=1)).astype(int))
-            length = int(np.hypot(x[0] - x[1], y[0] - y[1]) / 2)
-            angle = int(np.degrees(np.arctan2(y[0] - y[1], x[0] - x[1])))
-
-            polygon = cv2.ellipse2Poly(center, (length, stickwidth), angle, 0, 360, 1)
-            cv2.fillConvexPoly(out_img, polygon, color)
-
-        for idx_kp, (x, y) in enumerate(kps):
-            color = color_list[idx_kp % len(color_list)]  
-            cv2.circle(out_img, (int(x), int(y)), 10, color, thickness=-1)
-
-        out_img_pil = PIL.Image.fromarray(out_img)
-        return out_img_pil
+def draw_kps(image_pil, kps, color_list=[(255,0,0), (0,255,0), (0,0,255), (255,255,0), (255,0,255)]):
+    
+    stickwidth = 4
+    limbSeq = np.array([[0, 2], [1, 2], [3, 2], [4, 2]])
+    kps = np.array(kps)
+
+    w, h = image_pil.size
+    out_img = np.zeros([h, w, 3])
+
+    for i in range(len(limbSeq)):
+        index = limbSeq[i]
+        color = color_list[index[0]]
+
+        x = kps[index][:, 0]
+        y = kps[index][:, 1]
+        length = ((x[0] - x[1]) ** 2 + (y[0] - y[1]) ** 2) ** 0.5
+        angle = math.degrees(math.atan2(y[0] - y[1], x[0] - x[1]))
+        polygon = cv2.ellipse2Poly((int(np.mean(x)), int(np.mean(y))), (int(length / 2), stickwidth), int(angle), 0, 360, 1)
+        out_img = cv2.fillConvexPoly(out_img.copy(), polygon, color)
+    out_img = (out_img * 0.6).astype(np.uint8)
+
+    for idx_kp, kp in enumerate(kps):
+        color = color_list[idx_kp]
+        x, y = kp
+        out_img = cv2.circle(out_img.copy(), (int(x), int(y)), 10, color, -1)
+
+    out_img_pil = PIL.Image.fromarray(out_img.astype(np.uint8))
+    return out_img_pil
     
 class StableDiffusionXLInstantIDPipeline(StableDiffusionXLControlNetPipeline):
     
@@ -526,7 +526,7 @@ class StableDiffusionXLInstantIDPipeline(StableDiffusionXLControlNetPipeline):
     def set_image_proj_model(self, model_ckpt, image_emb_dim=512, num_tokens=16):
         
         image_proj_model = Resampler(
-            dim=512,
+            dim=1280,
             depth=4,
             dim_head=64,
             heads=20,