Upload 17 files

adapter_model.py

@@ -0,0 +1,68 @@
+from dotenv import load_dotenv
+import os 
+from diffusers import StableDiffusionInpaintPipelineLegacy,StableDiffusionControlNetPipeline,ControlNetModel, DDIMScheduler,AutoencoderKL
+import torch
+from ip_adapter import IPAdapter
+
+load_dotenv()
+
+BASE_MODEL_PATH = str(os.getenv(
+    "BASE_MODEL_PATH ")
+    )
+VAE_MODEL_PATH = str(os.getenv(
+    "VAE_MODEL_PATH ")
+    )
+IMAGE_ENCODER_PATH = str(os.getenv(
+    "IMAGE_ENCODER_PATH ")
+    )
+IP_CKPT_PATH = str(os.getenv(
+    "IP_CKPT ")
+    )
+DEVICE = str(os.getenv(
+    "DEVICE ")
+    )
+
+noise_scheduler = DDIMScheduler(
+    num_train_timesteps=1000,
+    beta_start=0.00085,
+    beta_end=0.012,
+    beta_schedule="scaled_linear",
+    clip_sample=False,
+    set_alpha_to_one=False,
+    steps_offset=1,
+)
+vae = AutoencoderKL.from_pretrained(VAE_MODEL_PATH).to(dtype=torch.float16)
+
+
+class MODEL:
+    def __init__(self, action):
+        self.action = action
+        self.model = self._init_ip_model()
+
+    def _init_ip_model(self):
+        if self.action == "pose":
+            pipe = StableDiffusionInpaintPipelineLegacy.from_pretrained(
+                    BASE_MODEL_PATH,
+                    torch_dtype=torch.float16,
+                    scheduler=noise_scheduler,
+                    vae=vae,
+                    feature_extractor=None,
+                    safety_checker=None
+                )
+        elif self.action == "inpaint":
+            controlnet = ControlNetModel.from_pretrained(
+                "lllyasviel/control_v11p_sd15_openpose",
+                torch_dtype=torch.float16)
+
+            pipe = StableDiffusionControlNetPipeline.from_pretrained(
+                BASE_MODEL_PATH,
+                controlnet=controlnet,
+                torch_dtype=torch.float16,
+                scheduler=noise_scheduler,
+                vae=vae,
+                feature_extractor=None,
+                safety_checker=None
+            )
+
+        ip_model = IPAdapter(pipe, IMAGE_ENCODER_PATH, IP_CKPT_PATH, DEVICE)
+        return ip_model

app.py

@@ -0,0 +1,96 @@
+import gradio as gr 
+import os
+from PIL import Image
+# from ip_adapter_openpose import generate as generate_ip_adapter_openpose
+# from ip_adapter_inpainting import generate as generate_ip_adapter_inpainting
+# from adapter_model import MODEL
+
+human = os.path.join(os.path.dirname(__file__), "humans/manken3.jpg")
+
+
+def get_tryon_result(human_path, top_path, down_path):
+    human_img = Image.open(human_path).convert("RGB")
+    # UPPER BODY 4 , LOWER BODY 6
+    if top_path:
+        segment_id = 4
+        clothes_img = Image.open(top_path).convert("RGB")
+    elif down_path:
+        segment_id = 6
+        clothes_img = Image.open(down_path).convert("RGB")
+    
+    # img_openpose_gen = generate_ip_adapter_openpose(human_img, clothes_img)
+    # final_gen = generate_ip_adapter_inpainting(img_openpose_gen,
+    #                                            human_img,
+    #                                            clothes_img,
+    #                                            segment_id
+    #                                            )
+    # return final_gen
+    print(segment_id)
+    return human_img
+
+
+with gr.Blocks(css=".output-image, .input-image, .image-preview {height: 400px !important} ") as demo:
+    gr.HTML(
+        """
+        <div style="display: flex; justify-content: center; align-items: center; text-align: center;">
+        <a href="https://github.com/altayavci" style="margin-right: 20px; text-decoration: none; display: flex; align-items: center;">
+        </a>
+        <div>
+            <h1 >Clothes Changer: SuperAppLabs Clothes Tryon Case Study</h1>
+            <h4 >v0.1</h4>
+            <h5 style="margin: 0;">Altay Avcı</h5>
+        </div>
+        </div>
+        """)
+
+    with gr.Column():
+        gr.HTML(
+                """
+                        <div style="display: flex; justify-content: center; align-items: center; text-align: center;">
+                        <div>
+                            <h3>TOP OR BOTTOM. NOT BOTH</h3>
+                        </div>
+                        </div>
+                        """)
+        
+        with gr.Row():
+            top = gr.Image(sources='upload', type="filepath", label="TOP")
+            example_top = gr.Examples(inputs=top,
+                                      examples_per_page=3,
+                                      examples=[os.path.join(os.path.dirname(__file__), "clothes/kıyafet.jpg"),
+                                                os.path.join(os.path.dirname(__file__), "clothes/kıyafet1.jpg"),
+                                                os.path.join(os.path.dirname(__file__), "clothes/kıyafet3.jpeg"),
+                                                            ])
+                
+            with gr.Column():
+                down = gr.Image(sources='upload', type="filepath", label="DOWN")
+                example_down = gr.Examples(inputs=down,
+                                           examples_per_page=3,
+                                           examples=[
+                                                os.path.join(os.path.dirname(__file__), "clothes/garments_bottom1.png"),
+                                                os.path.join(os.path.dirname(__file__), "clothes/indir (3).png"),
+                                                os.path.join(os.path.dirname(__file__), "clothes/WhatsApp Image 2024-01-02 at 01.24.44.jpeg")
+                                                            ])
+                
+        with gr.Row():      
+                init_image = gr.Image(sources='clipboard', type="filepath", label="HUMAN", value=human)              
+                example_models = gr.Examples(inputs=init_image,
+                                             examples_per_page=2,
+                                             examples=[os.path.join(os.path.dirname(__file__), "humans/manken3.jpg"),
+                                                       os.path.join(os.path.dirname(__file__), "humans/manken2.jpg")
+                                                      ])
+        with gr.Column():
+            run_button = gr.Button(value="Run") 
+            gallery = gr.Image()
+            run_button.click(fn=get_tryon_result, 
+                             inputs=[
+                                  init_image,
+                                  top,
+                                  down,
+                                  ],
+                                  outputs=[gallery]
+                                  )               
+    
+if __name__ == "__main__":
+    demo.queue(max_size=10)
+    demo.launch()

img2txt.py

@@ -0,0 +1,19 @@
+from transformers import pipeline
+
+captioner = None
+PROMPT = "The main subject of this picture is a"
+
+def init():
+    global captioner
+    captioner = pipeline(
+            "image-to-text",
+            model="Salesforce/blip-image-captioning-base",
+            prompt=PROMPT
+        )
+
+def derive_caption(image):
+    result = captioner(image, max_new_tokens=20)
+    raw_caption = result[0]["generated_text"]
+    caption = raw_caption.lower().replace(PROMPT.lower(), "").strip()
+    return caption
+

ip_adapter_inpainting.py

@@ -0,0 +1,48 @@
+from PIL import Image
+import os
+import torch
+
+
+from segmentation import get_cropped, get_blurred_mask, init_body as init_body_seg, init_face as init_face_seg
+from img2txt import derive_caption, init as init_img2txt
+from utils import alpha_composite
+from adapter_model import MODEL
+
+init_face_seg()
+init_body_seg()
+init_img2txt()
+
+ip_model = MODEL("inpaint")
+
+
+def generate(img_openpose_gen: Image, img_human: Image, img_clothes: Image, segment_id: int):
+    cropped_clothes = get_cropped(img_openpose_gen, segment_id, False).resize((512, 768)) 
+    cropped_body = get_cropped(img_human, segment_id, True).resize((512, 768))
+
+    composite = alpha_composite(cropped_body.convert('RGBA'),
+                                cropped_clothes.convert('RGBA')
+                                )
+    composite = alpha_composite(composite)
+
+    mask = get_blurred_mask(composite, segment_id, False)
+    prompt = derive_caption(img_clothes)
+
+    ip_gen = ip_model.model.generate(
+        prompt=prompt,
+        pil_image=img_clothes,
+        num_samples=1,
+        num_inference_steps=50,
+        seed=42,
+        image=composite,
+        mask_image=mask,
+        strength=0.8,
+        guidance_scale=7,
+        scale=0.8
+        ).images[0]
+
+    cropped_head = get_cropped(img_openpose_gen, 13, False)
+    ip_gen_final = alpha_composite(ip_gen.convert("RGBA"),
+                                   cropped_head.convert("RGBA")
+                                   )
+    torch.cuda.empty_cache()
+    return ip_gen_final

ip_adapter_openpose.py

@@ -0,0 +1,33 @@
+from PIL import Image
+import torch
+
+from openpose import get_openpose, init as init_openpose
+from adapter_model import MODEL
+
+init_openpose()
+ip_model = MODEL("pose")
+
+
+def generate(img_human: Image, img_clothes: Image):
+
+    img_human = img_human.resize((512, 512))
+    img_clothes = img_clothes.resize((512, 768))
+    img_openpose = get_openpose(img_human)
+
+    img_openpose_gen = ip_model.model.generate(
+        pil_image=img_clothes,
+        image=img_openpose,
+        width=512,
+        height=768,
+        num_samples=1,
+        num_inference_steps=30,
+        seed=42
+    ).images[0]
+
+    torch.cuda.empty_cache()
+    return img_openpose_gen.convert("RGB")
+
+
+
+
+

openpose.py

@@ -0,0 +1,12 @@
+from controlnet_aux import OpenposeDetector
+from PIL import Image
+
+
+def init():
+    global openpose
+    openpose = OpenposeDetector.from_pretrained('lllyasviel/ControlNet')
+
+
+def get_openpose(img: Image ):
+    img_openpose = openpose(img)
+    return img_openpose

requirements.txt

@@ -0,0 +1,14 @@
+git+https://github.com/tencent-ailab/IP-Adapter.git
+torch
+diffusers
+transformers
+xformers
+accelerate
+scipy
+safetensors
+controlnet_aux
+numpy
+pillow
+python-dotenv
+
+

segmentation.py

@@ -0,0 +1,82 @@
+from PIL import Image, ImageFilter
+from transformers import AutoFeatureExtractor, SegformerForSemanticSegmentation, SegformerImageProcessor, AutoModelForSemanticSegmentation
+
+import numpy as np
+import torch.nn as nn
+from scipy.ndimage import binary_dilation
+
+model_body = None
+extractor_body = None
+
+model_face = None
+extractor_face = None
+
+def init_body():
+    global model_body, extractor_body
+    extractor_body = AutoFeatureExtractor.from_pretrained("mattmdjaga/segformer_b2_clothes")
+    model_body = SegformerForSemanticSegmentation.from_pretrained("mattmdjaga/segformer_b2_clothes").to("cuda")
+
+def init_face():
+    global model_face, extractor_face
+    extractor_face = SegformerImageProcessor.from_pretrained("jonathandinu/face-parsing")
+    model_face = AutoModelForSemanticSegmentation.from_pretrained("jonathandinu/face-parsing")
+
+
+def get_mask(img: Image, body_part_id: int, inverse=False, face=False):
+    if face:
+        inputs = extractor_face(images=img, return_tensors="pt").to("cuda")
+        outputs = model_face(**inputs)
+    else:
+        inputs = extractor_body(images=img, return_tensors="pt").to("cuda")
+        outputs = model_body(**inputs)
+    logits = outputs.logits.cpu()
+
+    upsampled_logits = nn.functional.interpolate(
+        logits,
+        size=img.size[::-1],
+        mode="bilinear",
+        align_corners=False,
+    )
+    pred_seg = upsampled_logits.argmax(dim=1)[0]
+    if inverse:
+        pred_seg[pred_seg == body_part_id ] = 0
+    else:
+        pred_seg[pred_seg != body_part_id ] = 0
+    arr_seg = pred_seg.cpu().numpy().astype("uint8")
+    arr_seg *= 255
+    pil_seg = Image.fromarray(arr_seg)
+    return pil_seg
+
+
+def get_cropped(img: Image, body_part_id: int, inverse=False):  # img openpose gen image olucak
+    
+    pil_seg = get_mask(img, body_part_id, inverse)
+    crop_mask_np = np.array(pil_seg.convert('L'))
+    crop_mask_binary = crop_mask_np > 128
+
+    dilated_mask = binary_dilation(
+                crop_mask_binary, iterations=1)
+    dilated_mask = Image.fromarray((dilated_mask * 255).astype(np.uint8))
+
+    mask = Image.fromarray(np.array(dilated_mask)).convert('L')
+    im_rgb = img.convert("RGB")
+
+    cropped = im_rgb.copy()
+    cropped.putalpha(mask)
+    return cropped
+
+
+def get_blurred_mask(img: Image, body_part_id: int, inverse=False):
+    pil_seg = get_mask(img, body_part_id, inverse)
+    crop_mask_np = np.array(pil_seg.convert('L'))
+    crop_mask_binary = crop_mask_np > 128
+
+    dilated_mask = binary_dilation(
+                crop_mask_binary, iterations=25)
+    dilated_mask = Image.fromarray((dilated_mask * 255).astype(np.uint8))
+    dilated_mask_blurred = dilated_mask.filter(
+                ImageFilter.GaussianBlur(radius=4))
+    return dilated_mask_blurred
+
+
+

utils.py

@@ -0,0 +1,10 @@
+from PIL import Image
+
+
+def alpha_composite(img: Image, background: Image = None):
+    if not background:
+        background = Image.new("RGBA", img.size, (255, 255, 255))
+    result = Image.alpha_composite(background, img)
+    result = result.convert("RGB")
+    return result
+