app.py

import gradio as gr
import open_clip
import torch
import requests
import numpy as np
from PIL import Image
from io import BytesIO

# Sidebar content
sidebar_markdown = """

Note, this demo can classify 200 items. If you didn't find what you're looking for, reach out to us on our [Community](https://join.slack.com/t/marqo-community/shared_invite/zt-2iab0260n-QJrZLUSOJYUifVxf964Gdw) and request an item to be added.

## Documentation
📚 [Blog Post](https://www.marqo.ai/blog/search-model-for-fashion)

📝 [Use Case Blog Post](https://www.marqo.ai/blog/ecommerce-image-classification-with-marqo-fashionclip)

## Code
💻 [GitHub Repo](https://github.com/marqo-ai/marqo-FashionCLIP)

🤝 [Google Colab](https://colab.research.google.com/drive/1nq978xFJjJcnyrJ2aE5l82GHAXOvTmfd?usp=sharing)

🤗 [Hugging Face Collection](https://huggingface.co/collections/Marqo/marqo-fashionclip-and-marqo-fashionsiglip-66b43f2d09a06ad2368d4af6)

## Citation
If you use Marqo-FashionSigLIP or Marqo-FashionCLIP, please cite us:
```
@software{Jung_Marqo-FashionCLIP_and_Marqo-FashionSigLIP_2024,
author = {Jung, Myong Chol and Clark, Jesse},
month = aug,
title = {{Marqo-FashionCLIP and Marqo-FashionSigLIP}},
url = {https://github.com/marqo-ai/marqo-FashionCLIP},
version = {1.0.0},
year = {2024}
```
"""

# List of fashion items
items = [
    'abaya', 'anorak', 'apron', 'ball gown', 'bandanna', 'baseball cap',
    'beanie', 'belt', 'beret', 'Bermuda shorts', 'baby clothes', 
    'bib', 'bikini', 'blazer', 'blouse', 'boots', 'bow tie', 'boxer shorts', 'boxers', 'bra', 
    'bracelet', 'breeches', 'buckle', 'button', 'camouflage', 'cap', 'cape', 'cardigan', 'cloak', 'clogs', 
    'coat', 'corset', 'crown', 'cuff links', 'dress', 'dress shirt', 'dungarees', 'earmuffs', 
    'earrings', 'flannel shirt', 'flip-flops', 'fur coat', 'gilet', 'glasses', 'gloves', 'gown', 'handbag',
    'hat', 'Hawaiian shirt', 'helmet', 'hijab', 'high heels', 'hoodie', 'hospital gown', 'jacket', 
    'jeans', 'jewelry', 'jumper', 'jumpsuit', 'khakis', 'kilt', 'knickers', 'lab coat',
    'leather jacket', 'leggings', 'leotard', 'life jacket', 'lingerie', 'loafers',
    'miniskirt', 'mittens', 'necklace', 'nightgown', 'nightshirt', 'onesies','pajamas', 'pants', 
    'pantsuit', 'pantyhose', 'parka','polo shirt', 'poncho', 'purse', 'raincoat', 
    'ring', 'robe', 'rugby shirt', 'sandals', 'scarf', 'scrubs', 'shirt', 'shoes', 'shorts', 'skirt', 
    'slippers', 'sneakers', 'socks', 'spacesuit', 'stockings', 'stole', 'suit', 
    'sun hat', 'sundress', 'sunglasses', 'suspenders', 'sweater', 'sweatpants', 'sweatshirt', 'swimsuit', 
    't-shirt', 'tank top', 'tiara', 'tie', 'tie clip', 'tights', 'toga', 'top', 'top coat', 'top hat', 'train', 
    'trench coat', 'trousers', 'trunks', 'tube top', 'turban', 'turtleneck', 'tutu', 'tuxedo', 'umbrella',
    'veil', 'vest', 'waistcoat', 'wedding gown', 'wetsuit', 
    'windbreaker', 'jogger', 'palazzo', 'cargo', 'dresspants', 'chinos', 
    'crop top', 'romper', 'insulated jacket', 'fleece', 'rain jacket', 
    'running jacket', 'graphic top', 'legging', 'skort',
    'sports bra', 'water shorts', 'goggle', 'glove', 'mitten', 
    'leg gaiter', 'neck gaiter', 'watch', 'bag', 'swim trunk', 
    'pocket watch', 'insoles', "climbing shoes",
]


# Initialize the model and tokenizer
model_name = 'hf-hub:Marqo/marqo-fashionSigLIP'
model, preprocess_train, preprocess_val = open_clip.create_model_and_transforms(model_name)
tokenizer = open_clip.get_tokenizer(model_name)

def generate_description(item):
    if "Pants" in item or item in ["Leggings", "Jogger", "Cargo", "Chinos", "Palazzo", "Dresspants", "Sweatpants", "Pant", "Legging", "Skort", "Trouser"]:
        return f"A pair of {item} pants"
    elif item in ["Dress", "Blouse", "T-Shirt", "Tank Top", "Sweater", "Cardigan", "Hoodie", "Coat", "Jacket", "Polo Shirt", "Crop Top", "Romper", "Blazer", "Vest", "Bodysuit", "Maxi Dress", "Graphic Top", "Shirt", "Base Layer Top", "Base Layer Bottom", "Swimsuit", "Rashguard", "Cover Up", "Tuxedo"]:
        return f"A {item}"
    elif item in ["Hat", "Sunglasses", "Glasses", "Sun Hat", "Goggle", "Balaclava"]:
        return f"A {item} worn on the head or face"
    elif item in ["Shoes", "Sandals", "Heels", "Trainers", "Boots", "Slippers", "Sneakers", "Insoles", "Socks"]:
        return f"A pair of {item} worn on the feet"
    elif item in ["Jeans", "Skirt", "Shorts", "Dungarees", "Poncho", "Overalls", "Boxer", "Swim Trunk", "Ring", "Necklace", "Earing", "Pocket Watch"]:
        return f"A {item} piece of clothing"
    elif item in ["Boxing Gloves", "Glove", "Mitten"]:
        return f"An item of {item} worn on the hands"
    else:
        return f"A fashion item called {item}"
items_desc = [generate_description(item) for item in items]
text = tokenizer(items_desc)

# Encode text features (unchanged)
with torch.no_grad(), torch.amp.autocast('cuda'):
    text_features = model.encode_text(text)
    text_features /= text_features.norm(dim=-1, keepdim=True)

# Prediction function
def predict(image, url):
    if url:
        response = requests.get(url)
        image = Image.open(BytesIO(response.content))
    
    processed_image = preprocess_val(image).unsqueeze(0)

    with torch.no_grad(), torch.amp.autocast('cuda'):
        image_features = model.encode_image(processed_image)
        image_features /= image_features.norm(dim=-1, keepdim=True)

        text_probs = (100 * image_features @ text_features.T).softmax(dim=-1)
        
        sorted_confidences = sorted(
            {items[i]: float(text_probs[0, i]) for i in range(len(items))}.items(), 
            key=lambda x: x[1], 
            reverse=True
        )
        top_10_confidences = dict(sorted_confidences[:10])
        
    return image, top_10_confidences

# Clear function
def clear_fields():
    return None, ""

# Gradio interface
title = "Fashion Item Classifier with Marqo-FashionSigLIP"
description = "Upload an image or provide a URL of a fashion item to classify it using [Marqo-FashionSigLIP](https://huggingface.co/Marqo/marqo-fashionSigLIP)!"

examples = [
    ["images/dress.jpg", "Dress"],
    ["images/sweatpants.jpg", "Sweatpants"],
    ["images/t-shirt.jpg", "T-Shirt"],
    ["images/hat.jpg", "Hat"],
    ["images/blouse.jpg", "Blouse"],
    ["images/cargo.jpg", "Cargos"],
    ["images/sunglasses.jpg", "Sunglasses"],
    ["images/polo-shirt.jpg", "Polo Shirt"],
]

with gr.Blocks(css="""
    .remove-btn {
        font-size: 24px !important; /* Increase the font size of the cross button */
        line-height: 24px !important;
        width: 30px !important; /* Increase the width */
        height: 30px !important; /* Increase the height */
    }
""") as demo:
    with gr.Row():
        with gr.Column(scale=1):
            gr.Markdown(f"# {title}")
            gr.Markdown(description)
            gr.Markdown(sidebar_markdown)
            gr.Markdown(" ", elem_id="vertical-line")  # Add an empty Markdown with a custom ID
        with gr.Column(scale=2):
            input_image = gr.Image(type="pil", label="Upload Fashion Item Image", height=312)
            input_url = gr.Textbox(label="Or provide an image URL")
            with gr.Row():
                predict_button = gr.Button("Classify")
                clear_button = gr.Button("Clear")
            gr.Markdown("Or click on one of the images below to classify it:")
            gr.Examples(examples=examples, inputs=input_image)
            output_label = gr.Label(num_top_classes=6)
            predict_button.click(predict, inputs=[input_image, input_url], outputs=[input_image, output_label])
            clear_button.click(clear_fields, outputs=[input_image, input_url])
        
# Launch the interface
demo.launch()