toilaluan/Florence-2-base-Yes-No-VQA

Florence 2 Finetuned & Modified for Binary Answer: 1 - Yes, 0 - No

import requests
import torch
from PIL import Image
from transformers import AutoProcessor, AutoModelForCausalLM 

device = "cuda:0" if torch.cuda.is_available() else "cpu"
torch_dtype = torch.float32

model = AutoModelForCausalLM.from_pretrained("toilaluan/Florence-2-base-Yes-No-VQA", torch_dtype=torch_dtype, trust_remote_code=True).to(device)
processor = AutoProcessor.from_pretrained("toilaluan/Florence-2-base-Yes-No-VQA", trust_remote_code=True)

prompt = "Is there a car in front of a cat?"

url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/car.jpg?download=true"
image = Image.open(requests.get(url, stream=True).raw)

inputs = processor(text=prompt, images=image, return_tensors="pt").to(device, torch_dtype)
decoder_input_ids = torch.LongTensor([[model.language_model.config.pad_token_id, model.language_model.config.decoder_start_token_id]]).to(device)
outputs = model(
    input_ids=inputs["input_ids"],
    pixel_values=inputs["pixel_values"],
    decoder_input_ids=decoder_input_ids
)
logits = outputs.logits[:, -1]

print(f"The answer Yes has {logits[0].sigmoid().item()} probs")

All credit to original Florence-2 (Microsoft)