config.json

{
    "model_type": "Multimodal",
    "gpt2_model_name": "gpt2",
    "vit_model_name": "google/vit-base-patch16-224-in21k",
    "num_labels": 2,
    "task": "any-to-any"
}