Upload model

config.json

@@ -6,194 +6,36 @@
     "AutoModel": "modelling_cxrmate_ed.CXRMateEDModel"
   },
   "decoder": {
-    "_name_or_path": "",
-    "add_cross_attention": false,
     "add_time_deltas": true,
-    "architectures": null,
-    "attention_bias": false,
-    "attention_dropout": 0.0,
-    "bad_words_ids": null,
-    "begin_suppress_tokens": null,
-    "bos_token_id": 1,
-    "chunk_size_feed_forward": 0,
-    "cross_attention_hidden_size": null,
-    "decoder_start_token_id": null,
-    "diversity_penalty": 0.0,
-    "do_sample": false,
-    "early_stopping": false,
-    "encoder_no_repeat_ngram_size": 0,
-    "eos_token_id": 2,
-    "exponential_decay_length_penalty": null,
-    "finetuning_task": null,
-    "forced_bos_token_id": null,
-    "forced_eos_token_id": null,
-    "hidden_act": "silu",
     "hidden_size": 768,
     "history": 0,
-    "id2label": {
-      "0": "LABEL_0",
-      "1": "LABEL_1"
-    },
     "include_time_delta": true,
     "index_value_encoder_intermediate_size": 2048,
-    "initializer_range": 0.02,
     "intermediate_size": 3072,
     "is_decoder": true,
-    "is_encoder_decoder": false,
-    "label2id": {
-      "LABEL_0": 0,
-      "LABEL_1": 1
-    },
-    "length_penalty": 1.0,
-    "max_length": 20,
-    "max_position_embeddings": 2048,
-    "min_length": 0,
     "model_type": "llama",
-    "no_repeat_ngram_size": 0,
     "num_attention_heads": 12,
-    "num_beam_groups": 1,
-    "num_beams": 1,
     "num_hidden_layers": 6,
     "num_key_value_heads": 12,
-    "num_return_sequences": 1,
-    "output_attentions": false,
-    "output_hidden_states": false,
-    "output_scores": false,
     "pad_token_id": 4,
-    "prefix": null,
-    "pretraining_tp": 1,
-    "problem_type": null,
     "prompt_report_sections_filter": [
       "indication",
       "history"
     ],
-    "pruned_heads": {},
-    "remove_invalid_values": false,
-    "repetition_penalty": 1.0,
-    "return_dict": true,
-    "return_dict_in_generate": false,
-    "rms_norm_eps": 1e-06,
-    "rope_scaling": null,
-    "rope_theta": 10000.0,
-    "sep_token_id": null,
-    "suppress_tokens": null,
     "tables_filter": [
       "mimic_cxr_sectioned",
       "triage",
       "medrecon"
     ],
-    "task_specific_params": null,
-    "temperature": 1.0,
-    "tf_legacy_loss": false,
-    "tie_encoder_decoder": false,
-    "tie_word_embeddings": false,
     "time_delta_monotonic_inversion": true,
-    "tokenizer_class": null,
-    "top_k": 50,
-    "top_p": 1.0,
-    "torch_dtype": null,
-    "torchscript": false,
-    "typical_p": 1.0,
-    "use_bfloat16": false,
-    "use_cache": true,
     "vocab_size": 30000
   },
   "encoder": {
-    "_name_or_path": "",
-    "add_cross_attention": false,
-    "architectures": null,
-    "attn_drop_rate": 0.0,
-    "bad_words_ids": null,
-    "begin_suppress_tokens": null,
-    "bos_token_id": null,
-    "chunk_size_feed_forward": 0,
-    "conv_stem": false,
-    "cross_attention_hidden_size": null,
-    "decoder_start_token_id": null,
-    "depth": [
-      5,
-      8,
-      20,
-      7
-    ],
-    "diversity_penalty": 0.0,
-    "do_sample": false,
-    "drop_path_rate": 0.3,
-    "drop_rate": 0.0,
-    "early_stopping": false,
-    "embed_dim": [
-      64,
-      128,
-      320,
-      512
-    ],
-    "encoder_no_repeat_ngram_size": 0,
-    "eos_token_id": null,
-    "exponential_decay_length_penalty": null,
-    "finetuning_task": null,
-    "forced_bos_token_id": null,
-    "forced_eos_token_id": null,
-    "head_dim": 64,
-    "id2label": {
-      "0": "LABEL_0",
-      "1": "LABEL_1"
-    },
-    "image_size": 384,
-    "in_chans": 3,
-    "is_decoder": false,
-    "is_encoder_decoder": false,
-    "label2id": {
-      "LABEL_0": 0,
-      "LABEL_1": 1
-    },
-    "layer_norm_eps": 1e-06,
-    "length_penalty": 1.0,
-    "max_length": 20,
-    "min_length": 0,
-    "mlp_ratio": 4,
     "model_type": "vit",
-    "no_repeat_ngram_size": 0,
-    "num_beam_groups": 1,
-    "num_beams": 1,
-    "num_classes": 1000,
-    "num_return_sequences": 1,
-    "output_attentions": false,
-    "output_hidden_states": false,
-    "output_scores": false,
-    "pad_token_id": null,
-    "patch_size": [
-      4,
-      2,
-      2,
-      2
-    ],
-    "prefix": null,
-    "problem_type": null,
-    "projection_size": 768,
-    "pruned_heads": {},
-    "qk_scale": null,
-    "qkv_bias": true,
-    "remove_invalid_values": false,
-    "repetition_penalty": 1.0,
-    "representation_size": null,
-    "return_dict": true,
-    "return_dict_in_generate": false,
-    "sep_token_id": null,
-    "suppress_tokens": null,
-    "task_specific_params": null,
-    "temperature": 1.0,
-    "tf_legacy_loss": false,
-    "tie_encoder_decoder": false,
-    "tie_word_embeddings": true,
-    "tokenizer_class": null,
-    "top_k": 50,
-    "top_p": 1.0,
-    "torch_dtype": null,
-    "torchscript": false,
-    "typical_p": 1.0,
-    "use_bfloat16": false
+    "projection_size": 768
   },
-  "model_type": "cxrmate-ed",
+  "is_encoder_decoder": false,
+  "model_type": "vision-encoder-decoder",
   "tie_word_embeddings": false,
   "torch_dtype": "float32",
   "transformers_version": "4.39.3"

configuration_cxrmate_ed.py

@@ -1,23 +1,52 @@
+import transformers
 from transformers.configuration_utils import PretrainedConfig
 from transformers.utils import logging
 
+from .configuration_uniformer import UniFormerWithProjectionHeadConfig
+
 logger = logging.get_logger(__name__)
 
 
 class CXRMateEDConfig(PretrainedConfig):
 
-    model_type = "cxrmate-ed"
+    model_type = "vision-encoder-decoder"
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
-        if "encoder" not in kwargs or "decoder" not in kwargs:
-            raise ValueError(
-                f"A configuraton of type {self.model_type} cannot be instantiated because "
-                f"both `encoder` and `decoder` sub-configurations were not passed, only {kwargs}"
+
+        if 'decoder' not in kwargs:
+
+            self.decoder = transformers.LlamaConfig(
+                vocab_size=30000,
+                hidden_size=768,
+                intermediate_size=3072,
+                num_attention_heads=12,
+                num_hidden_layers=6,
+                max_position_embeddings=2048,
             )
+            self.decoder.is_decoder = True
+
+            self.decoder.index_value_encoder_intermediate_size = 2048
+            self.decoder.include_time_delta = True
+            self.decoder.time_delta_monotonic_inversion = True
+            self.decoder.add_time_deltas = True
+            self.decoder.history = 0
+            self.decoder.tables_filter = ["mimic_cxr_sectioned", "triage", "medrecon"]
+            self.decoder.prompt_report_sections_filter = ["indication", "history"]
+            self.decoder.pad_token_id = 4
+
+        else:
+            self.decoder = kwargs.pop("decoder")
+
+
+        if 'encoder' not in kwargs:
+            self.encoder = UniFormerWithProjectionHeadConfig(
+                projection_size=768,
+            )   
+        else:
+            self.encoder = kwargs.pop("encoder")
+        
 
-        self.encoder = kwargs.pop("encoder")
-        self.decoder = kwargs.pop("decoder")
         self.is_encoder_decoder = True
 
     @classmethod

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ffbf3e699a139ad98f20f8e057cd085586aea444b4b015471d697b43b440c14e
+oid sha256:dcee4d15b3c57def6d5179226c24381cf47f84ab19e3ed567b97dcce2e8804f5
 size 789958760