data/raw-eval-outputs/Qwen-Qwen2-7B_results.json

{
  "results": {
    "b4b": {
      "acc,none": 0.7028360049321827,
      "acc_stderr,none": 0.1004832322485701,
      "acc_norm,none": 0.7028360049321827,
      "acc_norm_stderr,none": 0.1004832322485701,
      "alias": "b4b"
    },
    "b4bqa": {
      "acc,none": 0.8041294642857143,
      "acc_stderr,none": 0.009377773744245437,
      "acc_norm,none": 0.8041294642857143,
      "acc_norm_stderr,none": 0.009377773744245437,
      "alias": " - b4bqa"
    },
    "medmcqa_g2b": {
      "acc,none": 0.5517241379310345,
      "acc_stderr,none": 0.02669739777037782,
      "acc_norm,none": 0.5517241379310345,
      "acc_norm_stderr,none": 0.02669739777037782,
      "alias": " - medmcqa_g2b"
    },
    "medmcqa_orig_filtered": {
      "acc,none": 0.6350574712643678,
      "acc_stderr,none": 0.025843659831273274,
      "acc_norm,none": 0.6350574712643678,
      "acc_norm_stderr,none": 0.025843659831273274,
      "alias": " - medmcqa_orig_filtered"
    },
    "medqa_4options_g2b": {
      "acc,none": 0.5370370370370371,
      "acc_stderr,none": 0.025680564640056882,
      "acc_norm,none": 0.5370370370370371,
      "acc_norm_stderr,none": 0.025680564640056882,
      "alias": " - medqa_4options_g2b"
    },
    "medqa_4options_orig_filtered": {
      "acc,none": 0.58994708994709,
      "acc_stderr,none": 0.025331202438944444,
      "acc_norm,none": 0.58994708994709,
      "acc_norm_stderr,none": 0.025331202438944444,
      "alias": " - medqa_4options_orig_filtered"
    }
  },
  "groups": {
    "b4b": {
      "acc,none": 0.7028360049321827,
      "acc_stderr,none": 0.1004832322485701,
      "acc_norm,none": 0.7028360049321827,
      "acc_norm_stderr,none": 0.1004832322485701,
      "alias": "b4b"
    }
  },
  "configs": {
    "b4bqa": {
      "task": "b4bqa",
      "dataset_path": "AIM-Harvard/b4b_drug_qa",
      "test_split": "test",
      "doc_to_text": "<function process_cd at 0x7f8319c60ee0>",
      "doc_to_target": "correct_choice",
      "doc_to_choice": [
        "A",
        "B",
        "C",
        "D"
      ],
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        },
        {
          "metric": "acc_norm",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false
    },
    "medmcqa_g2b": {
      "task": "medmcqa_g2b",
      "dataset_path": "AIM-Harvard/medmcqa_generic_to_brand",
      "training_split": "train",
      "validation_split": "validation",
      "test_split": "validation",
      "doc_to_text": "<function doc_to_text at 0x7f831a19e3a0>",
      "doc_to_target": "cop",
      "doc_to_choice": [
        "A",
        "B",
        "C",
        "D"
      ],
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        },
        {
          "metric": "acc_norm",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": true,
      "doc_to_decontamination_query": "{{question}}"
    },
    "medmcqa_orig_filtered": {
      "task": "medmcqa_orig_filtered",
      "dataset_path": "AIM-Harvard/medmcqa_original",
      "training_split": "train",
      "validation_split": "validation",
      "test_split": "validation",
      "doc_to_text": "<function doc_to_text at 0x7f8319ac8310>",
      "doc_to_target": "cop",
      "doc_to_choice": [
        "A",
        "B",
        "C",
        "D"
      ],
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        },
        {
          "metric": "acc_norm",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": true,
      "doc_to_decontamination_query": "{{question}}"
    },
    "medqa_4options_g2b": {
      "task": "medqa_4options_g2b",
      "dataset_path": "AIM-Harvard/gbaker_medqa_usmle_4_options_hf_generic_to_brand",
      "training_split": "train",
      "validation_split": "validation",
      "test_split": "test",
      "doc_to_text": "<function doc_to_text at 0x7f831a19e820>",
      "doc_to_target": "<function doc_to_target at 0x7f831a19eb80>",
      "doc_to_choice": [
        "A",
        "B",
        "C",
        "D"
      ],
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        },
        {
          "metric": "acc_norm",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false
    },
    "medqa_4options_orig_filtered": {
      "task": "medqa_4options_orig_filtered",
      "dataset_path": "AIM-Harvard/gbaker_medqa_usmle_4_options_hf_original",
      "training_split": "train",
      "validation_split": "validation",
      "test_split": "test",
      "doc_to_text": "<function doc_to_text at 0x7f8319c844c0>",
      "doc_to_target": "<function doc_to_target at 0x7f8319c30ee0>",
      "doc_to_choice": [
        "A",
        "B",
        "C",
        "D"
      ],
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        },
        {
          "metric": "acc_norm",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false
    }
  },
  "versions": {
    "b4b": "N/A",
    "b4bqa": "Yaml",
    "medmcqa_g2b": "Yaml",
    "medmcqa_orig_filtered": "Yaml",
    "medqa_4options_g2b": "Yaml",
    "medqa_4options_orig_filtered": "Yaml"
  },
  "n-shot": {
    "b4b": 0,
    "b4bqa": 0,
    "medmcqa_g2b": 0,
    "medmcqa_orig_filtered": 0,
    "medqa_4options_g2b": 0,
    "medqa_4options_orig_filtered": 0
  },
  "config": {
    "model": "hf",
    "model_args": "pretrained=Qwen/Qwen2-7B,load_in_4bit=True",
    "batch_size": "auto:64",
    "batch_sizes": [
      8,
      8,
      16,
      16,
      16,
      16,
      16,
      16,
      16,
      16,
      16,
      16,
      32,
      32,
      32,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64,
      64
    ],
    "device": "cuda:0",
    "use_cache": null,
    "limit": null,
    "bootstrap_iters": 100000,
    "gen_kwargs": null
  },
  "git_hash": "928c7657"
}