Update src/backend/run_eval_suite.py
Browse files
src/backend/run_eval_suite.py
CHANGED
@@ -74,20 +74,29 @@ def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_siz
|
|
74 |
)
|
75 |
task_names = ["medmcqa", "medqa_4options", "mmlu_anatomy", "mmlu_clinical_knowledge", "mmlu_college_biology", "mmlu_college_medicine", "mmlu_medical_genetics", "mmlu_professional_medicine", "pubmedqa"]
|
76 |
|
|
|
77 |
print(f"Selected Tasks: {task_names}")
|
78 |
|
79 |
url = os.environ.get("URL")
|
80 |
|
81 |
-
headers = {
|
82 |
-
'bypass-tunnel-reminder': 'anyvalue'
|
83 |
-
}
|
84 |
-
|
85 |
data = {"args": f"pretrained={eval_request.model}"}
|
86 |
print("datasending", data)
|
87 |
-
response = requests.post(url, json=data
|
88 |
print("response, response", response)
|
89 |
results_full = {'results': {}, 'config': {}}
|
90 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
91 |
results_full['results'] = response.json()['result']['results']
|
92 |
results_full["config"]["model_dtype"] = eval_request.precision
|
93 |
results_full["config"]["model_name"] = eval_request.model
|
|
|
74 |
)
|
75 |
task_names = ["medmcqa", "medqa_4options", "mmlu_anatomy", "mmlu_clinical_knowledge", "mmlu_college_biology", "mmlu_college_medicine", "mmlu_medical_genetics", "mmlu_professional_medicine", "pubmedqa"]
|
76 |
|
77 |
+
|
78 |
print(f"Selected Tasks: {task_names}")
|
79 |
|
80 |
url = os.environ.get("URL")
|
81 |
|
|
|
|
|
|
|
|
|
82 |
data = {"args": f"pretrained={eval_request.model}"}
|
83 |
print("datasending", data)
|
84 |
+
response = requests.post(url, json=data)
|
85 |
print("response, response", response)
|
86 |
results_full = {'results': {}, 'config': {}}
|
87 |
|
88 |
+
# url = os.environ.get("URL")
|
89 |
+
|
90 |
+
# headers = {
|
91 |
+
# 'bypass-tunnel-reminder': 'anyvalue'
|
92 |
+
# }
|
93 |
+
|
94 |
+
# data = {"args": f"pretrained={eval_request.model}"}
|
95 |
+
# print("datasending", data)
|
96 |
+
# response = requests.post(url, json=data, headers=headers)
|
97 |
+
# print("response, response", response)
|
98 |
+
# results_full = {'results': {}, 'config': {}}
|
99 |
+
|
100 |
results_full['results'] = response.json()['result']['results']
|
101 |
results_full["config"]["model_dtype"] = eval_request.precision
|
102 |
results_full["config"]["model_name"] = eval_request.model
|