Update

.gitignore

@@ -0,0 +1,2 @@
+.DS_Store
+__pycache__

data_loader.py

@@ -64,11 +64,12 @@ settings = {
     "Chain-of-Thought": "cot",
     "Fairness": "fairness",
     "Robustness": "robustness",
+    "Random Order Choices": "randchoice",
 }
 
 task_w_settings = {
     "Information Retrieval": ["Normal", "Few-shot Leanring", "Robustness", "Fairness"],
-    "Knowledge": ["Normal", "Few-shot Leanring", "Robustness"],
+    "Knowledge": ["Normal", "Few-shot Leanring", "Robustness", "Random Order Choices"],
     "Language Modelling": ["Normal", "Few-shot Leanring", "Fairness"],
     "Question Answering": ["Prompt Strategy 0", "Prompt Strategy 1", "Prompt Strategy 2", "Robustness", "Fairness"],
     "Reasoning": ["Few-shot Leanring", "Chain-of-Thought"],
@@ -121,8 +122,6 @@ datasets = {
     "reasoning": {
         "srnatural-azr": "Synthetic Reasoning (Natural) - Azure",
         "srnatural-gcp": "Synthetic Reasoning (Natural) - Google Cloud",
-        "srabstract-azr": "Synthetic Reasoning (Abstract Symbol)- Azure",
-        "srabstract-gcp": "Synthetic Reasoning (Abstract Symbol)- Google Cloud",
         "srinduction-azr": "Synthetic Reasoning (Induction) - Azure",
         "srinduction-gcp": "Synthetic Reasoning (Induction) - Google Cloud",
         "srpattern-azr": "Synthetic Introduction (Pattern Match) - Azure",