add application files

.gitignore

@@ -0,0 +1,5 @@
+.ipynb_checkpoints/
+Untitled.ipynb
+__pycache__/
+evaluate_trf.ipynb
+test.json

app.py

@@ -0,0 +1,100 @@
+from metrics import calc_metrics
+import gradio as gr
+from openai import OpenAI
+import os
+
+from transformers import pipeline
+# from dotenv import load_dotenv, find_dotenv
+import huggingface_hub
+import json
+# from simcse import SimCSE # use for gpt
+from evaluate_data import store_sample_data, get_metrics_trf
+
+store_sample_data()
+
+
+
+with open('./data/sample_data.json', 'r') as f:
+    # sample_data = [
+    #     {'id': "", 'text': "", 'orgs': ["", ""]}
+    # ]
+    sample_data = json.load(f)
+    
+# _ = load_dotenv(find_dotenv()) # read local .env file
+hf_token= os.environ['HF_TOKEN']
+huggingface_hub.login(hf_token)
+
+pipe = pipeline("token-classification", model="elshehawy/finer-ord-transformers", aggregation_strategy="first")
+
+
+llm_model = 'gpt-3.5-turbo-0125'
+# openai.api_key = os.environ['OPENAI_API_KEY']
+
+client = OpenAI(
+    api_key=os.environ.get("OPENAI_API_KEY"),
+)
+
+
+def get_completion(prompt, model=llm_model):
+    messages = [{"role": "user", "content": prompt}]
+    response = client.chat.completions.create(
+        messages=messages,
+        model=model,
+        temperature=0,
+    )
+    return response.choices[0].message.content
+
+
+
+def find_orgs_gpt(sentence):
+    prompt = f"""
+    In context of named entity recognition (NER), find all organizations in the text delimited by triple backticks.
+    
+    text:
+    ```
+    {sentence}
+    ```
+    You should output only a list of organizations and follow this output format exactly: ["org_1", "org_2", "org_3"]
+    """
+    
+    sent_orgs_str = get_completion(prompt)
+    sent_orgs = json.loads(sent_orgs_str)
+    
+    return sent_orgs
+
+
+    
+# def find_orgs_trf(sentence):
+#     org_list = []
+#     for ent in pipe(sentence):
+#         if ent['entity_group'] == 'ORG':
+#             # message += f'\n- {ent["word"]} \t- score: {ent["score"]}'
+#             # message += f'\n- {ent["word"]}'# \t- score: {ent["score"]}'
+#             org_list.append(ent['word'])
+#     return list(set(org_list))
+
+
+true_orgs = [sent['orgs'] for sent in sample_data]
+
+predicted_orgs_gpt = [find_orgs_gpt(sent['text']) for sent in sample_data]
+# predicted_orgs_trf = [find_orgs_trf(sent['text']) for sent in sample_data]
+
+all_metrics = {}
+
+# sim_model = SimCSE('sentence-transformers/all-MiniLM-L6-v2')
+# all_metrics['gpt'] = calc_metrics(true_orgs, predicted_orgs_gpt, sim_model)
+all_metrics['trf'] = get_metrics_trf()
+
+
+
+# example = """
+# My latest exclusive for The Hill : Conservative frustration over Republican efforts to force a House vote on reauthorizing the Export - Import Bank boiled over Wednesday during a contentious GOP meeting.
+
+# """
+def find_orgs(sentence, choice):
+    return all_metrics
+radio_btn = gr.Radio(choices=['GPT', 'iSemantics'], value='iSemantics', label='Available models', show_label=True)
+textbox = gr.Textbox(label="Enter your text", placeholder=str(all_metrics), lines=8)
+
+iface = gr.Interface(fn=find_orgs, inputs=[textbox, radio_btn], outputs="text",  examples=[[example]])
+iface.launch(share=True)

data/merged_dataset/dataset_dict.json

@@ -0,0 +1 @@
+{"splits": ["train", "validation", "test", "orig_train", "orig_validation", "orig_test"]}

data/merged_dataset/orig_test/cache-eeafde0b6770e328.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4b20ac141827d2e067e67afe6bb6efe6fdabf3d227c33b0764aff545c15ee6c
+size 953224

data/merged_dataset/orig_test/data-00000-of-00001.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc88467edb4babd0a7fc480903eed43b359e3755b5eecc87780fb33864530237
+size 437856

data/merged_dataset/orig_test/dataset_info.json

@@ -0,0 +1,34 @@
+{
+  "citation": "",
+  "description": "",
+  "features": {
+    "id": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "tokens": {
+      "feature": {
+        "dtype": "string",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "ner_tags": {
+      "feature": {
+        "names": [
+          "O",
+          "B-PER",
+          "I-PER",
+          "B-LOC",
+          "I-LOC",
+          "B-ORG",
+          "I-ORG"
+        ],
+        "_type": "ClassLabel"
+      },
+      "_type": "Sequence"
+    }
+  },
+  "homepage": "",
+  "license": ""
+}

data/merged_dataset/orig_test/state.json

@@ -0,0 +1,13 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00001.arrow"
+    }
+  ],
+  "_fingerprint": "0b33bf3dd398a19a",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": null
+}

data/merged_dataset/orig_train/cache-45d1543dc33c36be.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83332abe5b5e7d05e3ae4376018429896530b916ab3ff74eb8ca7aef94497961
+size 3009552

data/merged_dataset/orig_train/data-00000-of-00001.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac320be565428a08ad7a3c43d03ad14810775cb0620b47659321228b17a22148
+size 1371040

data/merged_dataset/orig_train/dataset_info.json

@@ -0,0 +1,34 @@
+{
+  "citation": "",
+  "description": "",
+  "features": {
+    "id": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "tokens": {
+      "feature": {
+        "dtype": "string",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "ner_tags": {
+      "feature": {
+        "names": [
+          "O",
+          "B-PER",
+          "I-PER",
+          "B-LOC",
+          "I-LOC",
+          "B-ORG",
+          "I-ORG"
+        ],
+        "_type": "ClassLabel"
+      },
+      "_type": "Sequence"
+    }
+  },
+  "homepage": "",
+  "license": ""
+}

data/merged_dataset/orig_train/state.json

@@ -0,0 +1,13 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00001.arrow"
+    }
+  ],
+  "_fingerprint": "74ec65c2b682826d",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": null
+}

data/merged_dataset/orig_validation/cache-afff9bbc07b5bee3.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94d972da8072255a5df65632859af45a7ce025dd587dc066106ea8e7224b0a1f
+size 387592

data/merged_dataset/orig_validation/data-00000-of-00001.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:758e0d9bfceefd51b3cef856c8e15786ce0493da10bdf231f27e067b6b66caec
+size 174712

data/merged_dataset/orig_validation/dataset_info.json

@@ -0,0 +1,34 @@
+{
+  "citation": "",
+  "description": "",
+  "features": {
+    "id": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "tokens": {
+      "feature": {
+        "dtype": "string",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "ner_tags": {
+      "feature": {
+        "names": [
+          "O",
+          "B-PER",
+          "I-PER",
+          "B-LOC",
+          "I-LOC",
+          "B-ORG",
+          "I-ORG"
+        ],
+        "_type": "ClassLabel"
+      },
+      "_type": "Sequence"
+    }
+  },
+  "homepage": "",
+  "license": ""
+}

data/merged_dataset/orig_validation/state.json

@@ -0,0 +1,13 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00001.arrow"
+    }
+  ],
+  "_fingerprint": "2b90f959ed79ba44",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": null
+}

data/merged_dataset/test/cache-3a6709085dd0f520.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28be4e971c5d73f16b208f4b15d5965cfc81fb8936ce1c711fedc6fff5b3479a
+size 953224

data/merged_dataset/test/cache-50fbc051d6b536f8.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:141c98d5d03e91f39e198897f147e0c2c6fa2c7a4c55174993392ec512599b34
+size 953224

data/merged_dataset/test/cache-7344e423192cdf30.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:855fbd5d5477353be5930ee9ed4435238d847ef0971abe8106056e8d93639cd8
+size 953240

data/merged_dataset/test/cache-861a0fd50d74bfe1.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:004f623f011a702e9eff454113818978e90f497b8ad806a8f86fa011868a0831
+size 12304024

data/merged_dataset/test/data-00000-of-00001.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc107fe614a3ee59fd5b302dc0a56896e63f2a3106fd88b5c52d4fd88b77a0fe
+size 437856

data/merged_dataset/test/dataset_info.json

@@ -0,0 +1,34 @@
+{
+  "citation": "",
+  "description": "",
+  "features": {
+    "id": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "tokens": {
+      "feature": {
+        "dtype": "string",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "ner_tags": {
+      "feature": {
+        "names": [
+          "O",
+          "B-PER",
+          "I-PER",
+          "B-LOC",
+          "I-LOC",
+          "B-ORG",
+          "I-ORG"
+        ],
+        "_type": "ClassLabel"
+      },
+      "_type": "Sequence"
+    }
+  },
+  "homepage": "",
+  "license": ""
+}

data/merged_dataset/test/state.json

@@ -0,0 +1,13 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00001.arrow"
+    }
+  ],
+  "_fingerprint": "538471187ad5b763",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": null
+}

data/merged_dataset/train/cache-f8f6a910898e33f3.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1060a3d1000579dc92f65efb200632efdafa80f5d750f0c2298d82193e648f3e
+size 3009552

data/merged_dataset/train/data-00000-of-00001.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1546be0dd9960d920988ce2bb6883fc567db03c2c80d0f8678d4bf95001a1a5f
+size 1371040

data/merged_dataset/train/dataset_info.json

@@ -0,0 +1,34 @@
+{
+  "citation": "",
+  "description": "",
+  "features": {
+    "id": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "tokens": {
+      "feature": {
+        "dtype": "string",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "ner_tags": {
+      "feature": {
+        "names": [
+          "O",
+          "B-PER",
+          "I-PER",
+          "B-LOC",
+          "I-LOC",
+          "B-ORG",
+          "I-ORG"
+        ],
+        "_type": "ClassLabel"
+      },
+      "_type": "Sequence"
+    }
+  },
+  "homepage": "",
+  "license": ""
+}

data/merged_dataset/train/state.json

@@ -0,0 +1,13 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00001.arrow"
+    }
+  ],
+  "_fingerprint": "13b20c4adf67dcf4",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": null
+}

data/merged_dataset/validation/cache-a70cdc1f600f2440.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0f9fa1d8fcd428b779d15f9f386d8b463c9f542dadb0056a16e3eb6b817cb5a
+size 387592

data/merged_dataset/validation/cache-c442280565074102.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca8b2d2e4610afe0cf6daf0ca37d02414e1df7e6a486c80e0ea2b25bf7808807
+size 387592

data/merged_dataset/validation/data-00000-of-00001.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fb30ccc26dc3d3172ffd54077d11217436ed169738251bf51fdb82908497868
+size 174712

data/merged_dataset/validation/dataset_info.json

@@ -0,0 +1,34 @@
+{
+  "citation": "",
+  "description": "",
+  "features": {
+    "id": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "tokens": {
+      "feature": {
+        "dtype": "string",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "ner_tags": {
+      "feature": {
+        "names": [
+          "O",
+          "B-PER",
+          "I-PER",
+          "B-LOC",
+          "I-LOC",
+          "B-ORG",
+          "I-ORG"
+        ],
+        "_type": "ClassLabel"
+      },
+      "_type": "Sequence"
+    }
+  },
+  "homepage": "",
+  "license": ""
+}

data/merged_dataset/validation/state.json

@@ -0,0 +1,13 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00001.arrow"
+    }
+  ],
+  "_fingerprint": "f95fe8e7a800be97",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": null
+}

data/ner_feature.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ecfd61f261845f22d0b83a72263f7326514d78d71d3c52534ede75671dacc70
+size 286

evaluate_model.py

@@ -0,0 +1,62 @@
+import evaluate
+import numpy as np
+import pickle
+
+metric = evaluate.load("seqeval")
+with open('./data/ner_feature.pickle', 'rb') as f:
+    ner_feature = pickle.load(f)
+    
+label_names = ner_feature.feature.names
+# label2id = {label: ner_feature.feature.str2int(label) for label in label_names}
+# id2label = {v: k for k, v in label2id.items()}
+
+def compute_metrics(eval_preds):
+    """
+    This compute_metrics() function first takes the argmax of the logits to convert them to predictions
+    (as usual, the logits and the probabilities are in the same order,
+    so we don’t need to apply the softmax).
+    Then we have to convert both labels and predictions from integers to strings.
+    We remove all the values where the label is -100, then pass the results to the metric.compute() method:
+    """
+
+    logits, labels = eval_preds
+    predictions = np.argmax(logits, axis=-1)
+
+    # Remove ignored index (special tokens) and convert to labels
+    true_labels = [[label_names[l] for l in label if l != -100] for label in labels]
+    true_predictions = [
+        [label_names[p] for (p, l) in zip(prediction, label) if l != -100]
+        for prediction, label in zip(predictions, labels)
+    ]
+    all_metrics = metric.compute(predictions=true_predictions, references=true_labels)
+    
+    # return all_metrics
+    # return {
+    #     "precision": all_metrics["overall_precision"],
+    #     "recall": all_metrics["overall_recall"],
+    #     "f1": all_metrics["overall_f1"],
+    #     "accuracy": all_metrics["overall_accuracy"],
+    # }
+
+    return {
+        # organization metrics
+        'org_precision': all_metrics['ORG']['precision'],
+        'org_recall': all_metrics['ORG']['recall'],
+        'org_f1': all_metrics['ORG']['f1'],
+        
+        # person metrics
+        'per_precision': all_metrics['PER']['precision'],
+        'per_recall': all_metrics['PER']['recall'],
+        'per_f1': all_metrics['PER']['f1'],
+        
+        # location metrics
+        'loc_precision': all_metrics['LOC']['precision'],
+        'loc_recall': all_metrics['LOC']['recall'],
+        'loc_f1': all_metrics['LOC']['f1'],
+        
+        # over all metrics
+        'precision': all_metrics['overall_precision'],
+        'recall': all_metrics['overall_recall'],
+        'f1': all_metrics['overall_f1'],
+        'accuracy': all_metrics['overall_accuracy']
+    }

metrics.py

@@ -0,0 +1,78 @@
+def calc_recall(true_pos, false_neg, eps=1e-8):
+    return true_pos / (true_pos + false_neg + eps)
+
+
+
+def calc_precision(true_pos, false_pos, eps=1e-8):
+    return true_pos / (true_pos + false_pos + eps)
+
+
+
+def calc_f1_score(precision, recall, eps=1e-8):
+    return (2*precision*recall) / (precision + recall + eps)
+
+
+
+def calc_metrics(true, predicted, model, threshold=0.95, eps=1e-8):
+    true_pos = 0
+    false_pos = 0
+    false_neg = 0
+    
+    false_pos_ids = []
+    false_neg_ids = []
+    
+    i = 0
+    total = len(true)
+    for j, (true_ents, pred_ents) in enumerate(zip(true, predicted)):
+        i += 1
+        # print(f'{i}/{total}')
+        # print('----------------------------')
+        
+        if len(true_ents) == 0:
+            false_pos += len(pred_ents)
+            
+            if len(pred_ents) > 0:
+                false_pos_ids.append(j)
+            
+            continue
+            
+        if len(pred_ents) == 0:
+            false_neg += len(true_ents)
+            
+            if len(true_ents) > 0:
+                # print('False Negative')
+                false_neg_ids.append(j)
+            
+            continue
+            
+        similarities = model.similarity(true_ents, pred_ents, device='cuda')
+        
+        for row in similarities:
+            if (row >= threshold).any():
+                true_pos += 1
+            else:
+                false_neg += 1
+                # print('False Negative 2222222')
+                false_neg_ids.append(j)
+
+        for row in similarities.T:
+            if (row >= threshold).any():
+                continue
+            else:
+                false_pos += 1
+                false_pos_ids.append(j)
+                
+    recall = calc_recall(true_pos, false_neg)
+    precision = calc_precision(true_pos, false_pos)
+    f1_score = calc_f1_score(precision, recall, eps=eps)
+    
+    return {
+        # 'true_pos': true_pos,
+        # 'false_pos': false_pos,
+        # 'false_neg': false_neg,
+        'recall': recall,
+        'precision': precision,
+        'f1': f1_score,
+        # 'false_pos_ids': list(set(false_pos_ids)),
+        # 'false_neg_ids': list(set(false_neg_ids))
+    }

requirements.txt

@@ -0,0 +1,7 @@
+openai
+transformers[torch]
+tqdm==4.66.1
+datasets==2.18.0
+evaluate
+seqeval
+rich

utils.py

@@ -0,0 +1,83 @@
+def find_broken_examples(data):
+    splits = list(data.keys())
+    broken = []
+    
+    for s in splits:
+        for i, tokens in enumerate(data[s]['tokens']):
+            for token in tokens:
+                if not token.isprintable():
+                    broken.append(s + '-' + str(i))
+    
+    return broken
+
+
+def update_data(examples, split, broken_ids):  
+    new_tags = []
+    new_tokens = []
+    for id_ in examples['id']:
+        sent_id = split + '-' + id_
+        if sent_id in broken_ids:
+            continue
+      
+        new_tokens.append(examples['tokens'][int(id_)])
+        new_tags.append(examples['ner_tags'][int(id_)])
+        
+        assert len(new_tokens) == len(new_tags)
+        assert len(new_tokens[-1]) == len(new_tags[-1])
+        
+    return {
+        'id': [str(i) for i in range(len(new_tokens))],
+        'tokens': new_tokens,
+        'ner_tags': new_tags
+    }
+
+
+def align_labels_with_tokens(labels, word_ids):
+    new_labels = []
+    current_word = None
+    for word_id in word_ids:
+        if word_id != current_word:
+            # Start of a new word!
+            current_word = word_id
+            label = -100 if word_id is None else labels[word_id]
+            new_labels.append(label)
+        elif word_id is None:
+            # Special token
+            new_labels.append(-100)
+        else:
+            # Same word as previous token
+            # label = labels[word_id]
+            # If the label is B-XXX we change it to I-XXX
+            # if label % 2 == 1:
+                # label += 1
+            label = -100
+            new_labels.append(label)
+
+    return new_labels
+
+
+def tokenize_and_align_labels(examples, tokenizer):
+    tokenized_inputs = tokenizer(
+        examples["tokens"], truncation=True, is_split_into_words=True, padding='max_length'
+    )
+    all_labels = examples["ner_tags"]
+    new_labels = []
+    word_ids = []
+    for i, labels in enumerate(all_labels):
+        word_ids.append(tokenized_inputs.word_ids(i))
+        new_labels.append(align_labels_with_tokens(labels, word_ids[i]))
+
+    tokenized_inputs["labels"] = new_labels
+    tokenized_inputs['word_ids'] = word_ids
+    
+    return tokenized_inputs
+
+
+# def model_init(checkpoint, id2label, label2id):
+#     model = AutoModelForTokenClassification.from_pretrained(
+#         checkpoint,
+#         id2label=id2label,
+#         label2id=label2id
+#     )
+
+#     return model