Upload 6 files

ner.py

@@ -0,0 +1,53 @@
+import json
+from transformers import AutoTokenizer, AutoModelForTokenClassification
+import nltk
+nltk.download('punkt')
+from nltk.tokenize import word_tokenize
+
+# Function to extract names using MAREFA NER model
+def extract_arabic_names(json_data, model, tokenizer):
+    arabic_names = set()
+
+    for entry in json_data:
+        if "Arabic Text" in entry:
+            text = entry["Arabic Text"]
+            tokenized_text = tokenizer.tokenize(text)
+            inputs = tokenizer(text, return_tensors="pt")
+            outputs = model(**inputs)
+            predictions = outputs.logits.argmax(dim=-1)
+            predicted_labels = [model.config.id2label[label_id] for label_id in predictions[0]]
+            
+            current_name = ""
+            for token, label in zip(tokenized_text, predicted_labels):
+                if label == "B-person":
+                    current_name = token
+                elif label == "I-person":
+                    current_name += " " + token
+                elif label != "O" and current_name:
+                    arabic_names.add(current_name)
+                    current_name = ""
+
+            if current_name:
+                arabic_names.add(current_name)
+
+    return arabic_names
+
+# Load the MAREFA NER model and tokenizer
+model_name = "marefa-nlp/marefa-ner"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForTokenClassification.from_pretrained(model_name)
+basic='cache/output/basic_info_frame.json'
+# Load JSON data from the file
+with open(basic, "r", encoding="utf-8") as file:
+    json_data = json.load(file)
+
+# Extract names from the JSON data using MAREFA model
+arabic_names = extract_arabic_names(json_data, model, tokenizer)
+
+# Print the extracted names
+if arabic_names:
+    print("Arabic names extracted:")
+    for name in arabic_names:
+        print("Name:", name)
+else:
+    print("No Arabic names found.")

ner_camel_MSA.py

@@ -0,0 +1,57 @@
+# Load model directly
+from transformers import AutoTokenizer, AutoModelForTokenClassification
+import json 
+
+#tokenizer = AutoTokenizer.from_pretrained("CAMeL-Lab/bert-base-arabic-camelbert-msa-ner")
+#model = AutoModelForTokenClassification.from_pretrained("CAMeL-Lab/bert-base-arabic-camelbert-msa-ner")
+#print("Model loaded successfully")
+
+from nltk.tokenize import word_tokenize
+
+# Function to extract names using MAREFA NER model
+def extract_arabic_names(json_data, model, tokenizer):
+    arabic_names = set()
+
+    for entry in json_data:
+        if "Arabic Text" in entry:
+            text = entry["Arabic Text"]
+            tokenized_text = tokenizer.tokenize(text)
+            inputs = tokenizer(text, return_tensors="pt")
+            outputs = model(**inputs)
+            predictions = outputs.logits.argmax(dim=-1)
+            predicted_labels = [model.config.id2label[label_id] for label_id in predictions[0]]
+            
+            current_name = ""
+            for token, label in zip(tokenized_text, predicted_labels):
+                if label == "B-person":
+                    current_name = token
+                elif label == "I-person":
+                    current_name += " " + token
+                elif label != "O" and current_name:
+                    arabic_names.add(current_name)
+                    current_name = ""
+
+            if current_name:
+                arabic_names.add(current_name)
+
+    return arabic_names
+
+# Load the MAREFA NER model and tokenizer
+model_name = "CAMeL-Lab/bert-base-arabic-camelbert-msa-ner"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForTokenClassification.from_pretrained(model_name)
+basic='cache/output/basic_info_frame.json'
+# Load JSON data from the file
+with open(basic, "r", encoding="utf-8") as file:
+    json_data = json.load(file)
+
+# Extract names from the JSON data using MAREFA model
+arabic_names = extract_arabic_names(json_data, model, tokenizer)
+
+# Print the extracted names
+if arabic_names:
+    print("Arabic names extracted:")
+    for name in arabic_names:
+        print("Name:", name)
+else:
+    print("No Arabic names found.")

pdf_to_imgs.py

@@ -0,0 +1,15 @@
+import fitz 
+import os 
+pdf_file_path="sample/GB.pdf"
+pdf=fitz.open(pdf_file_path)
+
+save_dir="cache/GB"
+os.makedirs(save_dir,exist_ok=True)
+for page_num in range(len(pdf)):
+    page=pdf[page_num]
+    pix=page.get_pixmap()
+    image_filename=os.path.join(save_dir,f"{os.path.splitext(os.path.basename(pdf_file_path))[0]}_page_{page_num+1}.png")
+    pix.save(image_filename)
+    
+
+pdf.close()

regex_extract.py

@@ -0,0 +1,102 @@
+import json 
+import csv
+from translate import Translator
+#print("translate imported")
+import re
+def load_regex_pattern(filename):
+    try:
+        with open(filename,'r',encoding="utf-8") as config_file:
+            config_data=json.load(config_file)
+            return config_data
+    except FileNotFoundError:
+        print("regex file not found")
+        return{}
+    
+def translate_date(date_text):
+    translation_dict={
+ "January":"يناير", "February":"فبراير",
+    "March":"مارس",
+    "April":"ابريل",
+    "May":"مايو",
+    "June":"يونيو",
+    "July":"يوليو",
+    "August":"أغسطس",
+    "September":"سبتمبر",
+    "October":"اكتوبر",
+    "November":"نوفمبر",
+    "ديسمبر":"December",
+    "٠":"0",
+"١":"1",
+"٢":"2",
+"٣":"3",
+"٤":"4",
+"٥":"5",
+"٦":"6",
+"٧":"7",
+"٨":"8",
+"٩":"9"
+
+}
+    #mapping digits and months into english
+    for ar_digit,en_digit in translation_dict.items():
+        date_text=date_text.replace(ar_digit,en_digit)
+
+    for ar_month,en_month in translation_dict.items():
+        date_text=date_text.replace(ar_month,en_month)
+    return date_text
+
+def translate_text(text):
+    translator=Translator(to_lang='en',from_lang='ar')
+    translated_text=translator.translate(text)
+    return translated_text
+
+def extract_and_store_info(input_file,output_csv,regex_patterns):
+    extracted_data={pattern_name:"" for pattern_name in regex_patterns}
+    with open(input_file,encoding="utf-8") as json_file:
+        json_data=json.load(json_file)
+
+    for pattern_name,pattern_data in regex_patterns.items():
+        if not extracted_data.get(pattern_name):
+            for entry in json_data:
+                if "Arabic text" in entry:
+                    text=entry.get("Arabic text","")
+                    if not extracted_data.get(pattern_name) and re.search(pattern_data["pattern"],text,re.IGNORECASE):
+                        extracted_data[pattern_name]=re.search(pattern_data["pattern"],text,re.IGNORECASE).group()
+
+                    if extracted_data.get(pattern_name):
+                        break
+
+
+#translate the company name into english 
+    if extracted_data.get("company_title"):
+        extracted_data["company_title"]=translate_text(extracted_data["company_title"])
+
+    if "annual_pattern" in regex_patterns and "date_pattern" in regex_patterns:
+        if re.search(regex_patterns["annual_pattern"]["pattern"],extracted_data["annual_pattern"],re.IGNORECASE):
+            extracted_data["annual_pattern"]="Annual"
+        elif re.search(regex_patterns["half_annual_pattern"]["pattern"],extracted_data["half_annual_pattern"],re.IGNORECASE):
+            extracted_data["half_annual_pattern"]="Half Annual"
+
+    if "date_pattern" in regex_patterns:
+        extracted_data["date_pattern"]=translate_date(extracted_data["date_pattern"])
+
+    with open(output_csv,mode='w',encoding="utf-8",newline='') as csv_output_file:
+        fieldnames=["pattern_name","extracted_data"]
+        writer=csv.DictWriter(csv_output_file,fieldnames=fieldnames)
+        writer.writeheader()
+        for pattern_name ,data in extracted_data.items():
+            if data:
+                writer.writerow({"pattern_name":pattern_name,"extracted_data":data})
+    extracted_count=sum(1 for data in extracted_data.values() if data)
+    print(f"{extracted_count} pieces of data extracted and stored in:",output_csv)
+
+
+if __name__=="__main__":
+    input_file="cache/output/basic_info_frame.json"
+    output_csv_file='cache/output/outputregex.csv'
+    regex_patterns=load_regex_pattern("mainpipeline/models/regex_config.json")
+    if regex_patterns:
+        extract_and_store_info(input_file,output_csv_file,regex_patterns)
+    else:
+        print("failed to load regex")
+

regex_format_after_OCR.py

@@ -0,0 +1,25 @@
+import json
+import glob
+def extract_arabic_data(input_files,output_file):
+    arabic_data=[]
+    for input_file in input_files:
+        #load json data from the input file 
+        with open(input_file,encoding="utf-8") as json_file:
+            json_data=json.load(json_file)
+
+        #extract only the arabic data 
+        for entry in json_data:
+            if "lines" in entry:
+                for line in entry["lines"]:
+                    text=line.get("text","")
+                    arabic_data.append({"Arabic text":text})
+    
+
+    with open(output_file,mode="w",encoding="utf-8") as json_output_file:
+        json.dump(arabic_data,json_output_file,ensure_ascii=False,indent=4)
+    print("Arabic data from",len(input_files),"json files has been extracted in :",output_file)
+
+if __name__ =="__main__":
+    input_files=glob.glob("cache/GB/ocr_output*.json")
+    output_json_file="cache/output/basic_info_frame.json"
+    extract_arabic_data(input_files,output_json_file)

stamp_detection.py

@@ -0,0 +1,26 @@
+from ultralytics import YOLO
+import os 
+import csv
+model=YOLO('mainpipeline/models/stamp_detection_model.pt')
+img_dir='cache/GB'
+#output_csv='cache/extracted_data.csv'
+#set a detection flag = false 
+output_csv='cache/output/appended_data.csv'
+revision_status='unrevised'
+
+for img_name in os.listdir(img_dir):
+    img_path=os.path.join(img_dir,img_name)
+    if os.path.isfile(img_path):
+        results=model.predict(img_path,conf=0.25,save=False)
+        if len(results)>0 :
+            revision_status="Revised"
+            break
+with open(output_csv,mode='a',newline='') as file:
+    writer=csv.writer(file)
+    #writer.writerow(['Revision status',revision_status])
+    file.seek(0,os.SEEK_END)
+    if file.tell()==0:
+       writer.writerow(["Revision_status"])
+    writer.writerow(["revision_status",revision_status])
+
+