Update services/embed_service/utils.py

services/embed_service/utils.py

@@ -9,6 +9,30 @@ from llama_index import Document
 import gc
 import re
 
+def clean_text(content: str) -> str:
+    """
+    Remove unwanted characters and patterns in text input.
+    :param content: Text input.
+    :return: Cleaned version of original text input.
+    """
+
+    # Fix hyphenated words broken by newline
+    content = re.sub(r'(\w+)-\n(\w+)', r'\1\2', content)
+
+    # Remove specific unwanted patterns and characters
+    unwanted_patterns = [
+        "\\n", "  —", "——————————", "—————————", "—————",
+        r'\\u[\dA-Fa-f]{4}', r'\uf075', r'\uf0b7'
+    ]
+    for pattern in unwanted_patterns:
+        content = re.sub(pattern, "", content)
+
+    # Fix improperly spaced hyphenated words and normalize whitespace
+    content = re.sub(r'(\w)\s*-\s*(\w)', r'\1-\2', content)
+    content = re.sub(r'\s+', ' ', content)
+
+    return content
+
 
 class VTTTranscriptLoader:
     """
@@ -171,7 +195,7 @@ class DocumentEmbedder:
         # Call clean function
         print("cleaning files")
         for d in range(len(results)):
-            results[d].text = self.clean_text(results[d].text)
+            results[d].text = clean_text(results[d].text)
 
         # run pinecone in batches (of 1) for memory preservation.
         print("reading into pinecone db")