Update services/embed_service/utils.py

services/embed_service/utils.py

@@ -160,30 +160,7 @@ class DocumentEmbedder:
         # basic items
         self.files = files
         #self.interactive = interactive
-    
-    def clean_text(self, content: str) -> str:
-        """
-        Remove unwanted characters and patterns in text input.
-        :param content: Text input.
-        :return: Cleaned version of original text input.
-        """
-
-        # Fix hyphenated words broken by newline
-        content = re.sub(r'(\w+)-\n(\w+)', r'\1\2', content)
-
-        # Remove specific unwanted patterns and characters
-        unwanted_patterns = [
-            "\\n", "  —", "——————————", "—————————", "—————",
-            r'\\u[\dA-Fa-f]{4}', r'\uf075', r'\uf0b7'
-        ]
-        for pattern in unwanted_patterns:
-            content = re.sub(pattern, "", content)
-
-        # Fix improperly spaced hyphenated words and normalize whitespace
-        content = re.sub(r'(\w)\s*-\s*(\w)', r'\1-\2', content)
-        content = re.sub(r'\s+', ' ', content)
-
-        return content
+        
     
     def embed(self):
         """stringing process above to embed and upsert directly to pinecone"""