loaded more docs

load_existing_docs.py

@@ -1,9 +1,11 @@
 import models
-import constants
-from langchain_experimental.text_splitter import SemanticChunker
+#import constants
+#from langchain_experimental.text_splitter import SemanticChunker
 from langchain_qdrant import QdrantVectorStore, Qdrant
-from langchain_community.document_loaders import PyPDFLoader
+from langchain_community.document_loaders import PyPDFLoader, UnstructuredURLLoader
 from qdrant_client.http.models import VectorParams
+import pymupdf
+import requests
 
 #qdrant = QdrantVectorStore.from_existing_collection(
 #    embedding=models.basic_embeddings,
@@ -11,26 +13,91 @@ from qdrant_client.http.models import VectorParams
 #    url=constants.QDRANT_ENDPOINT,
 #)
 
+def extract_links_from_pdf(pdf_path):
+    links = []
+    doc = pymupdf.open(pdf_path)
+    for page in doc:
+        for link in page.get_links():
+            if link['uri']:
+                links.append(link['uri'])
+    return links
+
+def load_documents_from_url(url):
+    try:
+        # Check if it's a PDF
+        if url.endswith(".pdf"):
+            try:
+                loader = PyPDFLoader(url)
+                return loader.load()
+            except Exception as e:
+                print(f"Error loading PDF from {url}: {e}")
+                return None
+        
+        # Fetch the content and check for video pages
+        try:
+            response = requests.head(url, timeout=10)  # Timeout for fetching headers
+            content_type = response.headers.get('Content-Type', '')
+        except Exception as e:
+            print(f"Error fetching headers from {url}: {e}")
+            return None
+        
+        # Ignore video content (flagged for now)
+        if 'video' in content_type:
+            return None
+        if 'youtube' in url:
+            return None
+        
+        # Otherwise, treat it as an HTML page
+        try:
+            loader = UnstructuredURLLoader([url])
+            return loader.load()
+        except Exception as e:
+            print(f"Error loading HTML from {url}: {e}")
+            return None
+    except Exception as e:
+        print(f"General error loading from {url}: {e}")
+        return None
+    
 
 #gather kai's docs
-filepaths = ["./test_docs/Employee Statistics FINAL.pdf","./test_docs/Employer Statistics FINAL.pdf"]
-all_documents = []
-for file in filepaths:
-    loader = PyPDFLoader(file)
-    documents = loader.load()
-    for doc in documents:
-        doc.metadata = {
-            "source": file,
-            "tag": "employee" if "employee" in file.lower() else "employer"
-        }
-    all_documents.extend(documents) 
+filepaths = ["./test_docs/Employee Statistics FINAL.pdf","./test_docs/Employer Statistics FINAL.pdf","./test_docs/Articles To Share.pdf"]
+
+all_links = []
+for pdf_path in filepaths:
+    all_links.extend(extract_links_from_pdf(pdf_path))
+
+unique_links = list(set(all_links))
+print(unique_links)
+
+documents = []
+for link in unique_links:
+    doc = load_documents_from_url(link)
+    #print(f"loaded doc from {link}")
+    if doc:
+        documents.extend(doc)
+
+
+#print(len(documents))
+semantic_split_docs = models.semanticChunker.split_documents(documents)
+RCTS_split_docs = models.RCTS.split_documents(documents)
+
+
+#for file in filepaths:
+#    loader = PyPDFLoader(file)
+#    documents = loader.load()
+#    for doc in documents:
+#        doc.metadata = {
+#            "source": file,
+#            "tag": "employee" if "employee" in file.lower() else "employer"
+#        }
+#    all_documents.extend(documents) 
 
 #chunk them
-semantic_split_docs = models.semanticChunker.split_documents(all_documents)
+#semantic_split_docs = models.semanticChunker.split_documents(all_documents)
 
 
 #add them to the existing qdrant client
-collection_name = "kai_test_docs"
+collection_name = "docs_from_ripped_urls_recursive"
 
 collections = models.qdrant_client.get_collections()
 collection_names = [collection.name for collection in collections.collections]
@@ -41,13 +108,13 @@ if collection_name not in collection_names:
         vectors_config=VectorParams(size=1536, distance="Cosine") 
     )
 
-qdrant_vector_store = Qdrant(
+qdrant_vector_store = QdrantVectorStore(
     client=models.qdrant_client,
     collection_name=collection_name,
-    embeddings=models.te3_small
+    embedding=models.te3_small
 )
 
-qdrant_vector_store.add_documents(semantic_split_docs)
+qdrant_vector_store.add_documents(RCTS_split_docs)
 
 
 

models.py

@@ -3,11 +3,11 @@ from langchain_openai import ChatOpenAI
 from langchain.callbacks.manager import CallbackManager
 from langchain.callbacks.tracers import LangChainTracer
 from langchain_huggingface.embeddings import HuggingFaceEmbeddings
-from langchain_community.embeddings import HuggingFaceInstructEmbeddings
 from langchain_experimental.text_splitter import SemanticChunker
 from langchain_openai.embeddings import OpenAIEmbeddings
 from langchain_community.vectorstores import Qdrant
 from qdrant_client import QdrantClient
+from langchain_text_splitters import RecursiveCharacterTextSplitter
 import constants
 import os
 
@@ -24,7 +24,7 @@ opus3 = ChatAnthropic(
     api_key=constants.ANTRHOPIC_API_KEY, 
     temperature=0, 
     model='claude-3-opus-20240229',
-    callback_manager=callback_manager
+    callbacks=callback_manager
 )
 
 sonnet35 = ChatAnthropic(
@@ -32,7 +32,7 @@ sonnet35 = ChatAnthropic(
     temperature=0, 
     model='claude-3-5-sonnet-20240620',
     max_tokens=4096,
-    callback_manager=callback_manager
+    callbacks=callback_manager
 )
 
 gpt4 = ChatOpenAI(
@@ -42,7 +42,7 @@ gpt4 = ChatOpenAI(
     timeout=None,
     max_retries=2,
     api_key=constants.OPENAI_API_KEY,
-    callback_manager=callback_manager
+    callbacks=callback_manager
 )
 
 gpt4o = ChatOpenAI(
@@ -52,7 +52,7 @@ gpt4o = ChatOpenAI(
     timeout=None,
     max_retries=2,
     api_key=constants.OPENAI_API_KEY,
-    callback_manager=callback_manager
+    callbacks=callback_manager
 )
 
 gpt4o_mini = ChatOpenAI(
@@ -62,7 +62,7 @@ gpt4o_mini = ChatOpenAI(
     timeout=None,
     max_retries=2,
     api_key=constants.OPENAI_API_KEY,
-    callback_manager=callback_manager
+    callbacks=callback_manager
 )
 
 basic_embeddings = HuggingFaceEmbeddings(model_name="snowflake/snowflake-arctic-embed-l")
@@ -76,4 +76,11 @@ te3_small = OpenAIEmbeddings(api_key=constants.OPENAI_API_KEY, model="text-embed
 semanticChunker = SemanticChunker(
     te3_small,
     breakpoint_threshold_type="percentile"
+)
+
+RCTS = RecursiveCharacterTextSplitter(
+    # Set a really small chunk size, just to show.
+    chunk_size=500,
+    chunk_overlap=25,
+    length_function=len,
 )