remove data

.gitignore

@@ -6,4 +6,7 @@
 root
 *.ipynb_checkpoints*
 .vscode
-/app/mymodels
+/app/mymodels
+/app/.cache
+/app/VectorStore*
+*chroma-embeddings.parquet*

app/7fd9b7dc7c8bd6c62fec03923bb04b6ba28e6c736017b43e67f3e808c92438c7.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:09bac093b25dfef86ce79a7325d893cf826fb0de7bcdf122b8ab0ec5692425c4
-size 3500346

app/VectorStore/chroma-embeddings.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:514430ced16df82f6b5355cc14ed912c5af38661418efb691ea8e73e6333ffed
-size 5782971
+oid sha256:df81409ab51d0acf63d8e0ad64f6af92daa192afb80667f08fe48fdcba095b1a
+size 9201921

app/VectorStore/index/id_to_uuid_52984ff2-d9c3-459b-acc0-0b0aa559d50f.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f39dc0dcfa56bb6584759d134c28bd53ac0165a2873cdd5b9e0ff70244840542
-size 131496
+oid sha256:25fca5ee26534c9df1cfaf6fe82d8aaea2c65bdc8e81e3c047c7546c7ea33466
+size 153298

app/VectorStore/index/index_52984ff2-d9c3-459b-acc0-0b0aa559d50f.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:de4cc0ee24b85680520f48ee0ee673443133127e18533255c60528cfe2f925be
-size 13050028
+oid sha256:111dc06e8480103cd32c60d5719a1091ac939ce09dbbdf0a979ed72590dd3095
+size 15211024

app/VectorStore/index/index_metadata_52984ff2-d9c3-459b-acc0-0b0aa559d50f.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:64da3bdfe4bc7727e421826a6459753a44eabcd37df7fe207fbde1014c0c2fe6
+oid sha256:950e65792fc96c61638586850536a6b92d603d0d8e357fa7028dee790eff793d
 size 74

app/VectorStore/index/uuid_to_id_52984ff2-d9c3-459b-acc0-0b0aa559d50f.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b153d0a7649253a0b5b095f1d126ba5a36b6a650e177b03393bd76cf8b399896
-size 153763
+oid sha256:d98cc469e10b5a1d092e046f96a528edf2f3b302da4340e56f27846728b5b853
+size 179261

app/fce3ad7ed281744c55c5838aa5677e15468ae6bae3aa5fe76e6aac20d2c19f2f.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a98a5f02d12b69d1b62de62ffef9fc98fbe229f2369e002e3f47ca78bdefeb3e
-size 29884059

app/load_vectors.py

@@ -94,7 +94,7 @@ def load_from_file(files):
 
 def load_from_web(urls, cache=True):
     docs_list = urls
-    filename=f"./{sha256(str(urls).encode('utf-8')).hexdigest()}.pkl"
+    filename=f"./.cache/{sha256(str(urls).encode('utf-8')).hexdigest()}.pkl"
 
     isFile = os.path.isfile(filename)
 

app/utils.py

@@ -4,7 +4,7 @@ from langchain.docstore.document import Document
 import chromadb
 from chromadb.config import Settings
 import load_model
-from load_vectors import load_from_file, load_and_split, create_and_add
+from load_vectors import load_from_file, load_and_split, create_and_add, load_from_web
 persist_directory = load_model.persist_directory
 
 def format_document(document: Document):
@@ -52,15 +52,31 @@ def load_files():
             retrieve_collections.clear()
             collections = retrieve_collections()
 
-        st.write('Source Documents:')
-        uploaded_files = st.file_uploader("Choose a PDF file", accept_multiple_files=True)
-        chunk_size = st.text_area('chunk Size:', 1000)
+        option = st.radio(
+            "",
+            options=["Upload Files", "Download Files"],
+        )
+        if option == "Upload Files":
+            st.write('Source Documents:')
+            uploaded_files = st.file_uploader("Choose a PDF file", accept_multiple_files=True)
+            chunk_size = st.text_area('chunk Size:', 1000)
+
+            if st.button('Upload'):
+                docs = load_from_file(uploaded_files)
+                sub_docs = load_and_split(docs, chunk_size=int(chunk_size))
+                create_and_add(selected_collection, sub_docs, "hkunlp/instructor-large")
+                uploaded_files=None
+        else:
+            st.write('Source Documents (Comma separated):')
+            urls = chunk_size = st.text_area('Urls:', '')
+            chunk_size = st.text_area('chunk Size:', 1000)
+            urls = urls.replace(",", "" ).replace('"', "" ).split(',')
 
-        if st.button('Upload'):
-            docs = load_from_file(uploaded_files)
-            sub_docs = load_and_split(docs, chunk_size=int(chunk_size))
-            create_and_add(selected_collection, sub_docs, "hkunlp/instructor-large")
-            uploaded_files=None
+            if st.button('Upload'):
+                docs = load_from_web(urls)     
+                sub_docs = load_and_split(docs, chunk_size=int(chunk_size))
+                create_and_add(selected_collection, sub_docs, "hkunlp/instructor-large")
+                uploaded_files=None
     else:
         collection = st.text_area('Name of your new collection:', '')
         if st.button('Create'):