Update app.py
Browse files
app.py
CHANGED
@@ -59,7 +59,10 @@ def create_retriever_from_chroma(vectorstore_path="./docs/chroma/", search_type=
|
|
59 |
docs = loader.load()
|
60 |
st.write("Docs loaded")
|
61 |
|
62 |
-
text_splitter = RecursiveCharacterTextSplitter(
|
|
|
|
|
|
|
63 |
split_docs = text_splitter.split_documents(docs)
|
64 |
|
65 |
|
|
|
59 |
docs = loader.load()
|
60 |
st.write("Docs loaded")
|
61 |
|
62 |
+
text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
|
63 |
+
chunk_size=chunk_size, chunk_overlap=chunk_overlap,
|
64 |
+
separators=["\n \n \n", "\n \n", "\n1" , "(?<=\. )", " ", ""]
|
65 |
+
)
|
66 |
split_docs = text_splitter.split_documents(docs)
|
67 |
|
68 |
|