ArturG9 commited on
Commit
91fbf6f
1 Parent(s): e1ca13f

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +4 -1
app.py CHANGED
@@ -59,7 +59,10 @@ def create_retriever_from_chroma(vectorstore_path="./docs/chroma/", search_type=
59
  docs = loader.load()
60
  st.write("Docs loaded")
61
 
62
- text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap,separators=["\n \n \n", "\n \n", "\n1" , "(?<=\. )", " ", ""])
 
 
 
63
  split_docs = text_splitter.split_documents(docs)
64
 
65
 
 
59
  docs = loader.load()
60
  st.write("Docs loaded")
61
 
62
+ text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
63
+ chunk_size=chunk_size, chunk_overlap=chunk_overlap,
64
+ separators=["\n \n \n", "\n \n", "\n1" , "(?<=\. )", " ", ""]
65
+ )
66
  split_docs = text_splitter.split_documents(docs)
67
 
68