dl4ds_tutor / code /config.yml
XThomasBU
Code to add metadata to the chunks
f0018f2
raw
history blame
1.16 kB
embedding_options:
embedd_files: False # bool
data_path: 'storage/data' # str
url_file_path: 'storage/data/urls.txt' # str
expand_urls: False # bool
db_option : 'RAGatouille' # str [FAISS, Chroma, RAGatouille]
db_path : 'vectorstores' # str
model : 'sentence-transformers/all-MiniLM-L6-v2' # str [sentence-transformers/all-MiniLM-L6-v2, text-embedding-ada-002']
search_top_k : 3 # int
score_threshold : 0.2 # float
llm_params:
use_history: False # bool
memory_window: 3 # int
llm_loader: 'local_llm' # str [local_llm, openai]
openai_params:
model: 'gpt-4' # str [gpt-3.5-turbo-1106, gpt-4]
local_llm_params:
model: "storage/models/llama-2-7b-chat.Q4_0.gguf"
model_type: "llama"
temperature: 0.2
splitter_options:
use_splitter: True # bool
split_by_token : True # bool
remove_leftover_delimiters: True # bool
remove_chunks: False # bool
chunk_size : 300 # int
chunk_overlap : 30 # int
chunk_separators : ["\n\n", "\n", " ", ""] # list of strings
front_chunks_to_remove : null # int or None
last_chunks_to_remove : null # int or None
delimiters_to_remove : ['\t', '\n', ' ', ' '] # list of strings