File size: 1,126 Bytes
6158da4
ce9ef3e
b83cc65
 
 
6158da4
 
 
b83cc65
57b7b8d
6158da4
57b7b8d
db6b619
b83cc65
6158da4
 
b83cc65
 
6158da4
b83cc65
6158da4
 
 
 
 
b83cc65
 
6158da4
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
embedding_options:
  embedd_files: False # bool
  data_path: 'storage/data' # str
  url_file_path: 'storage/data/urls.txt' # str
  expand_urls: True # bool
  db_option : 'FAISS' # str
  db_path : 'vectorstores' # str
  model : 'sentence-transformers/all-MiniLM-L6-v2' # str [sentence-transformers/all-MiniLM-L6-v2, text-embedding-ada-002']
  search_top_k : 3 # int
  score_threshold : 0.5 # float
llm_params: 
  use_history: True # bool
  memory_window: 3 # int
  llm_loader: 'local_llm' # str [local_llm, openai]
  openai_params:
    model: 'gpt-4' # str [gpt-3.5-turbo-1106, gpt-4]
  local_llm_params:
    model: "storage/models/llama-2-7b-chat.Q4_0.gguf"
    model_type: "llama"
    temperature: 0.2
splitter_options:
  use_splitter: True # bool
  split_by_token : True # bool
  remove_leftover_delimiters: True # bool
  remove_chunks: False # bool
  chunk_size : 300 # int
  chunk_overlap : 30 # int
  chunk_separators : ["\n\n", "\n", " ", ""] # list of strings
  front_chunks_to_remove : null # int or None
  last_chunks_to_remove : null # int or None
  delimiters_to_remove : ['\t', '\n', '   ', '  '] # list of strings