Tuchuanhuhuhu commited on
Commit
1d87c8b
1 Parent(s): 2c40dcd

修改检测是否同一个文件的方式为检测文件的md5哈希值

Browse files
Files changed (2) hide show
  1. modules/llama_func.py +11 -7
  2. modules/utils.py +0 -6
modules/llama_func.py CHANGED
@@ -17,13 +17,17 @@ from modules.presets import *
17
  from modules.utils import *
18
 
19
  def get_index_name(file_src):
20
- index_name = []
21
- for file in file_src:
22
- index_name.append(os.path.basename(file.name))
23
- index_name = sorted(index_name)
24
- index_name = "".join(index_name)
25
- index_name = sha1sum(index_name)
26
- return index_name
 
 
 
 
27
 
28
  def get_documents(file_src):
29
  documents = []
 
17
  from modules.utils import *
18
 
19
  def get_index_name(file_src):
20
+ file_paths = [x.name for x in file_src]
21
+ file_paths.sort(key=lambda x: os.path.basename(x))
22
+
23
+ md5_hash = hashlib.md5()
24
+ for file_path in file_paths:
25
+ with open(file_path, "rb") as f:
26
+ while chunk := f.read(8192):
27
+ md5_hash.update(chunk)
28
+
29
+ return md5_hash.hexdigest()
30
+
31
 
32
  def get_documents(file_src):
33
  documents = []
modules/utils.py CHANGED
@@ -367,12 +367,6 @@ def submit_key(key):
367
  return key, msg
368
 
369
 
370
- def sha1sum(filename):
371
- sha1 = hashlib.sha1()
372
- sha1.update(filename.encode("utf-8"))
373
- return sha1.hexdigest()
374
-
375
-
376
  def replace_today(prompt):
377
  today = datetime.datetime.today().strftime("%Y-%m-%d")
378
  return prompt.replace("{current_date}", today)
 
367
  return key, msg
368
 
369
 
 
 
 
 
 
 
370
  def replace_today(prompt):
371
  today = datetime.datetime.today().strftime("%Y-%m-%d")
372
  return prompt.replace("{current_date}", today)