xu-song commited on
Commit
f02dd94
1 Parent(s): 2461705
Files changed (1) hide show
  1. tokenizer/tiktoken_patch.py +3 -3
tokenizer/tiktoken_patch.py CHANGED
@@ -23,7 +23,7 @@ def decode(self, tokens, errors="replace", skip_special_tokens=False):
23
  logger.error(f"{e} for {tokens} -> return 'null'")
24
  decode_str = "null"
25
  except:
26
- logger.error(f"unknow exception for {tokens} -> return 'null'")
27
  decode_str = "null"
28
  return decode_str
29
 
@@ -40,8 +40,8 @@ def convert_ids_to_tokens(self, tokens, skip_special_tokens=False):
40
  logger.error(f"{e} for {tokens} -> return None")
41
  return [None for _ in tokens]
42
  except:
43
- logger.error(f"unknow exception for {tokens} -> return None")
44
- return
45
 
46
 
47
  def get_vocab(self, token_type="str"):
 
23
  logger.error(f"{e} for {tokens} -> return 'null'")
24
  decode_str = "null"
25
  except:
26
+ logger.error(f"unknown exception for {tokens} -> return 'null'")
27
  decode_str = "null"
28
  return decode_str
29
 
 
40
  logger.error(f"{e} for {tokens} -> return None")
41
  return [None for _ in tokens]
42
  except:
43
+ logger.error(f"unknown exception for {tokens} -> return None")
44
+ return [None for _ in tokens]
45
 
46
 
47
  def get_vocab(self, token_type="str"):