xu-song commited on
Commit
819cf7f
1 Parent(s): 7156337
Files changed (2) hide show
  1. utils/symbol.py +1 -0
  2. vocab/__init__.py +1 -1
utils/symbol.py CHANGED
@@ -1,5 +1,6 @@
1
  """
2
  special_symbols: https://github.com/google/sentencepiece/blob/master/doc/special_symbols.md
 
3
  """
4
 
5
  import sys
 
1
  """
2
  special_symbols: https://github.com/google/sentencepiece/blob/master/doc/special_symbols.md
3
+ emoji:
4
  """
5
 
6
  import sys
vocab/__init__.py CHANGED
@@ -44,7 +44,7 @@ uniq_tokenizers = [
44
 
45
  all_tokenizers = [
46
  "gpt_35_turbo",
47
- "gpt4",
48
  "gpt2",
49
  "gpt2_chinese",
50
  "bert_base_cased",
 
44
 
45
  all_tokenizers = [
46
  "gpt_35_turbo",
47
+ "gpt_4",
48
  "gpt2",
49
  "gpt2_chinese",
50
  "bert_base_cased",