xu-song commited on
Commit
f652c69
1 Parent(s): 97354e0

remove lru_cache: has some bugs

Browse files
character_app.py CHANGED
@@ -18,7 +18,6 @@ abbr2name = {column[1]: column[0].split('-')[-1] for column in all_columns}
18
 
19
 
20
  def get_column_info(columns):
21
- print(columns)
22
  markdown = ""
23
  for column in columns:
24
  markdown += f"- `num({column})`: num of tokens containing {abbr2name[column]} characters\n" \
 
18
 
19
 
20
  def get_column_info(columns):
 
21
  markdown = ""
22
  for column in columns:
23
  markdown += f"- `num({column})`: num of tokens containing {abbr2name[column]} characters\n" \
playground_examples.py CHANGED
@@ -37,7 +37,7 @@ examples = {
37
 
38
  more_examples = [
39
  # bert系列
40
- ("bert-base-cased", "bert-base-uncased", "", ""), # # clue VS kplug, bert VS clue
41
  ("bert-base-cased", "clue", "", "增加了[]()"),
42
  ("roberta-chinese-clue", "kplug", "", ""),
43
 
 
37
 
38
  more_examples = [
39
  # bert系列
40
+ ("google-bert/bert-base-cased", "google-bert/bert-base-uncased", "", ""), # # clue VS kplug, bert VS clue
41
  ("bert-base-cased", "clue", "", "增加了[]()"),
42
  ("roberta-chinese-clue", "kplug", "", ""),
43
 
playground_util.py CHANGED
@@ -15,7 +15,7 @@ Buenos días!
15
  default_tokenizer_name_1 = "gradientai/Llama-3-8B-Instruct-Gradient-1048k"
16
  default_tokenizer_name_2 = "openai/gpt-4"
17
 
18
- @lru_cache
19
  def tokenize(
20
  text: str,
21
  tokenizer_name: str,
 
15
  default_tokenizer_name_1 = "gradientai/Llama-3-8B-Instruct-Gradient-1048k"
16
  default_tokenizer_name_2 = "openai/gpt-4"
17
 
18
+ # @lru_cache
19
  def tokenize(
20
  text: str,
21
  tokenizer_name: str,