chrisvoncsefalvay commited on
Commit
e7eb7bf
1 Parent(s): d97b2df

Upload tokenizer

Browse files
Files changed (3) hide show
  1. added_tokens.json +3 -0
  2. tokenizer.json +9 -0
  3. tokenizer_config.json +8 -0
added_tokens.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ {
2
+ "Pfizer": 30522
3
+ }
tokenizer.json CHANGED
@@ -52,6 +52,15 @@
52
  "rstrip": false,
53
  "normalized": false,
54
  "special": true
 
 
 
 
 
 
 
 
 
55
  }
56
  ],
57
  "normalizer": {
 
52
  "rstrip": false,
53
  "normalized": false,
54
  "special": true
55
+ },
56
+ {
57
+ "id": 30522,
58
+ "content": "Pfizer",
59
+ "single_word": false,
60
+ "lstrip": false,
61
+ "rstrip": false,
62
+ "normalized": true,
63
+ "special": false
64
  }
65
  ],
66
  "normalizer": {
tokenizer_config.json CHANGED
@@ -39,6 +39,14 @@
39
  "rstrip": false,
40
  "single_word": false,
41
  "special": true
 
 
 
 
 
 
 
 
42
  }
43
  },
44
  "clean_up_tokenization_spaces": true,
 
39
  "rstrip": false,
40
  "single_word": false,
41
  "special": true
42
+ },
43
+ "30522": {
44
+ "content": "Pfizer",
45
+ "lstrip": false,
46
+ "normalized": true,
47
+ "rstrip": false,
48
+ "single_word": false,
49
+ "special": false
50
  }
51
  },
52
  "clean_up_tokenization_spaces": true,