jon-tow commited on
Commit
8562553
1 Parent(s): a2eb1af

update(tokenizer): convert to `GPT2Tokenizer`

Browse files
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
special_tokens_map.json ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|reg_extra|>",
4
+ "<|endoftext|>",
5
+ "<|fim_prefix|>",
6
+ "<|fim_middle|>",
7
+ "<|fim_suffix|>",
8
+ "<|fim_pad|>",
9
+ "<gh_stars>",
10
+ "<filename>",
11
+ "<issue_start>",
12
+ "<issue_comment>",
13
+ "<issue_closed>",
14
+ "<jupyter_start>",
15
+ "<jupyter_text>",
16
+ "<jupyter_code>",
17
+ "<jupyter_output>",
18
+ "<empty_output>",
19
+ "<commit_before>",
20
+ "<commit_msg>",
21
+ "<commit_after>",
22
+ "<reponame>",
23
+ "<|endofprompt|>",
24
+ "<|im_start|>",
25
+ "<|im_end|>",
26
+ "<|pause|>",
27
+ "<|reg0|>",
28
+ "<|reg1|>",
29
+ "<|reg2|>",
30
+ "<|reg3|>",
31
+ "<|reg4|>",
32
+ "<|reg5|>",
33
+ "<|reg6|>",
34
+ "<|reg7|>",
35
+ "<|extra0|>"
36
+ ],
37
+ "bos_token": "<|endoftext|>",
38
+ "eos_token": "<|endoftext|>",
39
+ "unk_token": "<|endoftext|>"
40
+ }
tokenization_arcade100k.py DELETED
@@ -1,292 +0,0 @@
1
- # coding=utf-8
2
- # Copyright (c) 2023 Alibaba Cloud & Stability AI.
3
- #
4
- # Tongyi Qianwen LICENSE AGREEMENT:
5
- # https://github.com/QwenLM/Qwen/blob/5aa84bdfd3237b37f01bc88cd49b3279b9a71d0b/Tongyi%20Qianwen%20LICENSE%20AGREEMENT
6
- """Tokenization classes for Arcade100k."""
7
-
8
- import base64
9
- import os
10
- import unicodedata
11
- from typing import Collection, Dict, List, Set, Tuple, Union
12
-
13
- import tiktoken
14
- from transformers.utils import logging
15
- from transformers import PreTrainedTokenizer, AddedToken
16
-
17
- logger = logging.get_logger(__name__)
18
-
19
- VOCAB_FILES_NAMES = {"vocab_file": "arcade100k.tiktoken"}
20
- NAME = "arcade100k"
21
-
22
-
23
- def _load_tiktoken_bpe(tiktoken_bpe_file: str) -> Dict[bytes, int]:
24
- with open(tiktoken_bpe_file, "rb") as f:
25
- contents = f.read()
26
- return {
27
- base64.b64decode(token): int(rank)
28
- for token, rank in (line.split() for line in contents.splitlines() if line)
29
- }
30
-
31
-
32
- ENDOFTEXT = "<|endoftext|>"
33
- FIM = [
34
- "<|fim_prefix|>",
35
- "<|fim_middle|>",
36
- "<|fim_suffix|>",
37
- "<|fim_pad|>",
38
- ]
39
- # `StarCoder` Tokens
40
- CODE = [
41
- "<gh_stars>",
42
- "<filename>",
43
- "<issue_start>",
44
- "<issue_comment>",
45
- "<issue_closed>",
46
- "<jupyter_start>",
47
- "<jupyter_text>",
48
- "<jupyter_code>",
49
- "<jupyter_output>",
50
- "<empty_output>",
51
- "<commit_before>",
52
- "<commit_msg>",
53
- "<commit_after>",
54
- "<reponame>",
55
- ]
56
- CHAT = [
57
- "<|im_start|>", # Chat: Input message start
58
- "<|im_end|>", # Chat: Input message end
59
- ]
60
- PAUSE = "<|pause|>" # Think before you speak (https://arxiv.org/abs/2310.02226)
61
- REGISTERS = [
62
- f"<|reg{i}|>" for i in range(0, 8)
63
- ] # Register 0 sink token (https://arxiv.org/abs/2309.17453)
64
- ENDOFPROMPT = "<|endofprompt|>"
65
- SPECIAL_TOKENS_NAMES = (
66
- [ENDOFTEXT]
67
- + FIM
68
- + CODE
69
- + [ENDOFPROMPT]
70
- + CHAT
71
- + [PAUSE]
72
- + REGISTERS
73
- + ["<|extra0|>"]
74
- )
75
- START_ID = 100257
76
- SPECIAL_TOKENS = {t: START_ID + i for i, t in enumerate(SPECIAL_TOKENS_NAMES)}
77
-
78
-
79
- def _arcade100k(vocab_file: str):
80
- mergeable_ranks = _load_tiktoken_bpe(vocab_file)
81
-
82
- return {
83
- "name": NAME,
84
- "pat_str": r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+""",
85
- "mergeable_ranks": mergeable_ranks,
86
- "special_tokens": SPECIAL_TOKENS,
87
- }
88
-
89
-
90
- class Arcade100kTokenizer(PreTrainedTokenizer):
91
- """
92
- Construct a Arcade100k tokenizer backed by `tiktoken`.
93
-
94
- Args:
95
- vocab_file (`str`):
96
- Path to the vocabulary file.
97
- errors (`str`, *optional*, defaults to `"replace"`):
98
- How to handle errors in decoding UTF-8 byte sequences.
99
- WARNING: the default behaviour of this function is lossy, since decoded bytes are not
100
- guaranteed to be valid UTF-8. You can control this behaviour using the `errors` parameter,
101
- for instance, setting `errors=strict`.
102
- """
103
-
104
- vocab_files_names = VOCAB_FILES_NAMES
105
- model_input_names = ["input_ids", "attention_mask"]
106
-
107
- def __init__(
108
- self,
109
- vocab_file: str,
110
- errors: str = "replace",
111
- **kwargs,
112
- ):
113
- super().__init__(errors=errors, **kwargs)
114
- self.errors = errors
115
-
116
- self._tiktoken_config = _arcade100k(vocab_file)
117
- self.tokenizer = tiktoken.Encoding(**self._tiktoken_config)
118
-
119
- # TODO: Remove this assertion
120
- assert (
121
- len(self.tokenizer._mergeable_ranks)
122
- + len(self.tokenizer._special_tokens)
123
- + 1
124
- == self.tokenizer.n_vocab
125
- ), f"{len(self.tokenizer._mergeable_ranks) + len(self.tokenizer._special_tokens)} != {self.tokenizer.n_vocab} in encoding"
126
-
127
- self.decoder = {i: n for n, i in self.tokenizer._mergeable_ranks.items()}
128
- self.decoder.update({i: n for n, i in self.tokenizer._special_tokens.items()})
129
- # Provide default `eos_token` and `pad_token`
130
- if self.eos_token is None:
131
- self.eos_token = self.decoder[self.tokenizer.eot_token]
132
- if self.pad_token is None:
133
- self.pad_token = self.decoder[self.tokenizer.pad_token]
134
-
135
- # Expose for convenience
136
- self.mergeable_ranks = self.tokenizer._mergeable_ranks
137
- self.special_tokens = self.tokenizer._special_tokens
138
-
139
- def __len__(self):
140
- return self.tokenizer.n_vocab
141
-
142
- def __getstate__(self):
143
- # Required for `pickle` support
144
- state = self.__dict__.copy()
145
- del state["tokenizer"]
146
- return state
147
-
148
- def __setstate__(self, state):
149
- self.__dict__.update(state)
150
- self.tokenizer = tiktoken.Encoding(**self._tiktoken_config)
151
-
152
- @property
153
- def vocab_size(self):
154
- return self.tokenizer.n_vocab
155
-
156
- def get_vocab(self) -> Dict[bytes, int]:
157
- return self.tokenizer._mergeable_ranks
158
-
159
- def convert_tokens_to_ids(
160
- self, tokens: Union[bytes, str, List[Union[bytes, str]]]
161
- ) -> List[int]:
162
- ids = []
163
- if isinstance(tokens, (str, bytes)):
164
- if tokens in self.tokenizer._special_tokens:
165
- return self.tokenizer._special_tokens[tokens]
166
- else:
167
- return self.tokenizer._mergeable_ranks.get(tokens)
168
- for token in tokens:
169
- if token in self.tokenizer._special_tokens:
170
- ids.append(self.tokenizer._special_tokens[token])
171
- else:
172
- ids.append(self.tokenizer._mergeable_ranks.get(token))
173
- return ids
174
-
175
- def _add_tokens(
176
- self,
177
- new_tokens: Union[List[str], List[AddedToken]],
178
- special_tokens: bool = False,
179
- ) -> int:
180
- if not special_tokens and new_tokens:
181
- raise ValueError("Adding regular tokens is not supported")
182
- for token in new_tokens:
183
- surface_form = token.content if isinstance(token, AddedToken) else token
184
- if surface_form not in SPECIAL_TOKENS:
185
- raise ValueError("Adding unknown special tokens is not supported")
186
- return 0
187
-
188
- def save_vocabulary(self, save_directory: str, **kwargs) -> Tuple[str]:
189
- """
190
- Save only the vocabulary of the tokenizer (vocabulary).
191
-
192
- Returns:
193
- `Tuple(str)`: Paths to the files saved.
194
- """
195
- file_path = os.path.join(save_directory, "arcade100k.tiktoken")
196
- with open(file_path, "w", encoding="utf8") as w:
197
- for k, v in self.tokenizer._mergeable_ranks.items():
198
- line = base64.b64encode(k).decode("utf8") + " " + str(v) + "\n"
199
- w.write(line)
200
- return (file_path,)
201
-
202
- def tokenize(
203
- self,
204
- text: str,
205
- allowed_special: Union[Set, str] = "all",
206
- disallowed_special: Union[Collection, str] = (),
207
- **kwargs,
208
- ) -> List[Union[bytes, str]]:
209
- """
210
- Converts a string in a sequence of tokens.
211
-
212
- Args:
213
- text (`str`):
214
- The sequence to be encoded.
215
- allowed_special (`Literal["all"]` or `set`):
216
- The surface forms of the tokens to be encoded as special tokens in regular texts.
217
- Default to "all".
218
- disallowed_special (`Literal["all"]` or `Collection`):
219
- The surface forms of the tokens that should not be in regular texts and trigger errors.
220
- Default to an empty tuple.
221
-
222
- kwargs (additional keyword arguments, *optional*):
223
- Will be passed to the underlying model specific encode method.
224
-
225
- Returns:
226
- `List[bytes|str]`: The list of tokens.
227
- """
228
- tokens = []
229
- text = unicodedata.normalize("NFC", text)
230
-
231
- # this implementation takes a detour: text -> token id -> token surface forms
232
- for t in self.tokenizer.encode(
233
- text, allowed_special=allowed_special, disallowed_special=disallowed_special
234
- ):
235
- tokens.append(self.decoder[t])
236
- return tokens
237
-
238
- def convert_tokens_to_string(self, tokens: List[Union[bytes, str]]) -> str:
239
- """
240
- Converts a sequence of tokens in a single string.
241
- """
242
- text = ""
243
- temp = b""
244
- for t in tokens:
245
- if isinstance(t, str):
246
- if temp:
247
- text += temp.decode("utf-8", errors=self.errors)
248
- temp = b""
249
- text += t
250
- elif isinstance(t, bytes):
251
- temp += t
252
- else:
253
- raise TypeError("token should only be of type types or str")
254
- if temp:
255
- text += temp.decode("utf-8", errors=self.errors)
256
- return text
257
-
258
- def _convert_id_to_token(self, index: int) -> Union[bytes, str]:
259
- """Converts an id to a token, special tokens included"""
260
- if index in self.decoder:
261
- return self.decoder[index]
262
- raise ValueError("unknown ids")
263
-
264
- def _convert_token_to_id(self, token: Union[bytes, str]) -> int:
265
- """Converts a token to an id using the vocab, special tokens included"""
266
- if token in self.tokenizer._special_tokens:
267
- return self.tokenizer._special_tokens[token]
268
- if token in self.tokenizer._mergeable_ranks:
269
- return self.tokenizer._mergeable_ranks[token]
270
- raise ValueError("unknown token")
271
-
272
- def _tokenize(self, text: str, **kwargs):
273
- """
274
- Converts a string in a sequence of tokens (string), using the tokenizer. Split in words for word-based
275
- vocabulary or sub-words for sub-word-based vocabularies (BPE/SentencePieces/WordPieces).
276
-
277
- Do NOT take care of added tokens.
278
- """
279
- raise NotImplementedError
280
-
281
- def _decode(
282
- self,
283
- token_ids: Union[int, List[int]],
284
- skip_special_tokens: bool = False,
285
- errors: str = None,
286
- **kwargs,
287
- ) -> str:
288
- if isinstance(token_ids, int):
289
- token_ids = [token_ids]
290
- if skip_special_tokens:
291
- token_ids = [i for i in token_ids if i < self.tokenizer.eot_token]
292
- return self.tokenizer.decode(token_ids)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json CHANGED
@@ -1,11 +1,43 @@
1
  {
2
- "tokenizer_class": "Arcade100kTokenizer",
3
- "auto_map": {
4
- "AutoTokenizer": [
5
- "tokenization_arcade100k.Arcade100kTokenizer",
6
- null
7
- ]
8
- },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9
  "eos_token": "<|endoftext|>",
10
- "pad_token": "<|endoftext|>"
 
11
  }
 
1
  {
2
+ "add_prefix_space": false,
3
+ "additional_special_tokens": [
4
+ "<|reg_extra|>",
5
+ "<|endoftext|>",
6
+ "<|fim_prefix|>",
7
+ "<|fim_middle|>",
8
+ "<|fim_suffix|>",
9
+ "<|fim_pad|>",
10
+ "<gh_stars>",
11
+ "<filename>",
12
+ "<issue_start>",
13
+ "<issue_comment>",
14
+ "<issue_closed>",
15
+ "<jupyter_start>",
16
+ "<jupyter_text>",
17
+ "<jupyter_code>",
18
+ "<jupyter_output>",
19
+ "<empty_output>",
20
+ "<commit_before>",
21
+ "<commit_msg>",
22
+ "<commit_after>",
23
+ "<reponame>",
24
+ "<|endofprompt|>",
25
+ "<|im_start|>",
26
+ "<|im_end|>",
27
+ "<|pause|>",
28
+ "<|reg0|>",
29
+ "<|reg1|>",
30
+ "<|reg2|>",
31
+ "<|reg3|>",
32
+ "<|reg4|>",
33
+ "<|reg5|>",
34
+ "<|reg6|>",
35
+ "<|reg7|>",
36
+ "<|extra0|>"
37
+ ],
38
+ "bos_token": "<|endoftext|>",
39
+ "clean_up_tokenization_spaces": true,
40
  "eos_token": "<|endoftext|>",
41
+ "tokenizer_class": "GPT2Tokenizer",
42
+ "unk_token": "<|endoftext|>"
43
  }
arcade100k.tiktoken → vocab.json RENAMED
The diff for this file is too large to render. See raw diff