Update tokenizer
Browse files- tokenization_chatglm.py +1 -3
tokenization_chatglm.py
CHANGED
|
@@ -15,8 +15,6 @@ from transformers.utils import logging
|
|
| 15 |
|
| 16 |
logger = logging.get_logger(__name__)
|
| 17 |
|
| 18 |
-
VOCAB_FILES_NAMES = {"vocab_file": "ice_text.model"}
|
| 19 |
-
|
| 20 |
PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
|
| 21 |
"THUDM/chatglm-6b": 2048,
|
| 22 |
}
|
|
@@ -179,7 +177,7 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
|
|
| 179 |
Path to the vocabulary file.
|
| 180 |
"""
|
| 181 |
|
| 182 |
-
vocab_files_names =
|
| 183 |
max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
|
| 184 |
model_input_names = ["input_ids"]
|
| 185 |
|
|
|
|
| 15 |
|
| 16 |
logger = logging.get_logger(__name__)
|
| 17 |
|
|
|
|
|
|
|
| 18 |
PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
|
| 19 |
"THUDM/chatglm-6b": 2048,
|
| 20 |
}
|
|
|
|
| 177 |
Path to the vocabulary file.
|
| 178 |
"""
|
| 179 |
|
| 180 |
+
vocab_files_names = {"vocab_file": "ice_text.model"}
|
| 181 |
max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
|
| 182 |
model_input_names = ["input_ids"]
|
| 183 |
|