Update tokenization_cpmbee.py
Browse files- tokenization_cpmbee.py +3 -3
tokenization_cpmbee.py
CHANGED
@@ -20,9 +20,9 @@ from typing import Any, Dict, List, Optional, Tuple, Union
|
|
20 |
import numpy as np
|
21 |
from typing_extensions import TypedDict
|
22 |
|
23 |
-
from
|
24 |
-
from
|
25 |
-
from
|
26 |
|
27 |
|
28 |
logger = logging.get_logger(__name__)
|
|
|
20 |
import numpy as np
|
21 |
from typing_extensions import TypedDict
|
22 |
|
23 |
+
from transformers.tokenization_utils import PaddingStrategy, PreTrainedTokenizer, TensorType
|
24 |
+
from transformers.tokenization_utils_base import AddedToken, BatchEncoding, TextInput, TruncationStrategy
|
25 |
+
from transformers.utils import logging
|
26 |
|
27 |
|
28 |
logger = logging.get_logger(__name__)
|