michael-guenther
commited on
Commit
•
9072f7f
1
Parent(s):
11b09c9
Update tokenizer.py
Browse files- tokenizer.py +2 -2
tokenizer.py
CHANGED
@@ -8,11 +8,11 @@ import warnings
|
|
8 |
def get_tokenizer(parent_class):
|
9 |
class TokenizerClass(parent_class):
|
10 |
class TaskTypes(IntEnum):
|
11 |
-
NULL =
|
12 |
QUERY = 1
|
13 |
DOCUMENT = 2
|
14 |
STS = 3
|
15 |
-
CLUSTERING =
|
16 |
CLASSIFICATION = 5
|
17 |
|
18 |
def __init__(self, *args, **kwargs):
|
|
|
8 |
def get_tokenizer(parent_class):
|
9 |
class TokenizerClass(parent_class):
|
10 |
class TaskTypes(IntEnum):
|
11 |
+
NULL = 0
|
12 |
QUERY = 1
|
13 |
DOCUMENT = 2
|
14 |
STS = 3
|
15 |
+
CLUSTERING = 4
|
16 |
CLASSIFICATION = 5
|
17 |
|
18 |
def __init__(self, *args, **kwargs):
|