Upload tokenizer

.gitattributes

@@ -32,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -0,0 +1,25 @@
+{
+  "[/ENT-01]": 250002,
+  "[/ENT-02]": 250003,
+  "[/ENT-03]": 250004,
+  "[/ENT-04]": 250005,
+  "[/ENT-05]": 250006,
+  "[/ENT-06]": 250007,
+  "[/ENT-07]": 250008,
+  "[/ENT-08]": 250009,
+  "[/OBJ]": 250010,
+  "[/SUB]": 250011,
+  "[ENT-01]": 250012,
+  "[ENT-02]": 250013,
+  "[ENT-03]": 250014,
+  "[ENT-04]": 250015,
+  "[ENT-05]": 250016,
+  "[ENT-06]": 250017,
+  "[ENT-07]": 250018,
+  "[ENT-08]": 250019,
+  "[GRP-00]": 250020,
+  "[GRP-01]": 250021,
+  "[OBJ]": 250022,
+  "[REL]": 250023,
+  "[SUB]": 250024
+}

sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865
+size 5069051

special_tokens_map.json

@@ -0,0 +1,40 @@
+{
+  "additional_special_tokens": [
+    "[/ENT-01]",
+    "[/ENT-02]",
+    "[/ENT-03]",
+    "[/ENT-04]",
+    "[/ENT-05]",
+    "[/ENT-06]",
+    "[/ENT-07]",
+    "[/ENT-08]",
+    "[/OBJ]",
+    "[/SUB]",
+    "[ENT-01]",
+    "[ENT-02]",
+    "[ENT-03]",
+    "[ENT-04]",
+    "[ENT-05]",
+    "[ENT-06]",
+    "[ENT-07]",
+    "[ENT-08]",
+    "[GRP-00]",
+    "[GRP-01]",
+    "[OBJ]",
+    "[REL]",
+    "[SUB]"
+  ],
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f6c6cd2187fd44407ae578446f51e29aef61147368380b76f051a6138cb41d5
+size 17086910

tokenizer_config.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "__type": "AddedToken",
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "max_length": 512,
+  "model_max_length": 512,
+  "name_or_path": "xlm-roberta-large",
+  "pad_token": "<pad>",
+  "padding": "max_length",
+  "sep_token": "</s>",
+  "special_tokens_map_file": null,
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "truncation": true,
+  "unk_token": "<unk>"
+}