add tokenizer

added_tokens.json

@@ -1,108 +1,4 @@
 {
-  "<|af|>": 50327,
-  "<|am|>": 50334,
-  "<|ar|>": 50272,
-  "<|as|>": 50350,
-  "<|az|>": 50304,
-  "<|ba|>": 50355,
-  "<|be|>": 50330,
-  "<|bg|>": 50292,
-  "<|bn|>": 50302,
-  "<|bo|>": 50347,
-  "<|br|>": 50309,
-  "<|bs|>": 50315,
-  "<|ca|>": 50270,
-  "<|cs|>": 50283,
-  "<|cy|>": 50297,
-  "<|da|>": 50285,
-  "<|de|>": 50261,
-  "<|el|>": 50281,
-  "<|en|>": 50259,
-  "<|es|>": 50262,
-  "<|et|>": 50307,
-  "<|eu|>": 50310,
-  "<|fa|>": 50300,
-  "<|fi|>": 50277,
-  "<|fo|>": 50338,
-  "<|fr|>": 50265,
-  "<|gl|>": 50319,
-  "<|gu|>": 50333,
-  "<|haw|>": 50352,
-  "<|ha|>": 50354,
-  "<|he|>": 50279,
-  "<|hi|>": 50276,
-  "<|hr|>": 50291,
-  "<|ht|>": 50339,
-  "<|hu|>": 50286,
-  "<|hy|>": 50312,
-  "<|id|>": 50275,
-  "<|is|>": 50311,
-  "<|it|>": 50274,
-  "<|ja|>": 50266,
-  "<|jw|>": 50356,
-  "<|ka|>": 50329,
-  "<|kk|>": 50316,
-  "<|km|>": 50323,
-  "<|kn|>": 50306,
-  "<|ko|>": 50264,
-  "<|la|>": 50294,
-  "<|lb|>": 50345,
-  "<|ln|>": 50353,
-  "<|lo|>": 50336,
-  "<|lt|>": 50293,
-  "<|lv|>": 50301,
-  "<|mg|>": 50349,
-  "<|mi|>": 50295,
-  "<|mk|>": 50308,
-  "<|ml|>": 50296,
-  "<|mn|>": 50314,
-  "<|mr|>": 50320,
-  "<|ms|>": 50282,
-  "<|mt|>": 50343,
-  "<|my|>": 50346,
-  "<|ne|>": 50313,
-  "<|nl|>": 50271,
-  "<|nn|>": 50342,
-  "<|nocaptions|>": 50362,
-  "<|notimestamps|>": 50363,
-  "<|no|>": 50288,
-  "<|oc|>": 50328,
-  "<|pa|>": 50321,
-  "<|pl|>": 50269,
-  "<|ps|>": 50340,
-  "<|pt|>": 50267,
-  "<|ro|>": 50284,
-  "<|ru|>": 50263,
-  "<|sa|>": 50344,
-  "<|sd|>": 50332,
-  "<|si|>": 50322,
-  "<|sk|>": 50298,
-  "<|sl|>": 50305,
-  "<|sn|>": 50324,
-  "<|so|>": 50326,
-  "<|sq|>": 50317,
-  "<|sr|>": 50303,
-  "<|startoflm|>": 50360,
-  "<|startofprev|>": 50361,
-  "<|startoftranscript|>": 50258,
-  "<|su|>": 50357,
-  "<|sv|>": 50273,
-  "<|sw|>": 50318,
-  "<|ta|>": 50287,
-  "<|te|>": 50299,
-  "<|tg|>": 50331,
-  "<|th|>": 50289,
-  "<|tk|>": 50341,
-  "<|tl|>": 50348,
-  "<|transcribe|>": 50359,
-  "<|translate|>": 50358,
-  "<|tr|>": 50268,
-  "<|tt|>": 50351,
-  "<|uk|>": 50280,
-  "<|ur|>": 50290,
-  "<|uz|>": 50337,
-  "<|vi|>": 50278,
-  "<|yi|>": 50335,
-  "<|yo|>": 50325,
-  "<|zh|>": 50260
+  "</s>": 47,
+  "<s>": 46
 }

special_tokens_map.json

@@ -1,133 +1,22 @@
 {
   "additional_special_tokens": [
-    "<|endoftext|>",
-    "<|startoftranscript|>",
-    "<|en|>",
-    "<|zh|>",
-    "<|de|>",
-    "<|es|>",
-    "<|ru|>",
-    "<|ko|>",
-    "<|fr|>",
-    "<|ja|>",
-    "<|pt|>",
-    "<|tr|>",
-    "<|pl|>",
-    "<|ca|>",
-    "<|nl|>",
-    "<|ar|>",
-    "<|sv|>",
-    "<|it|>",
-    "<|id|>",
-    "<|hi|>",
-    "<|fi|>",
-    "<|vi|>",
-    "<|he|>",
-    "<|uk|>",
-    "<|el|>",
-    "<|ms|>",
-    "<|cs|>",
-    "<|ro|>",
-    "<|da|>",
-    "<|hu|>",
-    "<|ta|>",
-    "<|no|>",
-    "<|th|>",
-    "<|ur|>",
-    "<|hr|>",
-    "<|bg|>",
-    "<|lt|>",
-    "<|la|>",
-    "<|mi|>",
-    "<|ml|>",
-    "<|cy|>",
-    "<|sk|>",
-    "<|te|>",
-    "<|fa|>",
-    "<|lv|>",
-    "<|bn|>",
-    "<|sr|>",
-    "<|az|>",
-    "<|sl|>",
-    "<|kn|>",
-    "<|et|>",
-    "<|mk|>",
-    "<|br|>",
-    "<|eu|>",
-    "<|is|>",
-    "<|hy|>",
-    "<|ne|>",
-    "<|mn|>",
-    "<|bs|>",
-    "<|kk|>",
-    "<|sq|>",
-    "<|sw|>",
-    "<|gl|>",
-    "<|mr|>",
-    "<|pa|>",
-    "<|si|>",
-    "<|km|>",
-    "<|sn|>",
-    "<|yo|>",
-    "<|so|>",
-    "<|af|>",
-    "<|oc|>",
-    "<|ka|>",
-    "<|be|>",
-    "<|tg|>",
-    "<|sd|>",
-    "<|gu|>",
-    "<|am|>",
-    "<|yi|>",
-    "<|lo|>",
-    "<|uz|>",
-    "<|fo|>",
-    "<|ht|>",
-    "<|ps|>",
-    "<|tk|>",
-    "<|nn|>",
-    "<|mt|>",
-    "<|sa|>",
-    "<|lb|>",
-    "<|my|>",
-    "<|bo|>",
-    "<|tl|>",
-    "<|mg|>",
-    "<|as|>",
-    "<|tt|>",
-    "<|haw|>",
-    "<|ln|>",
-    "<|ha|>",
-    "<|ba|>",
-    "<|jw|>",
-    "<|su|>",
-    "<|translate|>",
-    "<|transcribe|>",
-    "<|startoflm|>",
-    "<|startofprev|>",
-    "<|nocaptions|>",
-    "<|notimestamps|>"
+    {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false
+    }
   ],
-  "bos_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": "<|endoftext|>",
-  "unk_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  }
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "[PAD]",
+  "unk_token": "[UNK]"
 }

tokenizer_config.json

@@ -1,35 +1,12 @@
 {
-  "add_bos_token": false,
-  "add_prefix_space": false,
-  "bos_token": {
-    "__type": "AddedToken",
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "__type": "AddedToken",
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "errors": "replace",
-  "model_max_length": 1024,
-  "pad_token": null,
-  "processor_class": "WhisperProcessor",
-  "return_attention_mask": false,
+  "bos_token": "<s>",
+  "do_lower_case": false,
+  "eos_token": "</s>",
+  "name_or_path": "./",
+  "pad_token": "[PAD]",
+  "replace_word_delimiter_char": " ",
   "special_tokens_map_file": null,
-  "tokenizer_class": "WhisperTokenizer",
-  "unk_token": {
-    "__type": "AddedToken",
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  }
+  "tokenizer_class": "Wav2Vec2CTCTokenizer",
+  "unk_token": "[UNK]",
+  "word_delimiter_token": "|"
 }