add tokenizer

vocab.json

@@ -1 +1 @@
-{"ے": 0, "ط": 1, "ن": 2, "م": 3, "ؤ": 4, "چ": 5, "ڈ": 6, "ھ": 7, "ث": 8, "ش": 9, "ل": 10, "ق": 11, "ک": 12, "َ": 13, "ض": 14, "‌": 15, "ّ": 16, "ا": 17, "پ": 19, "ر": 20, "ذ": 21, "س": 22, "ص": 23, "ُ": 24, "ً": 25, "آ": 26, "ز": 27, "گ": 28, "ت": 29, "ئ": 30, "ہ": 31, "ف": 32, "ی": 33, "د": 34, "ج": 35, "و": 36, "ِ": 37, "ظ": 38, "ع": 39, "ٹ": 40, "خ": 41, "ڑ": 42, "غ": 43, "ء": 44, "ژ": 45, "ٔ": 46, "ں": 47, "ح": 48, "ب": 49, "|": 18, "[UNK]": 50, "[PAD]": 51}
+{"ی": 0, "ت": 1, "ج": 2, "ں": 3, "ّ": 4, "ظ": 5, "ؤ": 6, "ذ": 7, "غ": 8, "ئ": 9, "ل": 10, "و": 11, "خ": 12, "ے": 13, "ء": 14, "ھ": 15, "ث": 17, "ڑ": 18, "ب": 19, "ر": 20, "چ": 21, "ن": 22, "گ": 23, "ژ": 24, "ش": 25, "ُ": 26, "ض": 27, "ط": 28, "س": 29, "َ": 30, "ص": 31, "ِ": 32, "پ": 33, "ح": 34, "‌": 35, "ق": 36, "م": 37, "آ": 38, "ہ": 39, "ف": 40, "ٔ": 41, "د": 42, "ع": 43, "ڈ": 44, "ک": 45, "ً": 46, "ز": 47, "ا": 48, "ٹ": 49, "|": 16, "[UNK]": 50, "[PAD]": 51}