f77777 commited on
Commit
3818e7b
1 Parent(s): 0786b0e

Upload tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +60 -60
vocab.json CHANGED
@@ -1,64 +1,64 @@
1
  {
2
  "[PAD]": 61,
3
  "[UNK]": 60,
4
- "|": 9,
5
- "ँ": 31,
6
- "ं": 49,
7
- "ः": 27,
8
- "अ": 21,
9
- "आ": 1,
10
- "इ": 19,
11
- "ई": 43,
12
- "उ": 36,
13
- "ऊ": 55,
14
- "ए": 54,
15
- "ओ": 44,
16
- "औ": 22,
17
- "क": 41,
18
- "ख": 59,
19
- "ग": 7,
20
- "घ": 42,
21
- "ङ": 6,
22
- "च": 46,
23
- "छ": 15,
24
- "ज": 12,
25
- "झ": 13,
26
- "ञ": 2,
27
- "ट": 57,
28
- "ठ": 58,
29
- "ड": 50,
30
- "ढ": 30,
31
- "ण": 18,
32
- "त": 17,
33
- "थ": 3,
34
- "द": 52,
35
- "ध": 25,
36
- "न": 0,
37
- "प": 33,
38
- "फ": 23,
39
- "ब": 5,
40
- "भ": 53,
41
- "म": 48,
42
- "य": 4,
43
- "र": 11,
44
- "ल": 28,
45
- "व": 35,
46
- "श": 26,
47
- "ष": 37,
48
- "स": 47,
49
- "ह": 20,
50
- "ा": 14,
51
- "ि": 56,
52
- "ी": 40,
53
- "ु": 29,
54
- "ू": 10,
55
- "ृ": 32,
56
- "े": 45,
57
- "ै": 8,
58
- "ो": 34,
59
- "ौ": 39,
60
- "्": 38,
61
- "।": 16,
62
- "–": 24,
63
- "’": 51
64
  }
 
1
  {
2
  "[PAD]": 61,
3
  "[UNK]": 60,
4
+ "|": 20,
5
+ "ँ": 47,
6
+ "ं": 34,
7
+ "ः": 31,
8
+ "अ": 9,
9
+ "आ": 49,
10
+ "इ": 50,
11
+ "ई": 24,
12
+ "उ": 58,
13
+ "ऊ": 48,
14
+ "ए": 17,
15
+ "ओ": 38,
16
+ "औ": 1,
17
+ "क": 28,
18
+ "ख": 26,
19
+ "ग": 41,
20
+ "घ": 51,
21
+ "ङ": 15,
22
+ "च": 35,
23
+ "छ": 42,
24
+ "ज": 25,
25
+ "झ": 55,
26
+ "ञ": 57,
27
+ "ट": 8,
28
+ "ठ": 11,
29
+ "ड": 10,
30
+ "ढ": 7,
31
+ "ण": 16,
32
+ "त": 21,
33
+ "थ": 2,
34
+ "द": 36,
35
+ "ध": 39,
36
+ "न": 30,
37
+ "प": 56,
38
+ "फ": 44,
39
+ "ब": 13,
40
+ "भ": 43,
41
+ "म": 22,
42
+ "य": 14,
43
+ "र": 12,
44
+ "ल": 6,
45
+ "व": 46,
46
+ "श": 40,
47
+ "ष": 18,
48
+ "स": 54,
49
+ "ह": 4,
50
+ "ा": 27,
51
+ "ि": 29,
52
+ "ी": 32,
53
+ "ु": 59,
54
+ "ू": 19,
55
+ "ृ": 53,
56
+ "े": 0,
57
+ "ै": 37,
58
+ "ो": 3,
59
+ "ौ": 23,
60
+ "्": 52,
61
+ "।": 33,
62
+ "–": 45,
63
+ "’": 5
64
  }