Upload tokenizer

added_tokens.json

@@ -1,4 +1,4 @@
 {
-  "</s>": 71,
-  "<s>": 70
+  "</s>": 94,
+  "<s>": 93
 }

special_tokens_map.json

@@ -1,30 +1,6 @@
 {
-  "bos_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": {
-    "content": "[PAD]",
-    "lstrip": true,
-    "normalized": false,
-    "rstrip": true,
-    "single_word": false
-  },
-  "unk_token": {
-    "content": "[UNK]",
-    "lstrip": true,
-    "normalized": false,
-    "rstrip": true,
-    "single_word": false
-  }
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "[PAD]",
+  "unk_token": "[UNK]"
 }

tokenizer_config.json

@@ -1,6 +1,6 @@
 {
   "added_tokens_decoder": {
-    "68": {
+    "91": {
       "content": "[UNK]",
       "lstrip": true,
       "normalized": false,
@@ -8,7 +8,7 @@
       "single_word": false,
       "special": false
     },
-    "69": {
+    "92": {
       "content": "[PAD]",
       "lstrip": true,
       "normalized": false,
@@ -16,7 +16,7 @@
       "single_word": false,
       "special": false
     },
-    "70": {
+    "93": {
       "content": "<s>",
       "lstrip": false,
       "normalized": false,
@@ -24,7 +24,7 @@
       "single_word": false,
       "special": true
     },
-    "71": {
+    "94": {
       "content": "</s>",
       "lstrip": false,
       "normalized": false,

vocab.json

@@ -3,72 +3,95 @@
     "(": 1,
     ")": 2,
     "/": 3,
-    "[PAD]": 69,
-    "[UNK]": 68,
+    "[PAD]": 92,
+    "[UNK]": 91,
+    "a": 4,
+    "b": 5,
+    "c": 6,
+    "e": 7,
+    "f": 8,
+    "k": 9,
+    "o": 10,
     "|": 0,
-    "ँ": 4,
-    "ं": 5,
-    "ः": 6,
-    "अ": 7,
-    "आ": 8,
-    "इ": 9,
-    "ई": 10,
-    "उ": 11,
-    "ऊ": 12,
-    "ऋ": 13,
-    "ए": 14,
-    "ऐ": 15,
-    "ओ": 16,
-    "औ": 17,
-    "क": 18,
-    "ख": 19,
-    "ग": 20,
-    "घ": 21,
-    "ङ": 22,
-    "च": 23,
-    "छ": 24,
-    "ज": 25,
-    "झ": 26,
-    "ञ": 27,
-    "ट": 28,
-    "ठ": 29,
-    "ड": 30,
-    "ढ": 31,
-    "ण": 32,
-    "त": 33,
-    "थ": 34,
-    "द": 35,
-    "ध": 36,
-    "न": 37,
-    "प": 38,
-    "फ": 39,
-    "ब": 40,
-    "भ": 41,
-    "म": 42,
-    "य": 43,
-    "र": 44,
-    "ऱ": 45,
-    "ल": 46,
-    "व": 47,
-    "श": 48,
-    "ष": 49,
-    "स": 50,
-    "ह": 51,
-    "ा": 52,
-    "ि": 53,
-    "ी": 54,
-    "ु": 55,
-    "ू": 56,
-    "ृ": 57,
-    "े": 58,
-    "ै": 59,
-    "ो": 60,
-    "ौ": 61,
-    "्": 62,
-    "‌": 63,
-    "‍": 64,
-    "–": 65,
-    "—": 66,
-    "’": 67
+    "ँ": 11,
+    "ं": 12,
+    "ः": 13,
+    "अ": 14,
+    "आ": 15,
+    "इ": 16,
+    "ई": 17,
+    "उ": 18,
+    "ऊ": 19,
+    "ऋ": 20,
+    "ए": 21,
+    "ऐ": 22,
+    "ओ": 23,
+    "औ": 24,
+    "क": 25,
+    "ख": 26,
+    "ग": 27,
+    "घ": 28,
+    "ङ": 29,
+    "च": 30,
+    "छ": 31,
+    "ज": 32,
+    "झ": 33,
+    "ञ": 34,
+    "ट": 35,
+    "ठ": 36,
+    "ड": 37,
+    "ढ": 38,
+    "ण": 39,
+    "त": 40,
+    "थ": 41,
+    "द": 42,
+    "ध": 43,
+    "न": 44,
+    "प": 45,
+    "फ": 46,
+    "ब": 47,
+    "भ": 48,
+    "म": 49,
+    "य": 50,
+    "र": 51,
+    "ऱ": 52,
+    "ल": 53,
+    "व": 54,
+    "श": 55,
+    "ष": 56,
+    "स": 57,
+    "ह": 58,
+    "़": 59,
+    "ा": 60,
+    "ि": 61,
+    "ी": 62,
+    "ु": 63,
+    "ू": 64,
+    "ृ": 65,
+    "े": 66,
+    "ै": 67,
+    "ो": 68,
+    "ौ": 69,
+    "्": 70,
+    "ॐ": 71,
+    "॑": 72,
+    "ॠ": 73,
+    "०": 74,
+    "१": 75,
+    "२": 76,
+    "३": 77,
+    "४": 78,
+    "५": 79,
+    "६": 80,
+    "७": 81,
+    "८": 82,
+    "९": 83,
+    "॰": 84,
+    "‌": 85,
+    "‍": 86,
+    "‎": 87,
+    "–": 88,
+    "—": 89,
+    "’": 90
   }
 }