Upload tokenizer

README.md

@@ -1,10 +1,10 @@
 ---
-license: cc-by-nc-4.0
 base_model: facebook/mms-1b-all
-tags:
-- generated_from_trainer
+license: cc-by-nc-4.0
 metrics:
 - wer
+tags:
+- generated_from_trainer
 model-index:
 - name: wav2vec2-large-mms-1b-nepali
   results: []

special_tokens_map.json

@@ -1,6 +1,18 @@
 {
-  "bos_token": "<s>",
-  "eos_token": "</s>",
-  "pad_token": "[PAD]",
-  "unk_token": "[UNK]"
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "[pad]",
+  "unk_token": "[unk]"
 }

tokenizer_config.json

@@ -38,10 +38,10 @@
   "do_lower_case": false,
   "eos_token": "</s>",
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "[PAD]",
+  "pad_token": "[pad]",
   "replace_word_delimiter_char": " ",
   "target_lang": "npi",
   "tokenizer_class": "Wav2Vec2CTCTokenizer",
-  "unk_token": "[UNK]",
+  "unk_token": "[unk]",
   "word_delimiter_token": "|"
 }

vocab.json

@@ -1,252 +1,253 @@
-{"eng": {
-        "!": 75,
-        "\"": 62,
-        "$": 48,
-        "%": 47,
-        "&": 57,
-        "'": 31,
-        "+": 56,
-        ",": 44,
-        "-": 32,
-        ".": 37,
-        "/": 45,
-        "0": 27,
-        "1": 30,
-        "2": 35,
-        "3": 41,
-        "4": 39,
-        "5": 38,
-        "6": 40,
-        "7": 43,
-        "8": 42,
-        "9": 36,
-        ":": 46,
-        ";": 52,
-        "</s>": 2,
-        "<pad>": 0,
-        "<s>": 1,
-        "<unk>": 3,
-        "[": 59,
-        "]": 60,
-        "`": 92,
-        "a": 7,
-        "b": 24,
-        "c": 16,
-        "d": 15,
-        "e": 5,
-        "f": 19,
-        "g": 21,
-        "h": 13,
-        "i": 9,
-        "j": 29,
-        "k": 26,
-        "l": 14,
-        "m": 18,
-        "n": 10,
-        "o": 8,
-        "p": 20,
-        "q": 34,
-        "r": 12,
-        "s": 11,
-        "t": 6,
-        "u": 17,
-        "v": 25,
-        "w": 23,
-        "x": 28,
-        "y": 22,
-        "z": 33,
-        "|": 4,
-        "\u00a3": 68,
-        "\u00a5": 53,
-        "\u00a7": 139,
-        "\u00b0": 58,
-        "\u00b2": 76,
-        "\u00df": 82,
-        "\u00e0": 84,
-        "\u00e1": 49,
-        "\u00e2": 81,
-        "\u00e3": 66,
-        "\u00e4": 78,
-        "\u00e5": 123,
-        "\u00e6": 110,
-        "\u00e7": 64,
-        "\u00e8": 79,
-        "\u00e9": 55,
-        "\u00ea": 88,
-        "\u00eb": 87,
-        "\u00ed": 50,
-        "\u00ee": 97,
-        "\u00ef": 85,
-        "\u00f0": 98,
-        "\u00f1": 83,
-        "\u00f2": 105,
-        "\u00f3": 67,
-        "\u00f4": 86,
-        "\u00f5": 69,
-        "\u00f6": 74,
-        "\u00f8": 91,
-        "\u00fa": 70,
-        "\u00fb": 99,
-        "\u00fc": 54,
-        "\u00fd": 112,
-        "\u00fe": 132,
-        "\u0101": 100,
-        "\u0103": 102,
-        "\u0107": 94,
-        "\u010d": 89,
-        "\u0113": 143,
-        "\u0119": 119,
-        "\u011b": 140,
-        "\u011f": 153,
-        "\u012b": 107,
-        "\u0131": 103,
-        "\u0142": 73,
-        "\u0144": 118,
-        "\u0148": 137,
-        "\u014d": 80,
-        "\u0151": 120,
-        "\u0153": 96,
-        "\u0159": 108,
-        "\u015f": 116,
-        "\u0161": 90,
-        "\u016b": 101,
-        "\u017e": 95,
-        "\u0219": 104,
-        "\u0259": 138,
-        "\u02bb": 115,
-        "\u0301": 93,
-        "\u0307": 77,
-        "\u03b1": 106,
-        "\u03ba": 109,
-        "\u03c0": 117,
-        "\u03c7": 141,
-        "\u0430": 144,
-        "\u0432": 124,
-        "\u0435": 125,
-        "\u0437": 130,
-        "\u0438": 113,
-        "\u0439": 127,
-        "\u043a": 114,
-        "\u043b": 126,
-        "\u043d": 128,
-        "\u044c": 131,
-        "\u044f": 129,
-        "\u05e0": 147,
-        "\u05e2": 148,
-        "\u1e43": 121,
-        "\u1ea1": 111,
-        "\u1ea3": 145,
-        "\u1ecb": 146,
-        "\u1ee5": 122,
-        "\u200b": 71,
-        "\u2013": 61,
-        "\u2014": 51,
-        "\u2018": 72,
-        "\u2019": 65,
-        "\u201d": 63,
-        "\u2261": 142,
-        "\u4eac": 149,
-        "\u5148": 135,
-        "\u5927": 151,
-        "\u5c1a": 134,
-        "\u65f6": 133,
-        "\u751f": 136,
-        "\u90fd": 150,
-        "\u962a": 152
-    },
-  "npi": {
-    "(": 1,
-    ")": 2,
-    "/": 3,
-    "[PAD]": 92,
-    "[UNK]": 91,
-    "a": 4,
-    "b": 5,
-    "c": 6,
-    "e": 7,
-    "f": 8,
-    "k": 9,
-    "o": 10,
-    "|": 0,
-    "ँ": 11,
-    "ं": 12,
-    "ः": 13,
-    "अ": 14,
-    "आ": 15,
-    "इ": 16,
-    "ई": 17,
-    "उ": 18,
-    "ऊ": 19,
-    "ऋ": 20,
-    "ए": 21,
-    "ऐ": 22,
-    "ओ": 23,
-    "औ": 24,
-    "क": 25,
-    "ख": 26,
-    "ग": 27,
-    "घ": 28,
-    "ङ": 29,
-    "च": 30,
-    "छ": 31,
-    "ज": 32,
-    "झ": 33,
-    "ञ": 34,
-    "ट": 35,
-    "ठ": 36,
-    "ड": 37,
-    "ढ": 38,
-    "ण": 39,
-    "त": 40,
-    "थ": 41,
-    "द": 42,
-    "ध": 43,
-    "न": 44,
-    "प": 45,
-    "फ": 46,
-    "ब": 47,
-    "भ": 48,
-    "म": 49,
-    "य": 50,
-    "र": 51,
-    "ऱ": 52,
-    "ल": 53,
-    "व": 54,
-    "श": 55,
-    "ष": 56,
-    "स": 57,
-    "ह": 58,
-    "़": 59,
-    "ा": 60,
-    "ि": 61,
-    "ी": 62,
-    "ु": 63,
-    "ू": 64,
-    "ृ": 65,
-    "े": 66,
-    "ै": 67,
-    "ो": 68,
-    "ौ": 69,
-    "्": 70,
-    "ॐ": 71,
-    "॑": 72,
-    "ॠ": 73,
-    "०": 74,
-    "१": 75,
-    "२": 76,
-    "३": 77,
-    "४": 78,
-    "५": 79,
-    "६": 80,
-    "७": 81,
-    "८": 82,
-    "९": 83,
-    "॰": 84,
-    "‌": 85,
-    "‍": 86,
-    "‎": 87,
-    "–": 88,
-    "—": 89,
-    "’": 90
-  }
-}
+{
+  "eng": {
+    "!": 75,
+    "\"": 62,
+    "$": 48,
+    "%": 47,
+    "&": 57,
+    "'": 31,
+    "+": 56,
+    ",": 44,
+    "-": 32,
+    ".": 37,
+    "/": 45,
+    "0": 27,
+    "1": 30,
+    "2": 35,
+    "3": 41,
+    "4": 39,
+    "5": 38,
+    "6": 40,
+    "7": 43,
+    "8": 42,
+    "9": 36,
+    ":": 46,
+    ";": 52,
+    "</s>": 2,
+    "<pad>": 0,
+    "<s>": 1,
+    "<unk>": 3,
+    "[": 59,
+    "]": 60,
+    "`": 92,
+    "a": 7,
+    "b": 24,
+    "c": 16,
+    "d": 15,
+    "e": 5,
+    "f": 19,
+    "g": 21,
+    "h": 13,
+    "i": 9,
+    "j": 29,
+    "k": 26,
+    "l": 14,
+    "m": 18,
+    "n": 10,
+    "o": 8,
+    "p": 20,
+    "q": 34,
+    "r": 12,
+    "s": 11,
+    "t": 6,
+    "u": 17,
+    "v": 25,
+    "w": 23,
+    "x": 28,
+    "y": 22,
+    "z": 33,
+    "|": 4,
+    "£": 68,
+    "¥": 53,
+    "§": 139,
+    "°": 58,
+    "²": 76,
+    "ß": 82,
+    "à": 84,
+    "á": 49,
+    "â": 81,
+    "ã": 66,
+    "ä": 78,
+    "å": 123,
+    "æ": 110,
+    "ç": 64,
+    "è": 79,
+    "é": 55,
+    "ê": 88,
+    "ë": 87,
+    "í": 50,
+    "î": 97,
+    "ï": 85,
+    "ð": 98,
+    "ñ": 83,
+    "ò": 105,
+    "ó": 67,
+    "ô": 86,
+    "õ": 69,
+    "ö": 74,
+    "ø": 91,
+    "ú": 70,
+    "û": 99,
+    "ü": 54,
+    "ý": 112,
+    "þ": 132,
+    "ā": 100,
+    "ă": 102,
+    "ć": 94,
+    "č": 89,
+    "ē": 143,
+    "ę": 119,
+    "ě": 140,
+    "ğ": 153,
+    "ī": 107,
+    "ı": 103,
+    "ł": 73,
+    "ń": 118,
+    "ň": 137,
+    "ō": 80,
+    "ő": 120,
+    "œ": 96,
+    "ř": 108,
+    "ş": 116,
+    "š": 90,
+    "ū": 101,
+    "ž": 95,
+    "ș": 104,
+    "ə": 138,
+    "ʻ": 115,
+    "́": 93,
+    "̇": 77,
+    "α": 106,
+    "κ": 109,
+    "π": 117,
+    "χ": 141,
+    "а": 144,
+    "в": 124,
+    "е": 125,
+    "з": 130,
+    "и": 113,
+    "й": 127,
+    "к": 114,
+    "л": 126,
+    "н": 128,
+    "ь": 131,
+    "я": 129,
+    "נ": 147,
+    "ע": 148,
+    "ṃ": 121,
+    "ạ": 111,
+    "ả": 145,
+    "ị": 146,
+    "ụ": 122,
+    "​": 71,
+    "–": 61,
+    "—": 51,
+    "‘": 72,
+    "’": 65,
+    "”": 63,
+    "≡": 142,
+    "京": 149,
+    "先": 135,
+    "大": 151,
+    "尚": 134,
+    "时": 133,
+    "生": 136,
+    "都": 150,
+    "阪": 152
+  },
+  "npi": {
+    "(": 1,
+    ")": 2,
+    "/": 3,
+    "[PAD]": 92,
+    "[UNK]": 91,
+    "a": 4,
+    "b": 5,
+    "c": 6,
+    "e": 7,
+    "f": 8,
+    "k": 9,
+    "o": 10,
+    "|": 0,
+    "ँ": 11,
+    "ं": 12,
+    "ः": 13,
+    "अ": 14,
+    "आ": 15,
+    "इ": 16,
+    "ई": 17,
+    "उ": 18,
+    "ऊ": 19,
+    "ऋ": 20,
+    "ए": 21,
+    "ऐ": 22,
+    "ओ": 23,
+    "औ": 24,
+    "क": 25,
+    "ख": 26,
+    "ग": 27,
+    "घ": 28,
+    "ङ": 29,
+    "च": 30,
+    "छ": 31,
+    "ज": 32,
+    "झ": 33,
+    "ञ": 34,
+    "ट": 35,
+    "ठ": 36,
+    "ड": 37,
+    "ढ": 38,
+    "ण": 39,
+    "त": 40,
+    "थ": 41,
+    "द": 42,
+    "ध": 43,
+    "न": 44,
+    "प": 45,
+    "फ": 46,
+    "ब": 47,
+    "भ": 48,
+    "म": 49,
+    "य": 50,
+    "र": 51,
+    "ऱ": 52,
+    "ल": 53,
+    "व": 54,
+    "श": 55,
+    "ष": 56,
+    "स": 57,
+    "ह": 58,
+    "़": 59,
+    "ा": 60,
+    "ि": 61,
+    "ी": 62,
+    "ु": 63,
+    "ू": 64,
+    "ृ": 65,
+    "े": 66,
+    "ै": 67,
+    "ो": 68,
+    "ौ": 69,
+    "्": 70,
+    "ॐ": 71,
+    "॑": 72,
+    "ॠ": 73,
+    "०": 74,
+    "१": 75,
+    "२": 76,
+    "३": 77,
+    "४": 78,
+    "५": 79,
+    "६": 80,
+    "७": 81,
+    "८": 82,
+    "९": 83,
+    "॰": 84,
+    "‌": 85,
+    "‍": 86,
+    "‎": 87,
+    "–": 88,
+    "—": 89,
+    "’": 90
+  }
+}