add tokenizer

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "<pad>", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": "/root/.cache/huggingface/transformers/369f39682b6b3cd39c9d8f545ad034d6065554081a65da3c18512f204d4dd289.9d6cd81ef646692fb1c169a880161ea1cb95f49694f220aced9b704b457e51dd", "tokenizer_file": null, "name_or_path": "Harveenchadha/vakyansh-wav2vec2-bengali-bnm-200", "tokenizer_class": "Wav2Vec2CTCTokenizer"}

vocab.json

@@ -0,0 +1 @@
+{"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3, "|": 4, "ঁ": 5, "ং": 6, "ঃ": 7, "অ": 8, "আ": 9, "ই": 10, "ঈ": 11, "উ": 12, "ঊ": 13, "ঋ": 14, "এ": 15, "ঐ": 16, "ও": 17, "ঔ": 18, "ক": 19, "খ": 20, "গ": 21, "ঘ": 22, "ঙ": 23, "চ": 24, "ছ": 25, "জ": 26, "ঝ": 27, "ঞ": 28, "ট": 29, "ঠ": 30, "ড": 31, "ঢ": 32, "ণ": 33, "ত": 34, "থ": 35, "দ": 36, "ধ": 37, "ন": 38, "প": 39, "ফ": 40, "ব": 41, "ভ": 42, "ম": 43, "য": 44, "র": 45, "ল": 46, "শ": 47, "ষ": 48, "স": 49, "হ": 50, "়": 51, "া": 52, "ি": 53, "ী": 54, "ু": 55, "ূ": 56, "ৃ": 57, "ে": 58, "ৈ": 59, "ো": 60, "ৌ": 61, "্": 62, "ৎ": 63, "য়": 64}