add tokenizer

vocab.json

@@ -1 +1 @@
-{"ẳ": 0, "ũ": 1, "ụ": 2, "ệ": 3, "ò": 4, "ô": 5, "ú": 6, "â": 7, "ự": 8, "b": 9, "ấ": 10, "r": 11, "ề": 12, "j": 13, "ậ": 14, "a": 15, "ù": 16, "x": 17, "ặ": 18, "ă": 19, "ỉ": 20, "p": 21, "à": 22, "ộ": 23, "ế": 24, "m": 25, "ê": 26, "ỵ": 27, "ơ": 28, "ầ": 29, "w": 30, "ớ": 31, "ể": 32, "ư": 33, "e": 34, "k": 35, "ẽ": 36, "d": 37, "ẩ": 38, "õ": 39, "v": 40, "t": 41, "s": 42, "c": 43, "q": 44, "ừ": 45, "ọ": 46, "ổ": 47, "ứ": 48, "ẹ": 49, "ẵ": 50, "ó": 51, "ỹ": 52, "ồ": 53, "ắ": 54, "f": 55, "á": 56, "ạ": 57, "ở": 58, "ỏ": 59, "ả": 60, "ị": 61, "ữ": 62, "è": 63, "ỗ": 64, "ỷ": 65, "ỳ": 66, "i": 67, "z": 68, "é": 69, "ì": 70, "đ": 71, "ý": 72, "ã": 74, "ỡ": 75, "ợ": 76, "ằ": 77, "y": 78, "ĩ": 79, "í": 80, "ễ": 81, "ờ": 82, "ố": 83, "ẫ": 84, "n": 85, "u": 86, "h": 87, "o": 88, "l": 89, "ủ": 90, "ử": 91, "ẻ": 92, "g": 93, "|": 73, "[UNK]": 94, "[PAD]": 95}
+{"ỵ": 0, "ỳ": 1, "w": 2, "i": 3, "h": 4, "ỡ": 5, "ế": 6, "ệ": 7, "ợ": 8, "ó": 9, "v": 10, "ụ": 11, "r": 12, "ă": 13, "ỹ": 14, "ẳ": 15, "ọ": 16, "ỉ": 17, "á": 18, "s": 19, "p": 20, "n": 21, "o": 22, "ộ": 23, "z": 24, "ạ": 25, "u": 26, "g": 27, "d": 28, "ẻ": 29, "q": 30, "ầ": 31, "õ": 32, "a": 33, "ẫ": 34, "ô": 35, "ứ": 36, "ừ": 37, "ỗ": 38, "ẵ": 39, "x": 40, "c": 41, "ờ": 42, "j": 43, "ủ": 44, "ỷ": 45, "ậ": 46, "b": 47, "ể": 48, "ổ": 49, "ằ": 50, "à": 51, "ã": 53, "ồ": 54, "ớ": 55, "ê": 56, "ẹ": 57, "ẩ": 58, "t": 59, "ự": 60, "é": 61, "ò": 62, "ú": 63, "ý": 64, "ỏ": 65, "y": 66, "ĩ": 67, "ặ": 68, "k": 69, "ố": 70, "f": 71, "â": 72, "ắ": 73, "ấ": 74, "e": 75, "l": 76, "ị": 77, "ử": 78, "ề": 79, "ơ": 80, "í": 81, "ữ": 82, "ũ": 83, "è": 84, "ễ": 85, "ù": 86, "m": 87, "ở": 88, "ả": 89, "ẽ": 90, "ư": 91, "ì": 92, "đ": 93, "|": 52, "[UNK]": 94, "[PAD]": 95}