add tokenizer

vocab.json

@@ -1 +1 @@
-{"ỹ": 0, "ủ": 1, "ố": 2, "d": 3, "ơ": 4, "w": 5, "s": 6, "ũ": 7, "c": 8, "é": 9, "ứ": 10, "í": 11, "ợ": 12, "ẽ": 13, "ụ": 14, "ấ": 15, "ô": 16, "ù": 17, "r": 18, "m": 19, "ỗ": 20, "â": 21, "j": 22, "ề": 23, "ẫ": 24, "ì": 25, "ý": 26, "ổ": 27, "g": 28, "ằ": 29, "ỳ": 30, "h": 31, "ừ": 32, "ở": 33, "ắ": 34, "e": 35, "à": 36, "ể": 37, "l": 38, "z": 39, "è": 40, "ă": 41, "ẳ": 42, "ọ": 43, "ê": 44, "ặ": 45, "ỉ": 46, "u": 47, "ĩ": 48, "õ": 49, "b": 50, "á": 51, "ạ": 52, "o": 53, "ẻ": 54, "ờ": 55, "ữ": 56, "ỡ": 57, "ự": 58, "ỏ": 59, "k": 60, "ệ": 61, "ộ": 62, "ớ": 63, "ỵ": 64, "ả": 65, "p": 66, "ẵ": 67, "ã": 68, "ế": 69, "a": 70, "ó": 71, "ầ": 72, "x": 73, "n": 74, "t": 75, "ú": 76, "f": 77, "ẹ": 78, "q": 79, "ử": 80, "v": 81, "ị": 82, "ò": 83, "ồ": 84, "ễ": 85, "đ": 87, "ẩ": 88, "ư": 89, "y": 90, "ỷ": 91, "i": 92, "ậ": 93, "|": 86, "[UNK]": 94, "[PAD]": 95}
+{"ữ": 0, "ầ": 1, "ệ": 2, "ẵ": 3, "ĩ": 4, "ọ": 5, "ỉ": 6, "m": 7, "o": 8, "ử": 9, "ồ": 10, "ẹ": 11, "ấ": 12, "y": 13, "ũ": 14, "ị": 15, "ẻ": 16, "s": 17, "ớ": 18, "ả": 19, "ố": 20, "ỏ": 21, "ô": 22, "ạ": 23, "đ": 24, "ù": 25, "b": 26, "ỵ": 27, "ẩ": 28, "l": 29, "t": 30, "j": 31, "ă": 32, "ỳ": 33, "ề": 34, "f": 35, "ẳ": 36, "ó": 37, "ừ": 38, "ở": 39, "h": 40, "k": 41, "e": 42, "c": 43, "ụ": 45, "v": 46, "ờ": 47, "ễ": 48, "ỡ": 49, "p": 50, "ơ": 51, "ộ": 52, "z": 53, "é": 54, "ứ": 55, "ẫ": 56, "ế": 57, "ú": 58, "ể": 59, "r": 60, "ý": 61, "g": 62, "õ": 63, "q": 64, "n": 65, "ủ": 66, "í": 67, "ặ": 68, "ã": 69, "u": 70, "è": 71, "ự": 72, "ì": 73, "a": 74, "w": 75, "d": 76, "ò": 77, "ỗ": 78, "ẽ": 79, "ỹ": 80, "i": 81, "ư": 82, "ỷ": 83, "á": 84, "â": 85, "à": 86, "x": 87, "ậ": 88, "ắ": 89, "ằ": 90, "ợ": 91, "ổ": 92, "ê": 93, "|": 44, "[UNK]": 94, "[PAD]": 95}