add tokenizer

vocab.json

@@ -1 +1 @@
-{"g": 0, "ở": 1, "ồ": 2, "i": 3, "ã": 4, "â": 5, "ậ": 6, "ụ": 7, "n": 8, "ằ": 9, "ầ": 10, "í": 11, "ẽ": 12, "ớ": 13, "ắ": 14, "ẹ": 15, "õ": 16, "ó": 17, "ơ": 18, "đ": 19, "y": 20, "ỷ": 21, "ủ": 22, "ỗ": 23, "ổ": 24, "ỵ": 25, "ỳ": 26, "l": 27, "à": 28, "ừ": 29, "ẩ": 30, "ú": 31, "ũ": 32, "ấ": 33, "u": 34, "w": 35, "ĩ": 36, "ẻ": 37, "b": 38, "f": 39, "ợ": 40, "ý": 41, "ặ": 42, "ẵ": 43, "ả": 44, "ệ": 45, "ế": 46, "ọ": 47, "ạ": 48, "ờ": 49, "ỉ": 50, "è": 51, "ỡ": 52, "k": 53, "m": 55, "d": 56, "ỏ": 57, "c": 58, "ì": 59, "o": 60, "x": 61, "z": 62, "ê": 63, "e": 64, "á": 65, "ữ": 66, "ứ": 67, "h": 68, "ô": 69, "ỹ": 70, "p": 71, "ẳ": 72, "ă": 73, "ò": 74, "v": 75, "ẫ": 76, "é": 77, "ộ": 78, "ể": 79, "r": 80, "ù": 81, "ị": 82, "s": 83, "ễ": 84, "t": 85, "ử": 86, "ố": 87, "j": 88, "ự": 89, "q": 90, "ư": 91, "a": 92, "ề": 93, "|": 54, "[UNK]": 94, "[PAD]": 95}
+{"ư": 0, "ý": 1, "ặ": 2, "ỡ": 3, "ê": 4, "ó": 5, "ụ": 6, "í": 7, "ỳ": 8, "c": 9, "t": 10, "ỗ": 11, "n": 12, "ố": 13, "é": 14, "u": 15, "ă": 16, "h": 17, "e": 18, "ỷ": 19, "ổ": 20, "ù": 21, "a": 22, "è": 23, "ị": 24, "ỵ": 25, "ấ": 26, "ễ": 27, "y": 28, "ẳ": 29, "l": 30, "ể": 31, "ũ": 32, "i": 33, "x": 34, "ẹ": 35, "á": 36, "ẽ": 37, "r": 38, "ẵ": 39, "ỉ": 40, "b": 41, "ẩ": 42, "ạ": 43, "ỹ": 44, "ĩ": 46, "ẫ": 47, "ọ": 48, "ử": 49, "s": 50, "ỏ": 51, "g": 52, "ơ": 53, "ứ": 54, "ú": 55, "ậ": 56, "v": 57, "f": 58, "ở": 59, "o": 60, "ì": 61, "ế": 62, "ằ": 63, "đ": 64, "ệ": 65, "ề": 66, "ờ": 67, "ộ": 68, "ừ": 69, "ò": 70, "ã": 71, "ầ": 72, "õ": 73, "ủ": 74, "z": 75, "ữ": 76, "w": 77, "ự": 78, "ồ": 79, "ẻ": 80, "k": 81, "ớ": 82, "â": 83, "m": 84, "j": 85, "d": 86, "à": 87, "ắ": 88, "ô": 89, "q": 90, "p": 91, "ợ": 92, "ả": 93, "|": 45, "[UNK]": 94, "[PAD]": 95}