add tokenizer

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

vocab.json

@@ -1 +1 @@
-{"ẻ": 0, "a": 1, "ự": 2, "ỵ": 3, "á": 4, "ằ": 5, "b": 6, "ố": 7, "đ": 8, "g": 9, "p": 10, "s": 11, "ỹ": 12, "ọ": 13, "ủ": 14, "ẫ": 15, "ẹ": 16, "ò": 17, "à": 18, "ớ": 19, "â": 20, "v": 21, "ă": 22, "ữ": 23, "o": 24, "j": 25, "ợ": 26, "ễ": 27, "ể": 28, "ĩ": 29, "t": 30, "ó": 31, "ù": 32, "ỡ": 33, "h": 34, "ơ": 35, "ồ": 36, "ứ": 37, "l": 38, "ẩ": 39, "ẽ": 40, "ị": 41, "n": 42, "q": 43, "ở": 44, "r": 45, "x": 46, "õ": 47, "ề": 48, "ô": 49, "ổ": 50, "ê": 51, "ầ": 52, "ã": 53, "i": 54, "ũ": 55, "ạ": 56, "ử": 57, "y": 58, "ấ": 59, "é": 60, "ẳ": 61, "ư": 62, "ộ": 63, "ỉ": 64, "m": 66, "ế": 67, "ụ": 68, "ắ": 69, "e": 70, "w": 71, "c": 72, "ì": 73, "ỷ": 74, "ờ": 75, "z": 76, "k": 77, "è": 78, "í": 79, "ỏ": 80, "ý": 81, "ú": 82, "ậ": 83, "ừ": 84, "u": 85, "ả": 86, "ỗ": 87, "d": 88, "f": 89, "ẵ": 90, "ệ": 91, "ỳ": 92, "ặ": 93, "|": 65, "[UNK]": 94, "[PAD]": 95}
+{"ứ": 0, "đ": 1, "y": 2, "ả": 3, "ù": 4, "ĩ": 5, "h": 6, "ộ": 7, "e": 8, "è": 9, "ú": 10, "ồ": 11, "ỷ": 12, "ủ": 13, "é": 14, "ễ": 15, "ă": 16, "u": 17, "ệ": 18, "x": 19, "ô": 20, "ừ": 21, "ặ": 22, "ẹ": 23, "ổ": 24, "ẳ": 25, "ẻ": 26, "ỹ": 27, "ẩ": 28, "g": 29, "ề": 30, "ê": 31, "k": 32, "ẵ": 33, "ậ": 34, "c": 35, "ể": 36, "s": 37, "b": 38, "ụ": 39, "v": 40, "ờ": 41, "â": 42, "ử": 43, "w": 44, "ố": 45, "ớ": 46, "ấ": 47, "ỗ": 48, "í": 49, "t": 50, "l": 51, "ắ": 52, "ó": 53, "i": 54, "ế": 55, "ý": 56, "a": 57, "ỳ": 58, "ư": 59, "ợ": 60, "ị": 61, "ỏ": 62, "ở": 63, "ẽ": 64, "f": 65, "ự": 66, "m": 67, "ì": 68, "ạ": 69, "ỵ": 70, "ầ": 71, "ò": 72, "ã": 73, "ẫ": 74, "ọ": 75, "ỉ": 76, "p": 77, "o": 78, "q": 79, "ơ": 81, "ũ": 82, "ằ": 83, "n": 84, "õ": 85, "á": 86, "d": 87, "à": 88, "r": 89, "ữ": 90, "ỡ": 91, "z": 92, "j": 93, "|": 80, "[UNK]": 94, "[PAD]": 95}