add tokenizer

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

vocab.json

@@ -1 +1 @@
-{"o": 0, "ơ": 1, "ế": 2, "ỵ": 3, "ũ": 4, "ọ": 6, "k": 7, "ẩ": 8, "f": 9, "t": 10, "ó": 11, "ợ": 12, "q": 13, "ữ": 14, "c": 15, "ỳ": 16, "w": 17, "r": 18, "ỏ": 19, "ú": 20, "é": 21, "õ": 22, "ẽ": 23, "ỉ": 24, "ù": 25, "ộ": 26, "ừ": 27, "u": 28, "ã": 29, "ở": 30, "m": 31, "ă": 32, "á": 33, "ỷ": 34, "ệ": 35, "ồ": 36, "d": 37, "ẻ": 38, "s": 39, "ả": 40, "ề": 41, "ờ": 42, "ố": 43, "ẵ": 44, "ị": 45, "ì": 46, "ư": 47, "ằ": 48, "ẳ": 49, "l": 50, "j": 51, "ứ": 52, "p": 53, "ẹ": 54, "ê": 55, "e": 56, "z": 57, "ổ": 58, "đ": 59, "ặ": 60, "y": 61, "è": 62, "ĩ": 63, "i": 64, "ỡ": 65, "â": 66, "ý": 67, "ò": 68, "ầ": 69, "g": 70, "à": 71, "h": 72, "x": 73, "ễ": 74, "í": 75, "ỹ": 76, "ẫ": 77, "ô": 78, "ự": 79, "ắ": 80, "ậ": 81, "ụ": 82, "b": 83, "ử": 84, "ủ": 85, "ấ": 86, "ể": 87, "ớ": 88, "ỗ": 89, "ạ": 90, "v": 91, "n": 92, "a": 93, "|": 5, "[UNK]": 94, "[PAD]": 95}
+{"ỹ": 0, "ủ": 1, "ố": 2, "d": 3, "ơ": 4, "w": 5, "s": 6, "ũ": 7, "c": 8, "é": 9, "ứ": 10, "í": 11, "ợ": 12, "ẽ": 13, "ụ": 14, "ấ": 15, "ô": 16, "ù": 17, "r": 18, "m": 19, "ỗ": 20, "â": 21, "j": 22, "ề": 23, "ẫ": 24, "ì": 25, "ý": 26, "ổ": 27, "g": 28, "ằ": 29, "ỳ": 30, "h": 31, "ừ": 32, "ở": 33, "ắ": 34, "e": 35, "à": 36, "ể": 37, "l": 38, "z": 39, "è": 40, "ă": 41, "ẳ": 42, "ọ": 43, "ê": 44, "ặ": 45, "ỉ": 46, "u": 47, "ĩ": 48, "õ": 49, "b": 50, "á": 51, "ạ": 52, "o": 53, "ẻ": 54, "ờ": 55, "ữ": 56, "ỡ": 57, "ự": 58, "ỏ": 59, "k": 60, "ệ": 61, "ộ": 62, "ớ": 63, "ỵ": 64, "ả": 65, "p": 66, "ẵ": 67, "ã": 68, "ế": 69, "a": 70, "ó": 71, "ầ": 72, "x": 73, "n": 74, "t": 75, "ú": 76, "f": 77, "ẹ": 78, "q": 79, "ử": 80, "v": 81, "ị": 82, "ò": 83, "ồ": 84, "ễ": 85, "đ": 87, "ẩ": 88, "ư": 89, "y": 90, "ỷ": 91, "i": 92, "ậ": 93, "|": 86, "[UNK]": 94, "[PAD]": 95}