move the added tokens to the main vocab
Browse files- added_tokens.json +1 -32
added_tokens.json
CHANGED
|
@@ -1605,36 +1605,5 @@
|
|
| 1605 |
"<|vi|>": 50278,
|
| 1606 |
"<|yi|>": 50335,
|
| 1607 |
"<|yo|>": 50325,
|
| 1608 |
-
"<|zh|>": 50260
|
| 1609 |
-
"֑": 51865,
|
| 1610 |
-
"֒": 51866,
|
| 1611 |
-
"֓": 51867,
|
| 1612 |
-
"֔": 51868,
|
| 1613 |
-
"֕": 51869,
|
| 1614 |
-
"֖": 51870,
|
| 1615 |
-
"֗": 51871,
|
| 1616 |
-
"֘": 51872,
|
| 1617 |
-
"֙": 51873,
|
| 1618 |
-
"֚": 51874,
|
| 1619 |
-
"֛": 51875,
|
| 1620 |
-
"֜": 51876,
|
| 1621 |
-
"֝": 51877,
|
| 1622 |
-
"֞": 51878,
|
| 1623 |
-
"֟": 51879,
|
| 1624 |
-
"֠": 51880,
|
| 1625 |
-
"֡": 51881,
|
| 1626 |
-
"֢": 51882,
|
| 1627 |
-
"֣": 51883,
|
| 1628 |
-
"֤": 51884,
|
| 1629 |
-
"֥": 51885,
|
| 1630 |
-
"֦": 51886,
|
| 1631 |
-
"֧": 51887,
|
| 1632 |
-
"֨": 51888,
|
| 1633 |
-
"֩": 51889,
|
| 1634 |
-
"֪": 51890,
|
| 1635 |
-
"֫": 51891,
|
| 1636 |
-
"֬": 51892,
|
| 1637 |
-
"֭": 51893,
|
| 1638 |
-
"֮": 51894,
|
| 1639 |
-
"ֽ": 51895
|
| 1640 |
}
|
|
|
|
| 1605 |
"<|vi|>": 50278,
|
| 1606 |
"<|yi|>": 50335,
|
| 1607 |
"<|yo|>": 50325,
|
| 1608 |
+
"<|zh|>": 50260
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1609 |
}
|