Commit
·
c3c4a63
1
Parent(s):
ff88cd9
Update README.md
Browse files
README.md
CHANGED
|
@@ -12,5 +12,82 @@ inference: false
|
|
| 12 |
# explain
|
| 13 |
- base tokenizer from [baichuan-7B](https://github.com/baichuan-inc/baichuan-7B), this model add some maths symbol
|
| 14 |
|
| 15 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
|
|
|
|
| 12 |
# explain
|
| 13 |
- base tokenizer from [baichuan-7B](https://github.com/baichuan-inc/baichuan-7B), this model add some maths symbol
|
| 14 |
|
| 15 |
+
```
|
| 16 |
+
"approx": 64000,
|
| 17 |
+
"arccos": 64001,
|
| 18 |
+
"arcsin": 64002,
|
| 19 |
+
"arctan": 64003,
|
| 20 |
+
"backsim": 64004,
|
| 21 |
+
"begin{matrix}": 64005,
|
| 22 |
+
"begin{vmatrix}": 64006,
|
| 23 |
+
"beta": 64007,
|
| 24 |
+
"cdot": 64008,
|
| 25 |
+
"cdots": 64009,
|
| 26 |
+
"cong": 64010,
|
| 27 |
+
"delta": 64011,
|
| 28 |
+
"dot": 64012,
|
| 29 |
+
"downarrow": 64013,
|
| 30 |
+
"end{matrix}": 64014,
|
| 31 |
+
"end{vmatrix}": 64015,
|
| 32 |
+
"exists": 64016,
|
| 33 |
+
"forall": 64017,
|
| 34 |
+
"gamma": 64018,
|
| 35 |
+
"geq": 64019,
|
| 36 |
+
"infty": 64020,
|
| 37 |
+
"lambda": 64021,
|
| 38 |
+
"left.": 64022,
|
| 39 |
+
"left[": 64023,
|
| 40 |
+
"left{": 64024,
|
| 41 |
+
"leftrightarrow": 64025,
|
| 42 |
+
"leq": 64026,
|
| 43 |
+
"lg": 64027,
|
| 44 |
+
"neq": 64028,
|
| 45 |
+
"notin": 64029,
|
| 46 |
+
"omega": 64030,
|
| 47 |
+
"overline": 64031,
|
| 48 |
+
"overrightarrow": 64032,
|
| 49 |
+
"prime": 64033,
|
| 50 |
+
"psi": 64034,
|
| 51 |
+
"rho": 64035,
|
| 52 |
+
"right.": 64036,
|
| 53 |
+
"right}": 64037,
|
| 54 |
+
"right]": 64038,
|
| 55 |
+
"rightarrow": 64039,
|
| 56 |
+
"sigma": 64040,
|
| 57 |
+
"subset": 64041,
|
| 58 |
+
"subseteq": 64042,
|
| 59 |
+
"supset": 64043,
|
| 60 |
+
"supseteq": 64044,
|
| 61 |
+
"tan": 64045,
|
| 62 |
+
"textcircled": 64046,
|
| 63 |
+
"text{": 64047,
|
| 64 |
+
"therefore": 64048,
|
| 65 |
+
"theta": 64049,
|
| 66 |
+
"varepsilon": 64050,
|
| 67 |
+
"varphi": 64051,
|
| 68 |
+
"widehat": 64052,
|
| 69 |
+
"xrightarrow": 64053,
|
| 70 |
+
"…": 64054,
|
| 71 |
+
"℃": 64055,
|
| 72 |
+
"①": 64056,
|
| 73 |
+
"②": 64057,
|
| 74 |
+
"③": 64058,
|
| 75 |
+
"④": 64059,
|
| 76 |
+
"⑤": 64060,
|
| 77 |
+
"⑥": 64061,
|
| 78 |
+
"⑦": 64062,
|
| 79 |
+
"⑧": 64063,
|
| 80 |
+
"⑨": 64064,
|
| 81 |
+
"⑩": 64065,
|
| 82 |
+
"％": 64066,
|
| 83 |
+
"（": 64067,
|
| 84 |
+
"）": 64068,
|
| 85 |
+
"＋": 64069,
|
| 86 |
+
"－": 64070,
|
| 87 |
+
"．": 64071,
|
| 88 |
+
"；": 64072,
|
| 89 |
+
"＜": 64073,
|
| 90 |
+
"＝": 64074,
|
| 91 |
+
"＞": 64075
|
| 92 |
+
```
|
| 93 |
|