Update tokenizer.json
Browse files- tokenizer.json +6 -0
tokenizer.json
CHANGED
@@ -2829,6 +2829,12 @@
|
|
2829 |
"[\\IH1-1]": 2658,
|
2830 |
"[\\Tl]": 2659
|
2831 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
2832 |
"unk_token": "[UNK]"
|
2833 |
}
|
2834 |
}
|
|
|
2829 |
"[\\IH1-1]": 2658,
|
2830 |
"[\\Tl]": 2659
|
2831 |
},
|
2832 |
+
"merges":[
|
2833 |
+
"[C] [C]",
|
2834 |
+
"[C] [=C]",
|
2835 |
+
"[=N] [C]",
|
2836 |
+
"[=Branch1] [C]"
|
2837 |
+
],
|
2838 |
"unk_token": "[UNK]"
|
2839 |
}
|
2840 |
}
|