HoangHa commited on
Commit
dbc8d56
·
verified ·
1 Parent(s): d2fddd0

Update tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +6 -0
tokenizer.json CHANGED
@@ -2829,6 +2829,12 @@
2829
  "[\\IH1-1]": 2658,
2830
  "[\\Tl]": 2659
2831
  },
 
 
 
 
 
 
2832
  "unk_token": "[UNK]"
2833
  }
2834
  }
 
2829
  "[\\IH1-1]": 2658,
2830
  "[\\Tl]": 2659
2831
  },
2832
+ "merges":[
2833
+ "[C] [C]",
2834
+ "[C] [=C]",
2835
+ "[=N] [C]",
2836
+ "[=Branch1] [C]"
2837
+ ],
2838
  "unk_token": "[UNK]"
2839
  }
2840
  }