Eugene Cheah (picocreator) commited on
Commit
d8cac50
1 Parent(s): c30b41b

revert to 3B style tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -5
tokenizer_config.json CHANGED
@@ -8,9 +8,5 @@
8
  "tokenization_rwkv_world.RWKVWorldTokenizer",
9
  null
10
  ]
11
- },
12
-
13
- "clean_up_tokenization_spaces": true,
14
- "errors": "replace",
15
- "model_max_length": 1000000000000000019884624838656
16
  }
 
8
  "tokenization_rwkv_world.RWKVWorldTokenizer",
9
  null
10
  ]
11
+ }
 
 
 
 
12
  }