Lakoc
/

cz_ec_bpe1000

Lakoc commited on Jan 17

Commit

cb13848

verified ·

1 Parent(s): e7c10c0

Update tokenizer_config.json

Files changed (1) hide show

tokenizer_config.json CHANGED Viewed

@@ -58,7 +58,7 @@
     }
   },
   "bos_token": "(BOS)",
-  "clean_up_tokenization_spaces": true,
   "eos_token": "(EOS)",
   "mask_token": "(MASK)",
   "model_max_length": 1000000000000000019884624838656,

     }
   },
   "bos_token": "(BOS)",
+  "clean_up_tokenization_spaces": false,
   "eos_token": "(EOS)",
   "mask_token": "(MASK)",
   "model_max_length": 1000000000000000019884624838656,