kiyono commited on
Commit
626ec58
1 Parent(s): 52c0a6f

fix zenkaku option

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -0
tokenizer_config.json CHANGED
@@ -15,6 +15,7 @@
15
  },
16
  "sp_model_kwargs": {},
17
  "do_lower_case": false,
 
18
  "do_word_tokenize": true,
19
  "do_subword_tokenize": true,
20
  "word_tokenizer_type": "mecab",
 
15
  },
16
  "sp_model_kwargs": {},
17
  "do_lower_case": false,
18
+ "do_zenkaku": true,
19
  "do_word_tokenize": true,
20
  "do_subword_tokenize": true,
21
  "word_tokenizer_type": "mecab",