ABHIiiii1 commited on
Commit
ca7c93d
1 Parent(s): 1334e10

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +2 -2
  2. tokenizer_config.json +1 -1
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:92262b29204f8fdc169a63f9005a0e311a16262cef4d96ecfe2a7ed638662ed3
3
- size 13632172
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:09216b42d2697b7b4a26ac05ff09ba8bf52dc19b896c5ceee8bbff9f39055322
3
+ size 13631919
tokenizer_config.json CHANGED
@@ -47,7 +47,7 @@
47
  "do_lower_case": false,
48
  "full_tokenizer_file": null,
49
  "mask_token": "[MASK]",
50
- "model_max_length": 256,
51
  "never_split": null,
52
  "pad_token": "[PAD]",
53
  "sep_token": "[SEP]",
 
47
  "do_lower_case": false,
48
  "full_tokenizer_file": null,
49
  "mask_token": "[MASK]",
50
+ "model_max_length": 512,
51
  "never_split": null,
52
  "pad_token": "[PAD]",
53
  "sep_token": "[SEP]",