ababio commited on
Commit
af9421c
·
verified ·
1 Parent(s): 1c4e32e

Upload tokenizer

Browse files
special_tokens_map.json CHANGED
@@ -9,7 +9,7 @@
9
  "rstrip": false,
10
  "single_word": false
11
  },
12
- "pad_token": "</s>",
13
  "sep_token": "</s>",
14
  "unk_token": "<unk>"
15
  }
 
9
  "rstrip": false,
10
  "single_word": false
11
  },
12
+ "pad_token": "<pad>",
13
  "sep_token": "</s>",
14
  "unk_token": "<unk>"
15
  }
tokenizer.json CHANGED
@@ -12,9 +12,9 @@
12
  },
13
  "direction": "Right",
14
  "pad_to_multiple_of": null,
15
- "pad_id": 2,
16
  "pad_type_id": 0,
17
- "pad_token": "</s>"
18
  },
19
  "added_tokens": [
20
  {
 
12
  },
13
  "direction": "Right",
14
  "pad_to_multiple_of": null,
15
+ "pad_id": 1,
16
  "pad_type_id": 0,
17
+ "pad_token": "<pad>"
18
  },
19
  "added_tokens": [
20
  {
tokenizer_config.json CHANGED
@@ -50,7 +50,7 @@
50
  "extra_special_tokens": {},
51
  "mask_token": "<mask>",
52
  "model_max_length": 1000000000000000019884624838656,
53
- "pad_token": "</s>",
54
  "sep_token": "</s>",
55
  "tokenizer_class": "BartTokenizer",
56
  "trim_offsets": true,
 
50
  "extra_special_tokens": {},
51
  "mask_token": "<mask>",
52
  "model_max_length": 1000000000000000019884624838656,
53
+ "pad_token": "<pad>",
54
  "sep_token": "</s>",
55
  "tokenizer_class": "BartTokenizer",
56
  "trim_offsets": true,