FaisalGh commited on
Commit
ec4059e
·
verified ·
1 Parent(s): 28b793c

Upload special_tokens_map.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. special_tokens_map.json +1 -12
special_tokens_map.json CHANGED
@@ -1,12 +1 @@
1
- {
2
- "bos_token": "<|startoftext|>",
3
- "eos_token": "<|endoftext|>",
4
- "pad_token": "<|pad|>", // Critical change
5
- "unk_token": "<|endoftext|>",
6
- "additional_special_tokens": [
7
- {"content": "[JOKE]", "single_word": false},
8
- {"content": "[PUNCHLINE]", "single_word": false},
9
- {"content": "<|setup|>", "single_word": false},
10
- {"content": "<|punchline|>", "single_word": false}
11
- ]
12
- }
 
1
+ {"bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "pad_token": "<|endoftext|>", "unk_token": "<|endoftext|>", "additional_special_tokens": ["[JOKE]", "[ENDJOKE]", "<|setup|>", "<|punchline|>"]}