|
{ |
|
"additional_special_tokens": [ |
|
"<|endoftext|>", |
|
"[MASK]", |
|
"[gMASK]", |
|
"[sMASK]", |
|
"<sop>", |
|
"<eop>", |
|
"<|system|>", |
|
"<|user|>", |
|
"<|assistant|>", |
|
"<|observation|>", |
|
"<|begin_of_image|>", |
|
"<|end_of_image|>", |
|
"<|begin_of_video|>", |
|
"<|end_of_video|>", |
|
"<|begin_of_audio|>", |
|
"<|end_of_audio|>", |
|
"<|begin_of_transcription|>", |
|
"<|end_of_transcription|>", |
|
"<|code_prefix|>", |
|
"<|code_middle|>", |
|
"<|code_suffix|>", |
|
"/nothink" |
|
], |
|
"eos_token": { |
|
"content": "<|endoftext|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false |
|
}, |
|
"pad_token": { |
|
"content": "<|endoftext|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false |
|
} |
|
} |
|
|