|
{ |
|
"additional_special_tokens": [ |
|
"0", |
|
"1", |
|
"2", |
|
"3", |
|
"4", |
|
"5", |
|
"6", |
|
"7", |
|
"8", |
|
"9", |
|
"ββ", |
|
"ββββ", |
|
"ββββββββ", |
|
"ββββββββββββ", |
|
"ββββββββββββββββ", |
|
"β", |
|
"β", |
|
"β", |
|
"β", |
|
"***", |
|
"----", |
|
"β\"", |
|
"βAuthor", |
|
"βTitle", |
|
"βTags", |
|
"βGenre", |
|
"βStyle", |
|
"βKnowledge", |
|
"βSummary", |
|
"βRating", |
|
"Type", |
|
"Characters", |
|
"Glossary", |
|
"<|spmspace|>", |
|
"<|spmspace|><|spmspace|>", |
|
"<|spmspace|><|spmspace|><|spmspace|><|spmspace|>", |
|
"<|spmspace|><|spmspace|><|spmspace|><|spmspace|><|spmspace|><|spmspace|><|spmspace|><|spmspace|>", |
|
"<|spmspace|><|spmspace|><|spmspace|><|spmspace|><|spmspace|><|spmspace|><|spmspace|><|spmspace|><|spmspace|><|spmspace|><|spmspace|><|spmspace|>", |
|
"<|spmspace|><|spmspace|><|spmspace|><|spmspace|><|spmspace|><|spmspace|><|spmspace|><|spmspace|><|spmspace|><|spmspace|><|spmspace|><|spmspace|><|spmspace|><|spmspace|><|spmspace|><|spmspace|>", |
|
"<|mtvocab|>", |
|
"<|mtvenglish|>", |
|
"<|mtvjapanese|>", |
|
"<|mtsentence|>", |
|
"<|mtsjapanese|>", |
|
"<|mtsenglish|>", |
|
"<|mtsentenceend|>", |
|
"<|mtvocabend|>", |
|
"<|mtend|>", |
|
"<|mask|>", |
|
"<|masksingle|>", |
|
"<|maskshort|>", |
|
"<|maskmedium|>", |
|
"<|masklong|>", |
|
"<|maskparagraph|>", |
|
"<|maskend|>", |
|
"<|fill|>", |
|
"<|fillend|>", |
|
"<|rubycover|>", |
|
"<|rubystart|>", |
|
"<|rubyend|>", |
|
"<|reserved0|>", |
|
"<|reserved1|>", |
|
"<|reserved2|>", |
|
"<|reserved3|>", |
|
"<|reserved4|>", |
|
"<|reserved5|>", |
|
"<|reserved6|>", |
|
"<|reserved7|>", |
|
"<|reserved8|>", |
|
"<|reserved9|>", |
|
"<|reserved10|>" |
|
], |
|
"bos_token": "<|startoftext|>", |
|
"eos_token": "<|endoftext|>", |
|
"pad_token": "<|pad|>", |
|
"unk_token": "<|unknown|>" |
|
} |
|
|