「tokenizer_class」:「GPTNeoXTokenizerFast」に変更

#2
by alfredplpl - opened
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -23,6 +23,6 @@
23
  "eos_token": "<|endoftext|>",
24
  "model_max_length": 1000000000000000019884624838656,
25
  "pad_token": "<|padding|>",
26
- "tokenizer_class": "GPTNeoXTokenizer",
27
  "unk_token": "<|endoftext|>"
28
  }
 
23
  "eos_token": "<|endoftext|>",
24
  "model_max_length": 1000000000000000019884624838656,
25
  "pad_token": "<|padding|>",
26
+ "tokenizer_class": "GPTNeoXTokenizerFast",
27
  "unk_token": "<|endoftext|>"
28
  }