text_plain_output_1.png
from aitextgen import aitextgen

from aitextgen.TokenDataset import TokenDataset

from aitextgen.tokenizers import train_tokenizer

from aitextgen.utils import GPTNeoConfigCPU

file_name = '/kaggle/input/annomi/dataset.txt'

train_tokenizer(file_name)
tokenizer_file = 'aitextgen.tokenizer.json'
config = GPTNeoConfigCPU()
ai = aitextgen(tokenizer_file=tokenizer_file, config=config)
data = TokenDataset(file_name, tokenizer_file=tokenizer_file, block_size=64)