from torchtext import data
PAD_TOKEN = ‘’
TEXT = data.Field(sequential = True,batch_first = True,lower = True,pad_token = PAD_TOKEN)
LABEL = data.Field(sequential = False,batch_first = True,unk_tokn=None)
datafields = [(“PhraseId”, None), # 不需要的filed设置为None
(“SentenceId”, None),
(‘Phrase’, TEXT),
(‘Sentiment’, LABEL)]
train, val, test = data.TabularDataset.splits(
path=’./data/’, train=‘train.tsv’,
validation=‘val.tsv’, test=‘test.tsv’, format=‘tsv’,
fields=datafields)
print(train.shape,val.shape,test.shape)
image870×81 10.1 KB