text_plain_output_1.png
from datasets import load_dataset

from transformers import DataCollatorWithPadding, BertForSequenceClassification, AutoModelForSequenceClassification, AutoTokenizer, Trainer, TrainingArguments

train_dataset = load_dataset('csv', data_files='train_data_clean.csv')
val_dataset = load_dataset('csv', data_files='val_data_clean.csv')
test_dataset = load_dataset('csv', data_files='test_data_clean.csv')

train_dataset = train_dataset.rename_column('text', 'sentence1')
train_dataset = train_dataset.rename_column('target', 'label')
val_dataset = val_dataset.rename_column('text', 'sentence1')
val_dataset = val_dataset.rename_column('target', 'label')
test_dataset = test_dataset.rename_column('text', 'sentence1')

checkpoint = '../input/transformers/roberta-base'

tokenizer = AutoTokenizer.from_pretrained(checkpoint)
data_collator = DataCollatorWithPadding(tokenizer=tokenizer)

def tokenize_function(dset):
    return tokenizer(dset['sentence1'])

tokenized_train_dataset = train_dataset.map(tokenize_function, batched=True)
tokenized_val_dataset = val_dataset.map(tokenize_function, batched=True)
tokenized_test_dataset = test_dataset.map(tokenize_function, batched=True)

tokenized_train_dataset