image_output_1.png
(train_df.shape, val_df.shape)

train_X = train_df['entire_text'].fillna('##').values
val_X = val_df['entire_text'].fillna('##').values
print('before tokenization')
print(train_X.shape)
print(val_X.shape)