application_vnd.jupyter.stderr_output_1.png
from tqdm import tqdm

import pandas as pd

import torch

test_data = pd.read_pickle('/kaggle/input/nlp-with-disaster-tweets-eda-cleaning-and-bert/test.pkl')
test_text = test_data.text_cleaned.apply(lambda x: x.lower()).values.tolist()

bert.eval()
testtext_embedding = []
with torch.no_grad():
    for t in tqdm(test_text):
        t = tokenizer(t, return_tensors='pt').to(device)
        output = bert(**t).pooler_output.to('cpu').numpy()
        testtext_embedding.append(output)

test_data = pd.read_pickle('/kaggle/input/nlp-with-disaster-tweets-eda-cleaning-and-bert/test.pkl')
test_data.to_csv('preprocessed_test.csv')

train_data = pd.read_pickle('/kaggle/input/nlp-with-disaster-tweets-eda-cleaning-and-bert/train.pkl')
train_text = train_data.text_cleaned.apply(lambda x: x.lower()).values

bert.eval()
traintext_embedding = []
with torch.no_grad():
    for t in tqdm(train_text):
        t = tokenizer(t, return_tensors='pt').to(device)
        output = bert(**t).pooler_output.to('cpu').numpy()
        traintext_embedding.append(output)