application_vnd.jupyter.stderr_output_1.png
from pytorch_pretrained_bert import BertConfig

from pytorch_pretrained_bert import BertTokenizer, BertForSequenceClassification, BertAdam

from tqdm import tqdm

import numpy as np

import pandas as pd

import torch

device = torch.device('cuda')

def convert_lines(example, max_seq_length, tokenizer):
    max_seq_length -= 2
    all_tokens = []
    longer = 0
    for text in tqdm(example):
        tokens_a = tokenizer.tokenize(text)
        if len(tokens_a) > max_seq_length:
            tokens_a = tokens_a[:max_seq_length]
            longer += 1
        one_token = tokenizer.convert_tokens_to_ids(['[CLS]'] + tokens_a + ['[SEP]']) + [0] * (max_seq_length - len(tokens_a))
        all_tokens.append(one_token)
    return np.array(all_tokens)

MAX_SEQUENCE_LENGTH = 200
SEED = 42
BATCH_SIZE = 32
BERT_MODEL_PATH = '../input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'
np.random.seed(SEED)
torch.manual_seed(SEED)
torch.cuda.manual_seed(SEED)
torch.backends.cudnn.deterministic = True

bert_config = BertConfig('../input/bertinference/bert_config.json')
tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH, cache_dir=None, do_lower_case=True)
BERT_SMALL_PATH = '../input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'
BERT_LARGE_PATH = '../input/bert-pretrained-models/uncased_l-24_h-1024_a-16/uncased_L-24_H-1024_A-16/'

test_df = pd.read_csv('../input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
test_df['comment_text'] = test_df['comment_text'].astype(str)
X_test = convert_lines(test_df['comment_text'].fillna('DUMMY_VALUE'), MAX_SEQUENCE_LENGTH, tokenizer)

model = BertForSequenceClassification(bert_config, num_labels=1)
model.load_state_dict(torch.load('../input/bertinference/pytorch_bert_6.bin'))
model.to(device)
for param in model.parameters():
    param.requires_grad = False
model.eval()

test_preds = np.zeros(len(X_test))
test = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test, batch_size=32, shuffle=False)
tk0 = tqdm(test_loader)
for i, (x_batch,) in enumerate(tk0):
    pred = model(x_batch.to(device), attention_mask=(x_batch > 0).to(device), labels=None)
    test_preds[i * 32:(i + 1) * 32] = pred[:, 0].detach().cpu().squeeze().numpy()
test_pred = torch.sigmoid(torch.tensor(test_preds)).numpy().ravel()