text_plain_output_1.png
from sklearn.ensemble import RandomForestClassifier

from sklearn.linear_model import LogisticRegression

from sklearn.linear_model import LogisticRegression

from sklearn.metrics import f1_score

from sklearn.model_selection import cross_val_score, train_test_split, GridSearchCV

from tqdm import notebook

import numpy as np

import pandas as pd

import torch

import transformers as ppb

DEVICE = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')

try:
    data = pd.read_csv('https://code.s3.yandex.net/datasets/toxic_comments.csv')
except:
    data = pd.read_csv('/kaggle/input/toxic-commentscsv/toxic_comments.csv')

data = data.drop('Unnamed: 0', axis=1)

model_class, tokenizer_class, pretrained_weights = (ppb.DistilBertModel, ppb.DistilBertTokenizer, 'distilbert-base-uncased')
tokenizer = tokenizer_class.from_pretrained(pretrained_weights)
model = model_class.from_pretrained(pretrained_weights).to(DEVICE)

tokenized = data['text'].apply(lambda x: tokenizer.encode(x, add_special_tokens=True, truncation=True))

max_len = 0
for i in tokenized.values:
    if len(i) > max_len:
        max_len = len(i)
padded = np.array([i + [0] * (max_len - len(i)) for i in tokenized.values])
attention_mask = np.where(padded != 0, 1, 0)

batch_size = 400
embeddings = []
for i in notebook.tqdm(range(padded.shape[0] // batch_size)):
    batch = torch.LongTensor(padded[batch_size * i:batch_size * (i + 1)]).to(DEVICE)
    attention_mask_batch = torch.LongTensor(attention_mask[batch_size * i:batch_size * (i + 1)]).to(DEVICE)
    with torch.no_grad():
        batch_embeddings = model(batch, attention_mask=attention_mask_batch)
    embeddings.append(batch_embeddings[0][:, 0, :].cpu().detach().numpy())

features = np.concatenate(embeddings)

features.shape

target = data['toxic']

target = target[:159200]

params = {'C': [92], 'penalty': ['l2'], 'solver': ['lbfgs']}
lr_clf = LogisticRegression(max_iter=30000)
lr_model = GridSearchCV(lr_clf, param_grid=params, scoring='f1', cv=3)
lr_model.fit(features, target)
lr_model.best_score_
f1_score(target_train, lr_model.predict(features_train))

params = {'n_estimators': [10, 40, 100], 'max_depth': [1, 4, 9]}
rfr = RandomForestClassifier()
rfr_model = GridSearchCV(rfr, param_grid=params, scoring='f1', cv=3)
rfr_model.fit(features_train, target_train)

rfr_model.best_score_