text_plain_output_1.png
from sklearn.feature_extraction.text import TfidfVectorizer

from sklearn.linear_model import LogisticRegression

from sklearn.metrics import log_loss

import numpy as np

import pandas as pd

train = pd.read_csv('../input/train.csv')
test = pd.read_csv('../input/test.csv')
sampleSubmission = pd.read_csv('../input/sample_submission.csv')

col = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
trainTxt = train['comment_text']
testTxt = test['comment_text']
trainTxt = trainTxt.fillna('unknown')
testTxt = testTxt.fillna('unknown')
combinedTxt = pd.concat([trainTxt, testTxt], axis=0)

vect = TfidfVectorizer(decode_error='ignore', use_idf=True, smooth_idf=True, min_df=10, ngram_range=(1, 3), lowercase=True, stop_words='english')

combinedDtm = vect.fit_transform(combinedTxt)
trainDtm = combinedDtm[:train.shape[0]]
testDtm = vect.transform(testTxt)

lrpreds = np.zeros((test.shape[0], len(col)))
nbpreds = np.zeros((test.shape[0], len(col)))
svmpreds = np.zeros((test.shape[0], len(col)))
xgbpreds = np.zeros((test.shape[0], len(col)))

loss = []
for i, j in enumerate(col):
    lr = LogisticRegression(C=4)
    lr.fit(trainDtm, train[j])
    lrpreds[:, i] = lr.predict_proba(testDtm)[:, 1]
    train_preds = lr.predict_proba(trainDtm)[:, 1]
    loss.append(log_loss(train[j], train_preds))
np.mean(loss)