text_html_output_1.png
from sklearn.feature_extraction.text import TfidfVectorizer

from sklearn.linear_model import LogisticRegression

from sklearn.metrics import log_loss

import nltk

import pandas as pd

import re

null_text = X.comment_text[2]

X.shape

X.isnull().sum()

y = X[['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']]
try:
    X.drop(['id', 'toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate'], axis=1, inplace=True)
except:
    pass

import re
import nltk
stop_words = set(nltk.corpus.stopwords.words('english'))
def preprocess_input(t):
    t = t.strip()
    z = re.findall('[A-Za-z]+', t)
    z = [a for a in z if len(a) > 3]
    wnlemma = nltk.stem.WordNetLemmatizer()
    z = [wnlemma.lemmatize(a) for a in z]
    z = [a for a in z if not a in stop_words]
    t = ' '.join(z)
    return t

X.comment_text = X.comment_text.apply(lambda x: preprocess_input(x))

from sklearn.feature_extraction.text import TfidfVectorizer
from nltk.tokenize import word_tokenize
vect = TfidfVectorizer(min_df=5, max_df=0.7, ngram_range=(1, 2), strip_accents='unicode', smooth_idf=True, sublinear_tf=True, max_features=10000)
vect = vect.fit(X['comment_text'])
X_vect = vect.transform(X['comment_text'])

X_vect.shape

test = pd.read_csv('../input/test.csv')
test.fillna(value=null_text, inplace=True)

t_id = test['id']
test.drop(['id'], axis=1, inplace=True)
test.comment_text = test.comment_text.apply(lambda z: preprocess_input(z))

X_test = vect.transform(test['comment_text'])

X_test.shape

y.iloc[:, 1]

from sklearn.linear_model import LogisticRegression
from sklearn.metrics import log_loss
cols = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
y_pred = pd.read_csv('../input/sample_submission.csv')
for c in cols:
    clf = LogisticRegression(C=4, solver='sag')
    clf.fit(X_vect, y[c])
    y_pred[c] = clf.predict_proba(X_test)[:, 1]
    pred_train = clf.predict_proba(X_vect)[:, 1]
    print('log loss:', log_loss(y[c], pred_train))