text_plain_output_1.png
from sklearn import model_selection

from sklearn.feature_extraction.text import TfidfVectorizer

from sklearn.linear_model import LogisticRegression

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import re

import time

train = pd.read_csv('../input/train.csv')
test = pd.read_csv('../input/test.csv')
sub = pd.read_csv('../input/sample_submission.csv')

train.shape

test.shape

train.dtypes

test.dtypes

import re
def clean_text(text):
    text = re.sub("[^A-za-z0-9^,?!.\\/'+-=]", ' ', text)
    text = re.sub("what's", 'what is ', text)
    text = re.sub("\\'s", ' ', text)
    text = re.sub("\\'ve", ' have ', text)
    text = re.sub("can't", 'cannot ', text)
    text = re.sub("n't", ' not ', text)
    text = re.sub("i'm", 'i am ', text)
    text = re.sub("\\'re", ' are ', text)
    text = re.sub("\\'d", ' would ', text)
    text = re.sub("\\'ll", ' will ', text)
    text = re.sub("\\'scuse", ' excuse ', text)
    text = re.sub(',', ' ', text)
    text = re.sub('\\.', ' ', text)
    text = re.sub('!', ' _exclamationmark_ ', text)
    text = re.sub('\\?', ' _questionmark_ ', text)
    return text

def build_data_set(ngram=3, stem=False, max_features=2000, min_df=2, remove_stopwords=True):
    train = pd.read_csv('../input/train.csv')
    test = pd.read_csv('../input/test.csv')
    test.fillna('missing', inplace=True)
    clean_train_comments = []
    for i in range(train.shape[0]):
        clean_train_comments.append(clean_text(train['comment_text'][i]))
    for i in range(test.shape[0]):
        clean_train_comments.append(clean_text(test['comment_text'][i]))
    qs = pd.Series(clean_train_comments).astype(str)
    if not stem:
        vect = TfidfVectorizer(analyzer=u'word', stop_words='english', min_df=min_df, ngram_range=(1, ngram), max_features=max_features)
        ifidf_vect = vect.fit_transform(qs)
        X = ifidf_vect.toarray()
        X_train = X[:train.shape[0]]
        X_test = X[train.shape[0]:]
    else:
        vect_stem = StemmedTfidfVectorizer(analyzer=u'word', stop_words='english', min_df=min_df, ngram_range=(1, ngram), max_features=max_features)
        ifidf_vect_stem = vect_stem.fit_transform(qs)
        X = ifidf_vect_stem.toarray()
        X_train = X[:train.shape[0]]
        X_test = X[train.shape[0]:]
    Y_train = train[['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']]
    assert Y_train.shape[0] == X_train.shape[0]
    del train, test
    return (X_train, X_test, Y_train)

labels = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
params = {'toxic': {'ngrams': 1, 'stem': True, 'max_features': 1000, 'C': 10}, 'threat': {'ngrams': 1, 'stem': False, 'max_features': 1000, 'C': 10}, 'severe_toxic': {'ngrams': 1, 'stem': True, 'max_features': 1000, 'C': 1.2}, 'obscene': {'ngrams': 1, 'stem': True, 'max_features': 1000, 'C': 10}, 'insult': {'ngrams': 1, 'stem': True, 'max_features': 1000, 'C': 1.2}, 'identity_hate': {'ngrams': 1, 'stem': True, 'max_features': 1000, 'C': 10}}

start_time = time.time()
for label in labels:
    print('>>> processing ', label)
    X_train, X_test, Y_train = build_data_set(ngram=params[label]['ngrams'], stem=params[label]['stem'], max_features=params[label]['max_features'], min_df=2, remove_stopwords=True)
    Y_train_lab = Y_train[label]
    seed = 7
    scoring = 'accuracy'
    models = []
    models.append(('LR', LogisticRegression()))
    results = []
    names = []
    for name, model in models:
        kfold = model_selection.KFold(n_splits=10, random_state=seed)
        cv_results = model_selection.cross_val_score(model, X_train, Y_train_lab, cv=kfold, scoring=scoring)
        results.append(cv_results)
        names.append(name)
        msg = '%s: %f (%f)' % (name, cv_results.mean(), cv_results.std())
        print(msg)