text_html_output_1.png
from lime import lime_text

from nltk.corpus import stopwords

from sklearn.decomposition import TruncatedSVD

from sklearn.ensemble import RandomForestClassifier

from sklearn.metrics import confusion_matrix,jaccard_score,f1_score,recall_score

from sklearn.metrics import precision_score,roc_auc_score,log_loss

from sklearn.model_selection import train_test_split,GridSearchCV

from sklearn.preprocessing import FunctionTransformer

from tensorflow.keras.layers import Dense, Dropout, GlobalMaxPooling1D, Conv1D, Embedding, LSTM

from tensorflow.keras.models import Sequential, Model

from tensorflow.keras.wrappers.scikit_learn import KerasClassifier

from wordcloud import WordCloud, STOPWORDS

import numpy as np

import pandas as pd

import sklearn.calibration as cal

import sklearn.feature_extraction.text as ft

import sklearn.feature_selection as fs

import sklearn.pipeline as pipe

import sklearn.svm as svm

import warnings

import pandas as pd
import sklearn.feature_extraction.text as ft
import sklearn.feature_selection as fs
from sklearn.model_selection import train_test_split, GridSearchCV
import sklearn.svm as svm
import sklearn.calibration as cal
import matplotlib.pyplot as plt
from sklearn.metrics import plot_confusion_matrix, plot_roc_curve, brier_score_loss
from sklearn.metrics import plot_precision_recall_curve, classification_report
from sklearn.metrics import precision_score, roc_auc_score, log_loss
from sklearn.metrics import confusion_matrix, jaccard_score, f1_score, recall_score
from sklearn.preprocessing import FunctionTransformer
import seaborn as sns
import numpy as np
import time
import datetime
import sklearn.pipeline as pipe
from lime import lime_text
from nltk.corpus import stopwords
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.decomposition import TruncatedSVD
import re
import tensorflow as tf
from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.layers import Dense, Dropout, GlobalMaxPooling1D, Conv1D, Embedding, LSTM
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.wrappers.scikit_learn import KerasClassifier
from wordcloud import WordCloud, STOPWORDS
stop_words = set(stopwords.words('english'))
stopwords = set(STOPWORDS)
import warnings
warnings.filterwarnings('ignore', 'This pattern has match groups')

df = pd.read_csv('../input/d/brandonbenton/botezlive-chat-classification/botezlive_data.csv')

def split_data(df, params):
    df_ones = df.loc[df['is_offensive'] == 1]
    df_tmp = df.loc[df['is_offensive'] == 0]
    ratio = len(df_ones) / len(df_tmp)
    msk = np.random.rand(len(df_tmp)) < params['multiplier'] * ratio
    df_zeros = df_tmp.loc[msk]
    df = pd.concat([df_ones, df_zeros]).reset_index(drop=True)
    df_train, df_test = train_test_split(df, test_size=0.2, random_state=0)
    return (df_train, df_test)

def get_vectorizer(df_train, params):
    vparams = dict({'stop_words': stop_words, 'min_df': params['min_df'], 'max_df': params['max_df'], 'smooth_idf': params['smooth_idf'], 'analyzer': params['analyzer'], 'ngram_range': params['ngram_range'], 'sublinear_tf': params['sublinear_tf'], 'max_features': params['max_features']})
    vectorizer = ft.TfidfVectorizer(**vparams)
    x_train = vectorizer.fit_transform(df_train['text'])
    if params['trim features']:
        y = df_train['is_offensive']
        x_names = vectorizer.get_feature_names()
        p_value_limit = params['pvalue limit']
        dtf_features = pd.DataFrame()
        for cat in np.unique(y):
            chi2, p_value = fs.chi2(x_train, y == cat)
            entry = pd.DataFrame({'feature': x_names, 'score': 1 - p_value, 'y': cat})
            dtf_features = dtf_features.append(entry)
            dtf_features = dtf_features.sort_values(['y', 'score'], ascending=[True, False])
            dtf_features = dtf_features[dtf_features['score'] > p_value_limit]
        x_names = dtf_features['feature'].unique().tolist()
        vparams['vocabulary'] = x_names
        vectorizer = ft.TfidfVectorizer(**vparams)
    return vectorizer

def get_dual_vectorizer(df_train, p1, p2):
    vec1 = get_vectorizer(df_train, p1)
    vec2 = get_vectorizer(df_train, p2)
    features = set(vec1.get_feature_names()) | set(vec2.get_feature_names())
    features = list(features)
    vectorizer = ft.TfidfVectorizer(vocabulary=features, max_features=p1['max_features'])
    return vectorizer

def get_pca(params):
    pca = TruncatedSVD(n_components=params['n_components'])
    return pca

def transformer(x):
    return np.array(x.toarray())
def param_score(df, params):
    df_train, df_test = split_data(df, params)
    vectorizer = get_vectorizer(df_train, params)
    model = get_model(params)
    y_train = df_train['is_offensive']
    y_test = df_test['is_offensive']
    num_splits = 1
    if params['classifier_type'] == 'NN':
        nn = KerasClassifier(build_fn=lambda: model)
        nn._estimator_type = 'classifier'
        vectorizer.fit(df_train['text'])
        num_rows = df_train.shape[0]
        num_splits = 1
        if params['pca']:
            pca = get_pca(params)
            pca.fit(vectorizer.transform(df_train['text']))
        if num_splits == 1 and params['pca']:
            my_pipeline = pipe.Pipeline([('vectorizer', vectorizer), ('pca', pca), ('classifier', nn)])
            my_pipeline.fit(df_train['text'], df_train['is_offensive'], classifier__verbose=1, classifier__epochs=params['epochs'], classifier__batch_size=4)
        elif num_splits == 1 and (not params['pca']):
            my_pipeline = pipe.Pipeline([('vectorizer', vectorizer), ('transformer', FunctionTransformer(transformer)), ('classifier', nn)])
            my_pipeline.fit(df_train['text'], df_train['is_offensive'], classifier__verbose=1, classifier__epochs=params['epochs'], classifier__batch_size=4)
        elif num_splits > 1 and params['pca']:
            pca.fit(vectorizer.transform(df_train['text']))
            df_array = np.array_split(df_train, num_splits)
            for i in range(num_splits):
                df_tmp = df_array[i]
                nn.fit(pca.transform(vectorizer.transform(df_tmp['text'])), df_tmp['is_offensive'], verbose=1, epochs=params['epochs'], batch_size=4)
            my_pipeline = pipe.Pipeline([('vectorizer', vectorizer), ('pca', pca), ('classifier', nn)])
        elif num_splits > 1 and (not params['pca']):
            df_array = np.array_split(df_train, num_splits)
            for i in range(num_splits):
                df_tmp = df_array[i]
                nn.fit(transformer(vectorizer.transform(df_tmp['text'])), df_tmp['is_offensive'], verbose=1, epochs=params['epochs'], batch_size=4)
            my_pipeline = pipe.Pipeline([('vectorizer', vectorizer), ('transformer', FunctionTransformer(transformer)), ('classifier', nn)])
    else:
        clf = cal.CalibratedClassifierCV(model, cv=p1['cv'], method='sigmoid')
        if params['pca']:
            pca = get_pca(params)
            my_pipeline = pipe.Pipeline([('vectorizer', vectorizer), ('pca', pca), ('classifier', clf)])
        else:
            my_pipeline = pipe.Pipeline([('vectorizer', vectorizer), ('classifier', clf)])
        my_pipeline.fit(df_train['text'], df_train['is_offensive'])
    score = my_pipeline.score(df_test['text'], df_test['is_offensive'])
    discrete_preds = my_pipeline.predict(df_test['text'])
    confusion = confusion_matrix(y_test, discrete_preds)
    test_ones = sum(confusion[1][:])
    test_zeros = sum(confusion[0][:])
    scores = dict({'classifier_type': params['classifier_type'], 'splits': num_splits, 'n_samples': df_train.shape[0], 'ngram_range': params['ngram_range'], 'pca': params['pca'], 'n_components': params['n_components'], 'max_features': params['max_features'], 'trim features': params['trim features'], 'pvalue limit': params['pvalue limit'], 'multiplier': params['multiplier'], 'precision': precision_score(y_test, discrete_preds), 'jaccard': jaccard_score(y_test, discrete_preds), 'recall': recall_score(y_test, discrete_preds), 'F1': f1_score(y_test, discrete_preds), 'TP': confusion[1][1] / test_ones, 'FP': confusion[0][1] / test_zeros, 'TN': confusion[0][0] / test_zeros, 'FN': confusion[1][0] / test_ones})
    return (my_pipeline, discrete_preds, df_train, df_test, scores)

def get_model(params):
    if params['classifier_type'] == 'NN':
        if params['pca']:
            feature_num = params['n_components']
        else:
            feature_num = params['max_features']
        model = Sequential()
        model.add(Dense(128, activation='relu', input_shape=(feature_num,)))
        model.add(Dense(64, activation='relu'))
        model.add(Dense(1, activation='sigmoid'))
        model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
        model.summary()
    if params['classifier_type'] == 'SVM':
        cparams = dict({'max_iter': 1000, 'C': params['C'], 'class_weight': params['class_weight']})
        model = svm.LinearSVC(**cparams)
    if params['classifier_type'] == 'RFC':
        model = RandomForestClassifier()
    return model

txt_instance = 'its so hot out'
explainer = lime_text.LimeTextExplainer(class_names=np.unique(df_train['is_offensive']))
explained = explainer.explain_instance(txt_instance, model.predict_proba, num_features=5)
explained.show_in_notebook(text=txt_instance, predict_proba=True)