text_plain_output_1.png
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier

from sklearn.linear_model import RidgeClassifier, LogisticRegressionCV

from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score

from sklearn.model_selection import GridSearchCV, train_test_split

from sklearn.naive_bayes import GaussianNB, MultinomialNB

from sklearn.pipeline import Pipeline

from sklearn.preprocessing import LabelEncoder, PolynomialFeatures

from time import time

import numpy as np

import pandas as pd

import plotly.graph_objs as go

import plotly.offline as pyo

data_df = pd.read_csv('../input/mushrooms.csv')

data_df['y'] = data_df['class'].map({'p': 1, 'e': 0})
feature_columns = [c for c in data_df.columns if not c in ('class', 'y')]
stats_df = []
single_val_c = {}
for i, c in enumerate(feature_columns):
    if data_df[c].nunique() == 1:
        single_val_c[c] = data_df[c].unique()[0]
        continue
    gb = data_df.groupby(c)
    m = gb['y'].mean()
    s = gb.size()
    df = pd.DataFrame(index=range(len(m)))
    df['col'] = c
    df['val'] = m.index.values
    df['positive_percentage'] = m.values
    df['quantity_percentage'] = s.values / s.sum()
    stats_df.append(df)
    trace_prate = go.Bar(x=df['val'], y=df['positive_percentage'] * 100, name='positive percentage')
    trace_cnt = go.Bar(x=df['val'], y=df['quantity_percentage'] * 100, name='quantity percentage')
    layout = go.Layout(xaxis=dict(title=c), yaxis=dict(title='positive and quantity percentage'))
    fig = go.Figure(data=[trace_prate, trace_cnt], layout=layout)
stats_df = pd.concat(stats_df, axis=0)

for c in single_val_c.keys():
    feature_columns.remove(c)
data_df = data_df[feature_columns + ['y']].copy()

data_df = pd.concat((data_df, data_df), axis=0, ignore_index=True)

data_all = pd.get_dummies(data=data_df, columns=feature_columns, prefix=feature_columns)

def grid_search(base_model, param_grid, X_train, y_train):
    gs_c = GridSearchCV(base_model, param_grid=param_grid, n_jobs=-1, cv=3)
    gs_c.fit(X_train, y_train)
    return gs_c
def ridge_model(X_train, y_train):
    r_c = Pipeline([('poly', PolynomialFeatures(interaction_only=True)), ('clf', RidgeClassifier(random_state=1))])
    params_pool = dict(poly__degree=[2], clf__alpha=[0.01, 0.03, 0.1, 0.3, 1])
    return grid_search(r_c, params_pool, X_train, y_train)
def randomForest_model(X_train, y_train):
    rf_c = RandomForestClassifier(random_state=1)
    params_pool = dict(max_depth=[5, 7, 9], max_features=[0.3, 0.5], n_estimators=[12, 20, 36, 50])
    return grid_search(rf_c, params_pool, X_train, y_train)
def gaussianNB_model(X_train, y_train):
    gnb = Pipeline([('poly', PolynomialFeatures(interaction_only=True)), ('clf', GaussianNB())])
    gnb.fit(X_train, y_train)
    return gnb
def multinomialNB_model(X_train, y_train):
    mnb = Pipeline([('poly', PolynomialFeatures(interaction_only=True)), ('clf', MultinomialNB(alpha=1e-05))])
    mnb.fit(X_train, y_train)
    return mnb
def do_model_train(model_name, X_train, y_train, X_test, y_test):
    bg = time()
    if 'Ridge' == model_name:
        model = ridge_model(X_train, y_train)
    elif 'RandomForest' == model_name:
        model = randomForest_model(X_train, y_train)
    elif 'GaussianNB' == model_name:
        model = gaussianNB_model(X_train, y_train)
    elif 'multinomialNB' == model_name:
        model = multinomialNB_model(X_train, y_train)
    y_hat = model.predict(X_train)
    y_hat = model.predict(X_test)

X_all, y_all = (data_all[[c for c in data_all.columns if c != 'y']], data_all['y'])
X_train, X_test, y_train, y_test = train_test_split(X_all, y_all, test_size=0.33, random_state=1)
do_model_train('RandomForest', X_train, y_train, X_test, y_test)

def clean_data_df(df, threshold=0.02):
    feature_convert = dict()
    for col, sub in stats_df.groupby('col'):
        ns = sub[sub.quantity_percentage < threshold]
        n_ns = sub[sub.quantity_percentage >= threshold]
        for idx in ns.index:
            if ns.loc[idx, 'positive_percentage'] > 0.5:
                p_n_ns = n_ns[n_ns.positive_percentage > 0.5]
                if not p_n_ns.empty:
                    feature_convert.setdefault(col, []).append((ns.loc[idx, 'val'], p_n_ns['val'].values[0]))
                    df.loc[df[col] == ns.loc[idx, 'val'], col] = p_n_ns['val'].values[0]
            else:
                n_n_ns = n_ns[n_ns.positive_percentage <= 0.5]
                if not n_n_ns.empty:
                    feature_convert.setdefault(col, []).append((ns.loc[idx, 'val'], n_n_ns['val'].values[0]))
                    df.loc[df[col] == ns.loc[idx, 'val'], col] = n_n_ns['val'].values[0]
    return (pd.get_dummies(data=df, columns=feature_columns, prefix=feature_columns), feature_convert)
cleaned_df, feature_convert = clean_data_df(data_df.copy())

def get_features_importance(x, y):
    rf = RandomForestClassifier(n_estimators=500, class_weight={0: 1, 1: 1 / np.sqrt(np.mean(y))}, max_features=0.75, n_jobs=-1, random_state=1)
    rf.fit(x, y)
    feature_importance = pd.DataFrame(data={'columns': x.columns, 'importance': rf.feature_importances_})
    feature_importance.sort_values(by='importance', axis=0, ascending=False, inplace=True)
    feature_importance.loc[:, 'cum_importance'] = feature_importance.importance.cumsum()
    return feature_importance
def get_features_corr(df, ycol):
    corr_y = df.corr()[ycol].map(np.abs)
    corr_y = corr_y[[c for c in corr_y.index if c != ycol]]
    corr_y = corr_y / corr_y.sum()
    feature_importance = pd.DataFrame(data={'columns': corr_y.index.values, 'importance': corr_y.values})
    feature_importance.sort_values(by='importance', axis=0, ascending=False, inplace=True)
    feature_importance.loc[:, 'cum_importance'] = feature_importance.importance.cumsum()
    return feature_importance
data_all = pd.get_dummies(data=data_df, columns=feature_columns, prefix=feature_columns)
X_all, y_all = (data_all[[c for c in data_all.columns if c != 'y']], data_all['y'])
fi = get_features_importance(X_all, y_all)
bg = time()
accuracyScores, precisionScores, recallScores, f1Scores = ([], [], [], [])
for i in range(len(fi)):
    cols = fi.iloc[:i + 1]['columns'].values
    model = Pipeline([('poly', PolynomialFeatures(interaction_only=True, degree=2)), ('clf', GaussianNB())])
    model.fit(X_all[cols], y_all)
    y_p = model.predict(X_all[cols])
    accuracyScores.append(accuracy_score(y_true=y_all, y_pred=y_p))
    precisionScores.append(precision_score(y_true=y_all, y_pred=y_p))
    recallScores.append(recall_score(y_true=y_all, y_pred=y_p))
    f1Scores.append(f1_score(y_true=y_all, y_pred=y_p))
    if accuracyScores[-1] == 1:
        break
print()
print('It took %.3f seconds.' % (time() - bg))
traces = [go.Scatter(x=np.arange(len(fi)) + 1, y=fi['cum_importance'][:i + 1], mode='markers+lines', name='importance', text=fi['columns']), go.Scatter(x=np.arange(len(fi)) + 1, y=accuracyScores, mode='markers+lines', name='accuracy Score'), go.Scatter(x=np.arange(len(fi)) + 1, y=precisionScores, mode='markers+lines', name='precision Score'), go.Scatter(x=np.arange(len(fi)) + 1, y=recallScores, mode='markers+lines', name='recall Score'), go.Scatter(x=np.arange(len(fi)) + 1, y=f1Scores, mode='markers+lines', name='F1 Score')]
layout = go.Layout(title='Feature importance/accuracy/precision/recall/F1 Score on different number of features', xaxis=dict(title='number of features'), yaxis=dict(title='importance/accuracy/precision/recall/F1 Score'))
pyo.iplot(go.Figure(data=traces, layout=layout))