text_html_output_1.png
import pandas as pd

import plotly.graph_objs as go

import plotly.offline as pyo

data_df = pd.read_csv('../input/mushrooms.csv')

data_df['y'] = data_df['class'].map({'p': 1, 'e': 0})
feature_columns = [c for c in data_df.columns if not c in ('class', 'y')]
stats_df = []
single_val_c = {}
for i, c in enumerate(feature_columns):
    if data_df[c].nunique() == 1:
        single_val_c[c] = data_df[c].unique()[0]
        continue
    gb = data_df.groupby(c)
    m = gb['y'].mean()
    s = gb.size()
    df = pd.DataFrame(index=range(len(m)))
    df['col'] = c
    df['val'] = m.index.values
    df['positive_percentage'] = m.values
    df['quantity_percentage'] = s.values / s.sum()
    stats_df.append(df)
    trace_prate = go.Bar(x=df['val'], y=df['positive_percentage'] * 100, name='positive percentage')
    trace_cnt = go.Bar(x=df['val'], y=df['quantity_percentage'] * 100, name='quantity percentage')
    layout = go.Layout(xaxis=dict(title=c), yaxis=dict(title='positive and quantity percentage'))
    fig = go.Figure(data=[trace_prate, trace_cnt], layout=layout)
stats_df = pd.concat(stats_df, axis=0)

for c in single_val_c.keys():
    feature_columns.remove(c)
data_df = data_df[feature_columns + ['y']].copy()

data_df = pd.concat((data_df, data_df), axis=0, ignore_index=True)

data_all = pd.get_dummies(data=data_df, columns=feature_columns, prefix=feature_columns)

def clean_data_df(df, threshold=0.02):
    feature_convert = dict()
    for col, sub in stats_df.groupby('col'):
        ns = sub[sub.quantity_percentage < threshold]
        n_ns = sub[sub.quantity_percentage >= threshold]
        for idx in ns.index:
            if ns.loc[idx, 'positive_percentage'] > 0.5:
                p_n_ns = n_ns[n_ns.positive_percentage > 0.5]
                if not p_n_ns.empty:
                    feature_convert.setdefault(col, []).append((ns.loc[idx, 'val'], p_n_ns['val'].values[0]))
                    df.loc[df[col] == ns.loc[idx, 'val'], col] = p_n_ns['val'].values[0]
            else:
                n_n_ns = n_ns[n_ns.positive_percentage <= 0.5]
                if not n_n_ns.empty:
                    feature_convert.setdefault(col, []).append((ns.loc[idx, 'val'], n_n_ns['val'].values[0]))
                    df.loc[df[col] == ns.loc[idx, 'val'], col] = n_n_ns['val'].values[0]
    return (pd.get_dummies(data=df, columns=feature_columns, prefix=feature_columns), feature_convert)
cleaned_df, feature_convert = clean_data_df(data_df.copy())
cleaned_df.info()