text_plain_output_1.png image_output_1.png
from scipy.stats import chisquare

import pandas as pd

import pandas as pd
df = pd.read_csv('../input/people.csv')

from scipy.stats import chisquare
chars = [i for i in df.columns.values if 'char_' in i]
flags = []
for feat in df[chars]:
    group = df[chars].groupby(feat)
    for otherfeat in df[chars].drop(feat, axis=1):
        summary = group[otherfeat].count()
        if chisquare(summary)[1] < 0.05:
            flags.append(feat)
            flags.append(otherfeat)
flags = set(flags)

dums = pd.get_dummies(df[chars])
print('Before PCA the full size of the characteristics is {} features'.format(len(dums.columns.values)))