text_plain_output_1.png
from scipy.stats import chisquare

from sklearn.decomposition import PCA

from sklearn.preprocessing import MinMaxScaler

import pandas as pd

import pandas as pd
df = pd.read_csv('../input/people.csv')

from scipy.stats import chisquare
chars = [i for i in df.columns.values if 'char_' in i]
flags = []
for feat in df[chars]:
    group = df[chars].groupby(feat)
    for otherfeat in df[chars].drop(feat, axis=1):
        summary = group[otherfeat].count()
        if chisquare(summary)[1] < 0.05:
            flags.append(feat)
            flags.append(otherfeat)
flags = set(flags)

dums = pd.get_dummies(df[chars])

from sklearn.decomposition import PCA
from sklearn.preprocessing import MinMaxScaler
scaledums = MinMaxScaler().fit_transform(dums)
pca = PCA(n_components=2)
featurecomponents = pca.fit_transform(scaledums)

import numpy as np
components = {}
index = 0
for feature in dums.columns.values:
    components[feature] = [pca.components_[0][index]]
    index += 1
sortedcomps = pca.components_[0]
sortedcomps.sort()
maxcap = sortedcomps[-3]
mincap = sortedcomps[2]
components = {i: x for i, x in components.items() if x >= maxcap or x <= mincap}
components = pd.DataFrame(components)

featurecomponents = pd.DataFrame(featurecomponents, columns=['Principle Component 1', 'Principle Component 2'])
df['Principle Component 1'] = featurecomponents['Principle Component 1']
featurecomponents.plot(kind='scatter', x='Principle Component 1', y='Principle Component 2', figsize=(12, 12), s=1)