image_output_1.png
from sklearn.cluster import KMeans, SpectralClustering, AgglomerativeClustering

from sklearn.feature_selection import VarianceThreshold

from sklearn.manifold import MDS

from sklearn.preprocessing import StandardScaler

import matplotlib.pyplot as plt

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import numpy as np
import pandas as pd
from subprocess import check_output
frame = pd.read_csv('../input/HR_comma_sep.csv').rename(columns={'sales': 'position'})

pd.DataFrame({'dtypes': frame.dtypes, 'isnull': pd.isnull(frame).any(), 'count distinct': [len(frame[name].unique()) for name in frame.columns], 'max': frame.select_dtypes(exclude=['object']).max(), 'min': frame.select_dtypes(exclude=['object']).min(), 'std': frame.select_dtypes(exclude=['object']).std()}, index=frame.columns).sort_values('count distinct', ascending=False)

unique_cat = set(frame['position'].unique()) | {'unknown'}
frame['position_'] = pd.Categorical(frame['position'], categories=unique_cat).codes
frame['salary_'] = pd.Categorical(frame['salary'], categories=['low', 'medium', 'unknown', 'high'], ordered=True).codes

from sklearn.manifold import MDS
from sklearn.preprocessing import StandardScaler
from sklearn.feature_selection import VarianceThreshold
frame_ = pd.get_dummies(frame.drop(['salary_', 'position_'], axis=1)).sample(3000)
sampled_index = frame_.index
selector = VarianceThreshold(threshold=0.95 * (1 - 0.95))
frame_ = selector.fit_transform(frame_)
mds = MDS(n_components=2)
scaler = StandardScaler(with_mean=False)
frame_ = scaler.fit_transform(frame_)
frame_ = mds.fit_transform(frame_)
pc_1 = [frame_[i][0] for i in range(len(frame_))]
pc_2 = [frame_[i][1] for i in range(len(frame_))]

### Clustering with kmeans ###

from sklearn.cluster import KMeans, SpectralClustering, AgglomerativeClustering

def plot_pc (label, frame):
    
    pc_1 = frame[frame['label']==label].loc[:,'pc1']
    pc_2 = frame[frame['label']==label].loc[:,'pc2']
    
    plt.scatter(pc_1, pc_2, label=label)
    plt.legend()

def plot_clusters(function, n_clusters, function_kwargs=None):
    
    if function_kwargs==None:
        function_kwargs = dict()
    

    model = function(n_clusters=n_clusters, **function_kwargs)
    labels = model.fit_predict(frame_)

    results = pd.DataFrame({'label':labels, 'pc1':pc_1, 'pc2':pc_2})

    
    for i in range(n_clusters):
        plot_pc(i, results)
    
    plt.title('{} ({} clusters)'.format(function.__name__, n_clusters))
    
    
fig, ((ax1, ax2), (ax3, ax4)) =  plt.subplots(2,2, figsize=(15,13))
for i, ax in zip(range(7,11), fig.axes):
    plt.subplot(ax)
    plot_clusters(KMeans, n_clusters=i, function_kwargs={'init': 'random'});
    

fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(15, 13))
for i, ax in zip(range(7, 11), fig.axes):
    plt.subplot(ax)
    plot_clusters(AgglomerativeClustering, n_clusters=i, function_kwargs={'linkage': 'ward', 'affinity': 'euclidean'})