<repo_name>nbeaudoin/Enron-Scandal-Machine-Learning<file_sep>/poi_id.py
#!/usr/bin/python

### Goal: Determine predictors of being a person of interest (POI)

### Load environment
import os
#os.chdir('/Users/nicholasbeaudoin/Desktop/OneDrive/DAND/7 Intro to Machine Learning/ud120-projects-master/final_project')

### Import packages
import sys
import pickle
sys.path.append("../tools/")
import pandas as pd
import numpy as np
from sklearn import cross_validation
import matplotlib.pyplot as plt
from sklearn.preprocessing import MinMaxScaler
from feature_format import featureFormat, targetFeatureSplit
from tester import dump_classifier_and_data
from sklearn.cross_validation import train_test_split
from sklearn.naive_bayes import GaussianNB
from sklearn.metrics import accuracy_score
from sklearn.cluster import KMeans
from sklearn.preprocessing import MinMaxScaler
from sklearn import tree
from sklearn.linear_model import LinearRegression
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
from sklearn.decomposition import PCA
from sklearn.pipeline import Pipeline
from sklearn.tree import DecisionTreeClassifier
from tester import test_classifier, dump_classifier_and_data
from sklearn.metrics import precision_score, recall_score
from sklearn.feature_selection import SelectKBest, f_classif
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import GridSearchCV, StratifiedShuffleSplit


######### Task 1: Select what features you'll use. #########

### features_list is a list of strings, each of which is a feature name.
### The first feature must be "poi".
target_label = 'poi'

email_features_list = [
    'from_messages',
    'from_poi_to_this_person',
    'from_this_person_to_poi',
    'shared_receipt_with_poi',
    'to_messages',
    ]
financial_features_list = [
    'bonus',
    'deferral_payments',
    'deferred_income',
    'director_fees',
    'exercised_stock_options',
    'expenses',
    'loan_advances',
    'long_term_incentive',
    'other',
    'restricted_stock',
    'restricted_stock_deferred',
    'salary',
    'total_payments',
    'total_stock_value',
]

features_list = [target_label] + financial_features_list + email_features_list

### Load the dictionary containing the dataset
with open("final_project_dataset.pkl", "r") as data_file:
    data_dict = pickle.load(data_file)

### Convert to pandas dataframe
df = pd.DataFrame(data_dict.values())

### How many features do we have?
print len(features_list)

### Exploratory Analysis
print(df.shape)

### Vizualize outliers
features_outliers = ["salary", "bonus"]
data = featureFormat(data_dict, features_outliers)

### Vizualize outliers
for point in data:
    salary = point[0]
    bonus = point[1]
    plt.scatter( salary, bonus )

plt.xlabel("salary")
plt.ylabel("bonus")
plt.show()

### Examine outliers of salary more than $1M and bonus more than $5M
for employee in data_dict:
    if (data_dict[employee]["salary"] != 'NaN') and (data_dict[employee]["bonus"] != 'NaN'):
        if float(data_dict[employee]["salary"]) > 10000000 and float(data_dict[employee]["bonus"])>50000000:
            print employee
            
data_dict.pop("TOTAL", 0)

data = featureFormat(data_dict, features_outliers)

for point in data:
    salary = point[0]
    bonus = point[1]
    plt.scatter( salary, bonus )

plt.xlabel("salary")
plt.ylabel("bonus")
plt.show()

### Examine outliers of no salary
### Examine outliers of salary more than $1M and bonus more than $5M
for employee in data_dict:
    print employee
    
### Remove non-employee
data_dict.pop("THE TRAVEL AGENCY IN THE PARK", 0)
### Update dataframe from dictionary
df = pd.DataFrame(data_dict.values())

feature_count = {}
for f in features_list:
    feature_count[f] = df[f].size
print feature_count

# Count number of features that have NaN values
df.replace(to_replace='NaN', value=np.nan, inplace=True)
print df.isnull().sum()

# Drop anything with more than 50% missing values
# n=77 is half the observations so anything more than that we remove
#for f in features_list:
#    if df.isnull().sum()[f] > 77:
#        df.pop(f)
#        print f
        
### Checking to see if we removed these features with the previous code
#for f in df.columns.values:
#    print f  
    
### Fill NA function in pandas
df = df.fillna(df.mean())
print(df.head())

### Create a new feature list with our new feautres of df
features_list_new = list(df)
# print features_list_new
features_list_new.remove('email_address')

### Create "total compensation"
df['total_comp'] = df['bonus'] + df['salary']

### Add new feature to features_list_new
features_list_new.append('total_comp')
features_list_new.remove('poi')
features_list_new[0] = 'poi'
features_list_new.append('salary')
features_list_new.append('bonus')
print features_list_new

### Take a look at the numeric data
print(df.describe())

count = 0
for key in data_dict.keys():
    if (data_dict[key]["poi"]==1): 
        count += 1

print "Number of POIs:", count

### Select features that have less than 50% missing values
print features_list_new

### Extract features and labels from dataset for local testing
#data = featureFormat(data_dict, features_list_new, sort_keys = True)
#labels, features = targetFeatureSplit(data)
data_0 = df[features_list_new]
### Getting all cols except for POI
#features_list_new.remove('poi')
### Just want numpy array (pandas adds meta data on top of metadata and sklearn wants just matrix)
#labels = df[['poi']].values

### DF --> Dictionary
my_dataset = data_0.to_dict(orient="index")


### Extract features and labels from dataset for local testing
data = featureFormat(my_dataset, features_list_new, sort_keys = True)
labels, features = targetFeatureSplit(data)

features_train, features_test, labels_train, labels_test = \
    train_test_split(data, labels, test_size=0.3, random_state=42)

### Select top 10 using K-Best
kbest = SelectKBest(f_classif, k=3)
features_selected = kbest.fit_transform(features_train, labels_train)
print(features_selected.shape)

features_list_new = [features_list_new[i+1] for i in kbest.get_support(indices=True)]
print ('Features selected by SelectKBest:')
print (features_list_new)

features_list_final = ['poi', 'deferral_payments', 'expenses', 'bonus']

data_0 = df[features_list_final]

my_dataset = data_0.to_dict(orient="index")

#data = featureFormat(my_dataset, features_list_final, sort_keys = True)
#labels, features = targetFeatureSplit(data)

features_train, features_test, labels_train, labels_test = \
    train_test_split(data, labels, test_size=0.3, random_state=42)

### Naive Bayes 1
clf = GaussianNB()
clf.fit(features_train, labels_train)
gnb_pred1 = clf.predict(features_test)
gnb_score1 = clf.score(features_test, labels_test)
gnb_precision1 = precision_score(labels_test, gnb_pred1)
gnb_recall1 = recall_score(labels_test, gnb_pred1)
print ('GaussianNB 1 accuracy:', gnb_score1)
print ('GaussianNB 1 precision:', gnb_precision1)
print ('GaussianNB 1 recall:', gnb_recall1)

### Decision Tree 1
clf = DecisionTreeClassifier()
clf.fit(features_train, labels_train)
dt_pred1 = clf.predict(features_test)
dt_score1 = clf.score(features_test, labels_test)
dt_precision1 = precision_score(labels_test, dt_pred1)
dt_recall1 = recall_score(labels_test, dt_pred1)
print ('Decision Tree 1 accuracy:', dt_score1)
print ('Decision Tree 1 precision:', dt_precision1)
print ('Decision Tree 1 recall:', dt_recall1)


### Random Forest 1
from sklearn.ensemble import RandomForestClassifier
clf = RandomForestClassifier()
fit = clf.fit(features_train, labels_train)
rf_pred1 = clf.predict(features_test)
rf_score1 = clf.score(features_test, labels_test)
rf_precision1 = precision_score(labels_test, rf_pred1)
rf_recall1 = recall_score(labels_test, rf_pred1)
print ('Random Forest 1 accuracy:', rf_score1)
print ('Random Forest 1 precision:', rf_precision1)
print ('Random Forest 1 recall:', rf_recall1)

### Decision Tree 2
clf = DecisionTreeClassifier()
fit = clf.fit(features_train, labels_train)
dt_pred2 = clf.predict(features_test, labels_test)
dt_score2 = clf.score(features_test, labels_test)
dt_precision2 = precision_score(labels_test, dt_pred2)
dt_recall2 = recall_score(labels_test, dt_pred2)
print ('Decision Tree 2 accuracy:', dt_score2)
print ('Decision Tree 2 precision:', dt_precision2)
print ('Decision Tree 2 recall:', dt_recall2)

### Random Forest  2      
from sklearn.ensemble import RandomForestClassifier
clf = RandomForestClassifier()
fit = clf.fit(features_train, labels_train)
rf_pred2 = clf.predict(features_test)
rf_score2 = clf.score(features_test, labels_test)
rf_precision2 = precision_score(labels_test, rf_pred2)
rf_recall2 = recall_score(labels_test, rf_pred2)
print ('Random Forest 2 accuracy:', rf_score2)
print ('Random Forest 2 precision:', rf_precision2)
print ('Random Forest 2 recall:', rf_recall2)
    
### GridSearch: Decision Tree 3
param_grid_dt3 = {'max_depth': np.arange(3, 10), 'min_samples_split': np.arange(3, 10),
              'presort': [True, False]}

sss = StratifiedShuffleSplit(n_splits=100, test_size=0.5, random_state=0)
tree_dt3 = GridSearchCV(DecisionTreeClassifier(), param_grid_dt3)
tree_dt3.fit(features_train, labels_train)
tree_dt3.best_params_

clf = DecisionTreeClassifier(max_depth=3, min_samples_split=3, presort=True)
fit = clf.fit(features_train, labels_train)
dt_pred3 = clf.predict(features_test, labels_test)
dt_score3 = clf.score(features_test, labels_test)
dt_precision3 = precision_score(labels_test, dt_pred3)
dt_recall3 = recall_score(labels_test, dt_pred3)
print ('Decision Tree 3 accuracy:', dt_score3)
print ('Decision Tree 3 precision:', dt_precision3)
print ('Decision Tree 3 recall:', dt_recall3)

### GridSearch: Random Forest 3
param_grid_rf3 = {'max_depth': np.arange(3, 20), 'min_samples_split': np.arange(3, 10)}

sss = StratifiedShuffleSplit(n_splits=100, test_size=0.5, random_state=0)
tree_rf3 = GridSearchCV(DecisionTreeClassifier(random_state=0), param_grid_rf3, cv=sss, scoring='f1')
tree_rf3.fit(features_train, labels_train)
tree_rf3.best_params_

clf = RandomForestClassifier(max_depth=3, min_samples_split=3)
fit = clf.fit(features_train, labels_train)
rf_pred3 = clf.predict(features_test)
rf_score3 = clf.score(features_test, labels_test)
rf_precision3 = precision_score(labels_test, rf_pred3)
rf_recall3 = recall_score(labels_test, rf_pred3)
print ('Random Forest 3 accuracy:', rf_score3)
print ('Random Forest 3 precision:', rf_precision3)
print ('Random Forest 3 recall:', rf_recall3)



### Task 6: Dump your classifier, dataset, and features_list so anyone can ###
### check your results. You do not need to change anything below, but make sure
### that the version of poi_id.py that you submit can be run on its own and
### generates the necessary .pkl files for validating your results.
feature_list = features_list_final 
dataset = my_dataset

test_classifier(clf, dataset, feature_list)
dump_classifier_and_data(clf, dataset, feature_list)

<file_sep>/poi_v3.py
# -*- coding: utf-8 -*-


import pickle
import pandas as pd
import numpy as np
from sklearn.cross_validation import train_test_split
from tester import test_classifier, dump_classifier_and_data
from feature_format import featureFormat, targetFeatureSplit
from sklearn.grid_search import GridSearchCV
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
import matplotlib.pyplot as plt
from sklearn.ensemble import RandomForestClassifier
from sklearn.naive_bayes import GaussianNB

### Load the dictionary containing the dataset
with open("final_project_dataset.pkl", "r") as data_file:
    data_dict = pickle.load(data_file)
    
CLF_PICKLE_FILENAME = "my_classifier.pkl"
### Convert to pandas dataframe
df = pd.DataFrame(data_dict.values())

##############################################################################
###############       EXPLORATORY ANALYSIS      ##############################                
##############################################################################

### First look at the data
print df.head()
# Each feature is a column

### Data shape
print df.shape
# 146 observations
# 21 features including POI  

features_outliers = ["salary", "bonus"]
data = featureFormat(data_dict, features_outliers)

### Vizualize outliers
for point in data:
    salary = point[0]
    bonus = point[1]
    plt.scatter( salary, bonus )

plt.xlabel("salary")
plt.ylabel("bonus")
plt.show()

### Examine outliers of salary more than $1M and bonus more than $5M
for employee in data_dict:
    if (data_dict[employee]["salary"] != 'NaN') and (data_dict[employee]["bonus"] != 'NaN'):
        if float(data_dict[employee]["salary"]) > 10000000 and float(data_dict[employee]["bonus"])>50000000:
            print employee

### Looking for employees that have strange names; not empoyee?
for employee in data_dict:
    print employee
    
### Remove non-employee and outlier "total"
data_dict.pop("TOTAL", 0)
data_dict.pop("THE TRAVEL AGENCY IN THE PARK", 0)

### Update dataframe from dictionary
df = pd.DataFrame(data_dict.values())

### Number of POI
count = 0
for key in data_dict.keys():
    if (data_dict[key]["poi"]==1): 
        count += 1
print "Number of POIs:", count

# Replace 'NAN' strings with np.nan
df.replace(to_replace='NaN', value=np.nan, inplace=True)
   
### Fill NA function in pandas
df = df.fillna(df.mean())


### drop unwanted feature 
del df['email_address']



##### feature list based on df, dataset as dictionary
feature_list = list(df)
feature_list[0]='poi'
feature_list[12]='bonus'

#### Scale dataframe (minmax scaling)
df_copy=df.copy()
df_copy = df_copy.drop('poi', 1)
df_copy -= df_copy.min()  
df_copy /= (df_copy.max()-df_copy.min())
df_copy['poi']=df['poi']
df=df_copy


## Convert df to dictionary
dataset=df.to_dict(orient="index")

##### Clean dataset
X=featureFormat(dataset, feature_list, remove_NaN=True, remove_all_zeroes=True, remove_any_zeroes=False, sort_keys = False)

### split dataset into target and features
target,features=targetFeatureSplit(X)

### Feature selection
df_new=pd.DataFrame()
clf = DecisionTreeClassifier(min_samples_split=2,min_samples_leaf=1)
clf.fit(features, target)
tree_scores = zip(feature_list[1:], clf.feature_importances_)
for i in range(19):
    if tree_scores[i][1]>0.01:
        print tree_scores[i][0]
        df_new[tree_scores[i][0]]=df_copy[tree_scores[i][0]]
        
feature_list=list(df_new)
feat=feature_list[0]
feature_list[0]='poi'



df_new['poi']=df['poi']
       
## Convert df to dictionary
dataset=df_new.to_dict(orient="index")

##### Clean dataset
X=featureFormat(dataset, feature_list, remove_NaN=True, remove_all_zeroes=True, remove_any_zeroes=False, sort_keys = False)

### split dataset into target and features
target,features=targetFeatureSplit(X)
 
###### Train Test Split
features_train, features_test, labels_train, labels_test = train_test_split(features, target, test_size=0.00, random_state=42)

################ Select Classifiers #########################

### Naive Bayes 1
clf = GaussianNB()
clf.fit(features_train, labels_train)  
test_classifier(clf, dataset, feature_list)

### Decision Tree 1
clf = DecisionTreeClassifier()
clf.fit(features_train, labels_train)   
test_classifier(clf, dataset, feature_list)

### Random Forest 1
clf = RandomForestClassifier()
clf.fit(features_train, labels_train)  
test_classifier(clf, dataset, feature_list)

### SVC
clf_r=SVC()
clf_r.fit(features_train, labels_train)

#################### Find optimal parameters #######################
parameters = {'gamma': [1,2,3,4,5,6,7,8,9,10,10.5,11,11.5,12,12.5,13,13.5,14,14.5,15,15.5,16,16.5,17,17.5,18,18.5,19,19.5,20]} 
scores = ['f1']

clf_svc=SVC(class_weight='balanced',kernel='rbf', C=10000)

A={}
for score in scores:
    print("# Tuning hyper-parameters for %s" % score)
    print()

    clf = GridSearchCV(clf_svc,parameters,
                       scoring='%s_macro' % score)
    clf.fit(features_train, labels_train)

    print("Best parameters set found on development set:")
    print()
    print(clf.best_params_)
    print()
    print("Grid scores on development set:")
    print()
    A[score]=clf.best_params_
    

######################################
print 'FINAL RESULTS: Best Classifier'
#####################################  
  
clf_r=SVC(class_weight='balanced', kernel='rbf', C=10000, gamma=A['f1']['gamma'])
clf_r.fit(features_train, labels_train)
 
print '########## Classifier tuned for f1 ################'
test_classifier(clf_r, dataset, feature_list)
dump_classifier_and_data(clf_r, dataset, feature_list)

<file_sep>/README.md
# Udacity Completed Machine Learning Capstone

Dataset Enron, courtesy of Udacity

Completed 10/1/2018
