<file_sep>case-study-churn-analysis
==============================

Mobile app user churn analysis.

A collaborative case study performed as part of the Galvanize Data Science Immersive.
<file_sep>import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.ensemble import GradientBoostingClassifier


class Cleaner:
    def __init__(self):
        self.len_train = 0
        self.feature_names = []
        self.today = pd.datetime(2014,7,1)

    def stage_cleaning(self,train_data):

        self.rider_null =  train_data['avg_rating_by_driver'].median()
        self.driver_null =  train_data['avg_rating_of_driver'].median()

        self.dist_null =  train_data['avg_dist'].median()
        self.dist_mu =  train_data['avg_dist'].mean()
        self.dist_sigma =  train_data['avg_dist'].std()

        self.surge_null =  train_data['avg_surge'].median()
        self.surgepct_null =  train_data['surge_pct'].median()
        self.first30_null =  train_data['trips_in_first_30_days'].median()
        self.black_null =  0
        self.weekday_null =  train_data['weekday_pct'].median()
        return None

    def clean(self,data):
        #first create DF with android iphone dummy
        clean_data = self.phone_types(data)
        #account age, days since last trip
        #clean_data['inactive_days']=self.inactive_days(data)
        #clean_data['acct_age']=self.acct_age(data)
        clean_data['rider_rtg']=self.rider_rtg(data)
        clean_data['driver_rtg']=self.driver_rtg(data)
        #observed non-null in training, will write cleaning methods if null appears in test
        clean_data['avg_dist']=data['avg_dist'].apply(lambda x: (self.dist_null-self.dist_mu)/self.dist_sigma if pd.isnull(x) else (x-self.dist_mu)/self.dist_sigma)
        clean_data['avg_surge']=data['avg_surge'].apply(lambda x: self.surge_null if pd.isnull(x) else x)
        clean_data['surge_pct']=data['surge_pct'].apply(lambda x: self.surgepct_null/100 if pd.isnull(x) else x/100)
        clean_data['first_30']=data['trips_in_first_30_days'].apply(lambda x: self.first30_null if pd.isnull(x) else x)
        clean_data['black']= data['luxury_car_user'].astype('int64').apply(lambda x: self.black_null if pd.isnull(x) else x)
        clean_data['weekday_pct']= data['weekday_pct'].apply(lambda x: self.weekday_null/100 if pd.isnull(x) else x/100)
        #clean_data['surge_bool']=data['surge_pct'].where(data['surge_pct']==0,1)
        clean_data['weekend_signup']=data['signup_date'].apply(lambda x: x.weekday()>4)
        clean_data['weekday_pct']= data['weekday_pct'].apply(lambda x: self.weekday_null/100 if pd.isnull(x) else x/100)

        #map cities
        clean_data = clean_data.merge(self.cities(data),
                                    left_index=True,right_index = True)
        clean_data = clean_data.drop('iPhone',axis = 1)
        return clean_data



        #

    def phone_types(self,df):
        return pd.get_dummies(df['phone'].map({'Android':'Android','iPhone':'iPhone'}))

    def inactive_days(self,df):
        return (self.today-df['last_trip_date']).apply(lambda x: x.days)

    def acct_age(self,df):
        return (self.today-df['signup_date']).apply(lambda x: x.days)

    def rider_rtg(self,df):
        return df['avg_rating_by_driver'].apply(lambda x: self.rider_null if pd.isnull(x) else x)

    def driver_rtg(self,df):
        return df['avg_rating_of_driver'].apply(lambda x: (self.driver_null) if pd.isnull(x) else x)
    def cities(self,df):
        return pd.get_dummies(df['city'].map({'Astapor':'Astapor','Winterfell':'Winterfell'}))


if __name__ == '__main__':
    df = pd.read_csv('data/churn_train.csv',parse_dates = ['last_trip_date','signup_date'])
    from data_clean import Cleaner

    today = pd.datetime(2014,7,1)
    churn_thresh = pd.Timedelta(30, 'D')
    churn =(today-df['last_trip_date'] >churn_thresh).astype('int64').values

    clnr = Cleaner()
    clnr.stage_cleaning(df)
    clean_data = clnr.clean(df)
    gb = GradientBoostingClassifier(learning_rate=0.005,max_depth=10,
                                n_estimators = 1000,min_samples_leaf=50, max_features = 5)
    df_test = pd.read_csv('data/churn_train.csv',parse_dates = ['last_trip_date','signup_date'])

    churn_test =(today-df_test['last_trip_date'] >churn_thresh).astype('int64').values
    clean_data_test = clnr.clean(df_test)

    gb.fit(clean_data,churn)
    print(gb.score(clean_data_test,churn_test))
<file_sep>import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn import preprocessing
from data_clean import Cleaner  #Shane's class for cleaning

class MeanVisualization():
    ''' 
    For use with the ridesharing dataset/case study: 
    https://github.com/gSchool/dsi-churn-case-study
    
    Using Shane's Cleaner class, this class will clean 
    the data and then visualize the means of each
    feature broken down by churned vs. did not churn.

    Attributes:
    ---------------
    filepath: string with the filepath of the data to import (must be csv)
              a filepath must be passed in each time this class is instantiated

    df: pandas dataframe created automatically once filepath in passed in

    Methods:
    ---------------
    raw_viz: return barplot with means from raw (unscaled) data
    standardized_viz: return barplot with means from standardized data
    
    '''

    def __init__(self, filepath):
        self.filepath = filepath  #should be a string with the filepath of the raw data (must be csv)
        self.df = pd.read_csv(self.filepath, parse_dates=['last_trip_date','signup_date'])  #parse_dates is hard coded for our specific reqs.

    def raw_viz(self):
        '''
        Creates barplot of the means using raw (unscaled) data.
        '''
        #cleaning the data
        df = self.df
        clnr = Cleaner()
        clnr.stage_cleaning(df)
        clean_data = clnr.clean(df)
        #creating binary column for churn
        clean_data.loc[df['last_trip_date'] > '2014-05-31', 'churn'] = 0  # did not churns
        clean_data.loc[df['last_trip_date'] < '2014-06-01', 'churn'] = 1  # did churns
        #creating pivot table with means to visualize, re-formatting for easy seaborn barplotting
        pivot_mean = pd.pivot_table(clean_data, columns='churn', aggfunc=np.mean)
        churn_col = pd.DataFrame(pivot_mean[1])
        churn_col['churn'] = 1
        churn_col.columns = ['mean', 'churn']
        non_churn_col = pd.DataFrame(pivot_mean[0])
        non_churn_col['churn'] = 0
        non_churn_col.columns = ['mean', 'churn']
        new_df = churn_col.append(non_churn_col)
        new_df.reset_index(level=0, inplace=True)
        # if they exist, dropping acct_age and inactive_days columns
        # because they're uninformative and skew the y-axis of the plot
        if 'inactive_days' in new_df:
            viz_df = new_df[(new_df['index'] != 'acct_age') & (new_df['index'] != 'inactive_days')]
        else:
            viz_df = new_df
        #plotting
        fig, ax = plt.subplots(figsize=(15, 5))
        ax = sns.barplot(x='index', y='mean', hue='churn', data=viz_df)
        plt.title('Unscaled means')


    def standardized_viz(self):
        '''
        Creates barplot of the means using scaled data.
        Uses standard scaler from sklearn on all features.
        '''
        #cleaning the data
        df = self.df
        clnr = Cleaner()
        clnr.stage_cleaning(df)
        clean_data_unscaled = clnr.clean(df)
        #scaling the data
        x = clean_data_unscaled.values  #returns a numpy array
        scaler = preprocessing.StandardScaler()
        x_scaled = scaler.fit_transform(x)
        clean_data = pd.DataFrame(x_scaled)
        clean_data.columns = list(clean_data_unscaled)  #replaces column names lost during scaling
        #creating binary churn column
        clean_data.loc[df['last_trip_date'] > '2014-05-31', 'churn'] = 0  # did not churns
        clean_data.loc[df['last_trip_date'] < '2014-06-01', 'churn'] = 1  # did churns
        #creating pivot table with means to visualize, re-formatting for easy seaborn barplotting
        pivot_mean = pd.pivot_table(clean_data, columns='churn', aggfunc=np.mean)
        churn_col = pd.DataFrame(pivot_mean[1])
        churn_col['churn'] = 1
        churn_col.columns = ['mean', 'churn']
        non_churn_col = pd.DataFrame(pivot_mean[0])
        non_churn_col['churn'] = 0
        non_churn_col.columns = ['mean', 'churn']
        new_df = churn_col.append(non_churn_col)
        new_df.reset_index(level=0, inplace=True)
        # if they exist, dropping acct_age and inactive_days columns
        # because they're uninformative and skew the y-axis of the plot
        if 'inactive_days' in new_df:
            viz_df = new_df[(new_df['index'] != 'acct_age') & (new_df['index'] != 'inactive_days')]
        else:
            viz_df = new_df
        #plotting
        fig, ax = plt.subplots(figsize=(15, 5))
        ax = sns.barplot(x='index', y='mean', hue='churn', data=viz_df)
        plt.title('Scaled means')<file_sep>import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC


class Cleaner:

    def __init__(self):
        self.len_train = 0
        self.feature_names = []
        self.today = pd.to_datetime('7/1/2014')

    def stage_and_clean(self, df):
        # send data to staging for initialization and separation
        X, y = self.stage(df)

        # clean x and y
        X = self.cleanX(X)
        y = self.cleany(y)
        return X, y

    def stage(self, df):
        # save null values
        self.rider_null = df['avg_rating_by_driver'].median()
        self.driver_null = df['avg_rating_of_driver'].median()

        df['last_trip_date'] = pd.to_datetime(df['last_trip_date'])
        df['signup_date'] = pd.to_datetime(df['signup_date'])

        # separate x and y
        y = df.drop('last_trip_date', axis=1)
        X = df
        return X, y

    def cleany(self, y):
        inactive_days = (pd.Timedelta self.today - y).apply(lambda x: x.days)
        churn_mask = inactive_days > 30
        return churn_mask.astype('int64')

    def cleanX(self, X):
        # first create clean df with android iphone dummy
        clean_X = self.phone_types(X)

        # convert datetimes to datetimes
        clean_X['acct_age'] = (self.today - X['signup_date'])\
                                .apply(lambda x: x.days)

        # map dummies
        clean_X = clean_X.merge(self.cities(X),
                                      left_index=True, right_index=True)
        clean_X['black'] = X['luxury_car_user'].astype('int64')

        # ensure full columns in rating columns, using staged values as needed
        clean_X['rider_rtg'] = self.rider_rtg(X)
        clean_X['driver_rtg'] = self.driver_rtg(X)

        # observed to be non-null in training data
        # will write cleaning methods if null appears in test
        clean_X['avg_dist'] = X['avg_dist']
        clean_X['avg_surge'] = X['avg_surge']
        clean_X['surge_pct'] = X['surge_pct']
        clean_X['first_30'] = X['trips_in_first_30_days']
        clean_X['weekday_pct'] = X['weekday_pct']

        # save cleaned changes
        return clean_X

    def phone_types(self,df):
        return pd.get_dummies(df['phone'].map({'Android':'Android','iPhone':'iPhone'}))

    def rider_rtg(self,df):
        return df['avg_rating_by_driver'].apply(lambda x: self.rider_null if pd.isnull(x) else x)

    def driver_rtg(self,df):
        return df['avg_rating_of_driver'].apply(lambda x: self.driver_null if pd.isnull(x) else x)

    def cities(self,df):
        return pd.get_dummies(df['city'].map({'Astapor':'Astapor','Winterfell':'Winterfell'}))
<file_sep>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

from sklearn.preprocessing import StandardScaler

from sklearn.linear_model import LinearRegression
from sklearn.linear_model import LogisticRegression, LogisticRegressionCV
# from sklearn.linear_model import Ridge
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score

from data_clean import Cleaner

sns.set(style="ticks")
df = pd.read_csv('data/churn_train.csv')
dcc = Cleaner()

df['last_trip_date'] = pd.to_datetime(df['last_trip_date'])
df['signup_date'] = pd.to_datetime(df['signup_date'])
dcc.stage_cleaning(df)
df = dcc.clean(df)
# sns.pairplot(df)
# plt.savefig('scatter_matrix.png')

print(df.head())


# plt.hist(df['driver_rtg'])
# plt.show()

# high_driver_rating = df[df['driver_rtg'] == 5.0]
# low_driver_rating = df[df['driver_rtg'] < 5.0]

y = df.pop('surge_bool')
X = df


# kf = KFold(n_splits=5, shuffle=True)
# preds = [] 
# for i, (train_index, test_index) in enumerate(kf.split(y)):
    # X_train, X_test = X.iloc[train_index], X.iloc[test_index]
    # y_train, y_test = y.iloc[train_index], y.iloc[test_index]
    # log_reg = LogisticRegression(solver='lbfgs')
    # log_reg.fit(X_train, y_train)
    # preds.append(log_reg.predict(X_test))
    

# print(preds)
# preds = np.array([np.mean(preds[:, i]) for i in range(len(preds[0]))])


clf = LogisticRegressionCV(cv=5, random_state=0, solver='liblinear', n_jobs=-1).fit(X, y)
print("Logistic Regression score: ", clf.score(X, y))
print(clf.predict_proba(X))
