text_plain_output_1.png image_output_1.png
import glob

import matplotlib.pyplot as plt

import numpy as np

import os

import pandas as pd

import scipy.stats as stats

path = '../input/cyclistic-trip-data'
all_files = glob.glob(os.path.join(path, '*.csv'))
all_files

merged_df = pd.DataFrame()
for f in all_files:
    df_each_file = pd.read_csv(f)
    merged_df = merged_df.append(df_each_file, ignore_index=True)

def explore_stats(df):
    nrows, ncols = df.shape
    print("Total records:", nrows)
    print("Total columns:", ncols)
    
    # create columns list and check dtype
    feature = []
    type_lst = []
    for key, value in df.dtypes.iteritems():
        feature.append(key)
        type_lst.append(value)
        
    # check distinct value
    distinct = []
    for i in df.columns:
        num_distinct = df[i].unique().size
        distinct_pct = num_distinct / nrows * 100
        distinct.append("{} ({:0.2f}%)".format(num_distinct, distinct_pct))
        
    # check null values
    null = []
    for i in df.columns:
        num_null = df[i].isna().sum()
        null_pct = num_null / nrows * 100
        null.append("{} ({:0.2f}%)".format(num_null, null_pct))
        
    # check negative values
    negative = []
    for i in df.columns:
        try:
            num_neg = (df[i].astype('float') < 0).sum()
            neg_pct = num_neg / nrows * 100
            negative.append("{} ({:0.2f}%)".format(num_neg, neg_pct))
        except:
            negative.append(str(0) + " (0%)")
            continue
            
    # check zeros
    zeros = []
    for i in df.columns:
        try:
            num_zero = (df[i] == 0).sum()
            zero_pct = num_zero / nrows * 100
            zeros.append("{} ({:0.2f}%)".format(num_zero, zero_pct))
        except:
            zeros.append(str(0) + " (0%)")
            continue
            
    # check stats measure
    stats = df.describe().transpose()
    
    # put measures into a dataframe
    data = {'feature': feature,
           'data_type': type_lst,
           'n_distinct': distinct,
           'n_missing': null,
           'n_negative': negative,
           'n_zeros': zeros}
    for y in stats.columns:
        data[y] = []
        for x in df.columns:
            try:
                data[y].append(stats.loc[x, y])
            except:
                data[y].append(0.0)
    
    df_stats = pd.DataFrame(data)
    
    return df_stats

merged_df['started_at'] = pd.to_datetime(merged_df['started_at'])
merged_df['ended_at'] = pd.to_datetime(merged_df['ended_at'])

merged_df['ride_duration'] = (merged_df['ended_at'] - merged_df['started_at']) / pd.Timedelta(minutes=1)
merged_df['ride_duration']

conditions = [merged_df['starting_month'] <= 3, (merged_df['starting_month'] >= 4) & (merged_df['starting_month'] <= 6), (merged_df['starting_month'] >= 7) & (merged_df['starting_month'] <= 9), (merged_df['starting_month'] >= 10) & (merged_df['starting_month'] <= 12)]
values = ['spring', 'summer', 'autumn', 'winter']
merged_df['season'] = np.select(conditions, values)

explore_stats(merged_df.loc[:, 'ride_duration':'season'])

merged_df_v2 = merged_df[merged_df['ride_duration'] > 0]
merged_df_v2.shape

duration_data = sorted(merged_df_v2['ride_duration'])
q1 = np.percentile(duration_data, 25)
q3 = np.percentile(duration_data, 75)
iqr = q3 - q1
lower_bound = q1 - 1.5 * iqr
upper_bound = q3 + 1.5 * iqr

merged_df_v3 = merged_df_v2[merged_df_v2['ride_duration'] < upper_bound]

merged_df_v4 = merged_df_v3.dropna(axis=0, how='any', subset=['start_station_name', 'end_station_name'])
explore_stats(merged_df_v4).sort_values(by=['data_type'])

used_df = merged_df_v4.drop(['start_station_id', 'end_station_id'], axis=1)

a = used_df[used_df['member_casual'] == 'member']['ride_duration']
b = used_df[used_df['member_casual'] == 'casual']['ride_duration']

for i in range(7):
    result = stats.ttest_ind(a + i, b, equal_var=False, alternative='two-sided')

num_ride_by_hour = pd.pivot_table(used_df, values='ride_id', index=['starting_hour'], columns=['member_casual'], aggfunc='count')
num_ride_by_hour['pct_col_casual'] = num_ride_by_hour['casual'] / num_ride_by_hour['casual'].sum()
num_ride_by_hour['pct_col_member'] = num_ride_by_hour['member'] / num_ride_by_hour['member'].sum()

fig, ax = plt.subplots()

x = np.arange(len(num_ride_by_hour.index))
width = 0.4

casual = ax.bar(x-width/2, num_ride_by_hour['pct_col_casual'], width=width, label='Casual')
member = ax.bar(x+width/2, num_ride_by_hour['pct_col_member'], width=width, label='Member')

plt.xticks(x, num_ride_by_hour.index)
ax.legend()
plt.show()

stat, p, dof, expected = stats.chi2_contingency(num_ride_by_hour[['casual', 'member']])
print('T-statistics: ' + str(stat))
print('P-value: ' + str(p))