text_plain_output_1.png
import glob

import numpy as np

import os

import pandas as pd

import scipy.stats as stats

path = '../input/cyclistic-trip-data'
all_files = glob.glob(os.path.join(path, '*.csv'))
all_files

merged_df = pd.DataFrame()
for f in all_files:
    df_each_file = pd.read_csv(f)
    merged_df = merged_df.append(df_each_file, ignore_index=True)

def explore_stats(df):
    nrows, ncols = df.shape
    print("Total records:", nrows)
    print("Total columns:", ncols)
    
    # create columns list and check dtype
    feature = []
    type_lst = []
    for key, value in df.dtypes.iteritems():
        feature.append(key)
        type_lst.append(value)
        
    # check distinct value
    distinct = []
    for i in df.columns:
        num_distinct = df[i].unique().size
        distinct_pct = num_distinct / nrows * 100
        distinct.append("{} ({:0.2f}%)".format(num_distinct, distinct_pct))
        
    # check null values
    null = []
    for i in df.columns:
        num_null = df[i].isna().sum()
        null_pct = num_null / nrows * 100
        null.append("{} ({:0.2f}%)".format(num_null, null_pct))
        
    # check negative values
    negative = []
    for i in df.columns:
        try:
            num_neg = (df[i].astype('float') < 0).sum()
            neg_pct = num_neg / nrows * 100
            negative.append("{} ({:0.2f}%)".format(num_neg, neg_pct))
        except:
            negative.append(str(0) + " (0%)")
            continue
            
    # check zeros
    zeros = []
    for i in df.columns:
        try:
            num_zero = (df[i] == 0).sum()
            zero_pct = num_zero / nrows * 100
            zeros.append("{} ({:0.2f}%)".format(num_zero, zero_pct))
        except:
            zeros.append(str(0) + " (0%)")
            continue
            
    # check stats measure
    stats = df.describe().transpose()
    
    # put measures into a dataframe
    data = {'feature': feature,
           'data_type': type_lst,
           'n_distinct': distinct,
           'n_missing': null,
           'n_negative': negative,
           'n_zeros': zeros}
    for y in stats.columns:
        data[y] = []
        for x in df.columns:
            try:
                data[y].append(stats.loc[x, y])
            except:
                data[y].append(0.0)
    
    df_stats = pd.DataFrame(data)
    
    return df_stats

conditions = [merged_df['starting_month'] <= 3, (merged_df['starting_month'] >= 4) & (merged_df['starting_month'] <= 6), (merged_df['starting_month'] >= 7) & (merged_df['starting_month'] <= 9), (merged_df['starting_month'] >= 10) & (merged_df['starting_month'] <= 12)]
values = ['spring', 'summer', 'autumn', 'winter']
merged_df['season'] = np.select(conditions, values)

explore_stats(merged_df.loc[:, 'ride_duration':'season'])

merged_df_v2 = merged_df[merged_df['ride_duration'] > 0]
merged_df_v2.shape

duration_data = sorted(merged_df_v2['ride_duration'])
q1 = np.percentile(duration_data, 25)
q3 = np.percentile(duration_data, 75)
iqr = q3 - q1
lower_bound = q1 - 1.5 * iqr
upper_bound = q3 + 1.5 * iqr
print('Lower bound and upper bound are: ' + str(lower_bound) + ', ' + str(upper_bound))
print('Percentile of lower bound: ' + str(stats.percentileofscore(duration_data, lower_bound)))
print('Percentile of upper bound: ' + str(stats.percentileofscore(duration_data, upper_bound)))