text_plain_output_1.png
from functools import partial

import pandas as pd

train = pd.read_csv('../input/titanic/train.csv')
test = pd.read_csv('../input/titanic/test.csv')

data_x = train.loc[:, [col for col in train.columns if col not in ['Survived']]]
data_y = train.loc[:, ['Survived']]
test_x = test.loc[:, [col for col in train.columns if col not in ['Survived']]]

all_cols = list(data_x.columns)

# Exploratory Data Analysis

# Col-wise
univ_numeric = train_x.describe().T
numeric_cols = univ_numeric.index.to_list()
cat_cols = [col for col in all_cols if col not in numeric_cols]
univ_numeric['missing_perc'] = 1 - univ_numeric['count'] / train_x.shape[0]
univ_numeric['cov'] = univ_numeric['std'] / univ_numeric['mean']

univ_numeric['skew'] = train_x.loc[:, numeric_cols].apply(func=lambda x: x.skew(), axis=0)
univ_numeric['kurtosis'] = train_x.loc[:, numeric_cols].apply(func=lambda x: x.kurtosis(), axis=0)

def return_outlier(ser, threshold=3, return_mask=False):
    not_missing = ~pd.isna(ser)
    z = (ser - ser.mean()) / ser.std()
    out_mask = z.apply(lambda x: not -threshold < x < +threshold)
    if not return_mask:
        out_mask = out_mask * not_missing
        out_mask.sum() / not_missing.sum()
        return out_mask.sum() / not_missing.sum()
    else:
        return out_mask
univ_numeric['outlier_perc'] = train_x.loc[:, numeric_cols].apply(func=return_outlier, axis=0)
out_mask = train_x.loc[:, numeric_cols].apply(func=partial(return_outlier, return_mask=True), axis=0)