application_vnd.jupyter.stderr_output_2.png text_plain_output_3.png text_plain_output_1.png
import numpy as np

import pandas as pd

def one_hot_encoder(df, nan_as_category=True):
    original_columns = list(df.columns)
    categorical_columns = [col for col in df.columns if df[col].dtype == 'object']
    df = pd.get_dummies(df, columns=categorical_columns, dummy_na=nan_as_category)
    new_columns = [c for c in df.columns if c not in original_columns]
    return (df, new_columns)

def train_test():
    train = pd.read_csv('../input/application_train.csv')
    test = pd.read_csv('../input/application_test.csv')
    (print(train.shape), print(test.shape))
    ind_train = train['SK_ID_CURR']
    ind_test = test['SK_ID_CURR']
    new_df = train.append(test).reset_index()
    avg = [col for col in new_df.columns if 'AVG' in col]
    new_df['AVG_MEAN'] = new_df[avg].sum(axis=1)
    new_df['AVG_SUM'] = new_df[avg].sum(axis=1)
    new_df['AVG_STD'] = new_df[avg].std(axis=1)
    new_df['AVG_MEDIAN'] = new_df[avg].median(axis=1)
    mode = [col for col in new_df.columns if 'MODE' in col]
    new_df['MODE_MEAN'] = new_df[mode].sum(axis=1)
    new_df['MODE_SUM'] = new_df[mode].sum(axis=1)
    new_df['MODE_STD'] = new_df[mode].std(axis=1)
    new_df['MODE_MEDIAN'] = new_df[mode].median(axis=1)
    medi = [col for col in new_df.columns if 'MEDI' in col]
    new_df['MEDI_MEAN'] = new_df[medi].sum(axis=1)
    new_df['MEDI_SUM'] = new_df[medi].sum(axis=1)
    new_df['MEDI_STD'] = new_df[medi].std(axis=1)
    new_df['MEDI_MEDIAN'] = new_df[medi].median(axis=1)
    new_df['DAYS_EMPLOYED'].replace(365243, np.nan, inplace=True)
    new_df['AGE_CLIENT'] = new_df['DAYS_BIRTH'] / -365
    new_df['EMPLOYED_YEAR'] = new_df['DAYS_EMPLOYED'] / -365
    new_df['_REGISTRATION_YEAR'] = new_df['DAYS_REGISTRATION'] / -365
    new_df['ID_PUBLISH_YEAR'] = new_df['DAYS_ID_PUBLISH'] / -365
    new_df['RATIO_CHILD_MEMBERS_FAM'] = new_df['CNT_CHILDREN'] / new_df['CNT_FAM_MEMBERS']
    new_df['RATIO_INCOME_MEMBERS_FAM'] = new_df['AMT_INCOME_TOTAL'] / new_df['CNT_FAM_MEMBERS']
    new_df['RATIO_INCOME_CREDIT'] = new_df['AMT_INCOME_TOTAL'] / new_df['AMT_CREDIT']
    new_df['RATIO_INCOME_ANNUITY'] = new_df['AMT_INCOME_TOTAL'] / new_df['AMT_ANNUITY']
    new_df['RATIO_PRICE_INCOME'] = new_df['AMT_GOODS_PRICE'] / new_df['AMT_INCOME_TOTAL']
    new_df['RATIO_PRICE_CREDIT'] = new_df['AMT_GOODS_PRICE'] / new_df['AMT_CREDIT']
    new_df['EXT_SCORE_SUM'] = new_df['EXT_SOURCE_1'] + new_df['EXT_SOURCE_2'] + new_df['EXT_SOURCE_3']
    new_df['EXT_SCORE_MEAN'] = (new_df['EXT_SOURCE_1'] + new_df['EXT_SOURCE_2'] + new_df['EXT_SOURCE_3']) / 3
    new_df['OBS_90_CNT_SOCIAL_CIRCLE_SUM'] = new_df['OBS_30_CNT_SOCIAL_CIRCLE'] + new_df['OBS_60_CNT_SOCIAL_CIRCLE']
    new_df['OBS_90_CNT_SOCIAL_CIRCLE_MEAN'] = (new_df['OBS_30_CNT_SOCIAL_CIRCLE'] + new_df['OBS_60_CNT_SOCIAL_CIRCLE']) / 2
    new_df['DEF_90_CNT_SOCIAL_CIRCLE_MEAN'] = (new_df['DEF_60_CNT_SOCIAL_CIRCLE'] + new_df['DEF_30_CNT_SOCIAL_CIRCLE']) / 2
    new_df['DEF_90_CNT_SOCIAL_CIRCLE_SUM'] = new_df['DEF_60_CNT_SOCIAL_CIRCLE'] + new_df['DEF_30_CNT_SOCIAL_CIRCLE']
    flag_doc_col = [col for col in new_df.columns if 'FLAG_DOCUMENT' in col]
    new_df['FLAG_DOC_MEAN'] = train[flag_doc_col].mean(axis=1)
    new_df['FLAG_DOC_SUM'] = train[flag_doc_col].sum(axis=1)
    new_df, col = one_hot_encoder(new_df)
    train = new_df.loc[new_df['SK_ID_CURR'].isin(ind_train)]
    test = new_df.loc[new_df['SK_ID_CURR'].isin(ind_test)]
    (print(train.shape), print(test.shape))
    return (train, test)

train, test = train_test()