text_plain_output_1.png
import datetime

import numpy as np

import pandas as pd

def LeaveOneOut(data1, data2, groupcolumns, columnName, useLOO=False, cut=1, addNoise=False):
    features = list([])
    for a in groupcolumns:
        features.append(a)
    if columnName is not None:
        features.append(columnName)
    grpCount = data1.groupby(features)['visitors'].count().reset_index().rename(columns={'visitors': 'Count'})
    grpCount = grpCount[grpCount.Count >= cut]
    grpMean = data1.groupby(features)['visitors'].mean().reset_index().rename(columns={'visitors': 'Mean'})
    grpMedian = data1.groupby(features)['visitors'].median().reset_index().rename(columns={'visitors': 'Median'})
    grpMin = data1.groupby(features)['visitors'].min().reset_index().rename(columns={'visitors': 'Min'})
    grpMax = data1.groupby(features)['visitors'].max().reset_index().rename(columns={'visitors': 'Max'})
    grpStd = data1.groupby(features)['visitors'].std().reset_index().rename(columns={'visitors': 'Std'})
    grpOutcomes = grpCount.merge(grpMean, on=features)
    grpOutcomes = grpOutcomes.merge(grpMedian, on=features)
    grpOutcomes = grpOutcomes.merge(grpMin, on=features)
    grpOutcomes = grpOutcomes.merge(grpMax, on=features)
    grpOutcomes = grpOutcomes.merge(grpStd, on=features)
    x = pd.merge(data2[features], grpOutcomes, suffixes=('x_', ''), how='left', on=features, left_index=True)[['Count', 'Mean', 'Median', 'Max', 'Min', 'Std']]
    x['Outcomes'] = data2['visitors'].values
    if useLOO:
        nonnulls = ~x.Count.isnull()
        x.loc[nonnulls, 'Mean'] = x[nonnulls].Mean * x[nonnulls].Count - x[nonnulls].Outcomes
        x.loc[nonnulls, 'Median'] = x[nonnulls].Median * x[nonnulls].Count - x[nonnulls].Outcomes
        if addNoise is True:
            x.loc[nonnulls & (x.Std > 0), 'Mean'] += np.random.normal(0, x[nonnulls & (x.Std > 0)].Std, x[nonnulls & (x.Std > 0)].shape[0])
            x.loc[nonnulls & (x.Std > 0), 'Median'] += np.random.normal(0, x[nonnulls & (x.Std > 0)].Std, x[nonnulls & (x.Std > 0)].shape[0])
        else:
            x.loc[nonnulls, 'Count'] -= 1
        x.loc[nonnulls, 'Mean'] /= x[nonnulls].Count
        x.loc[nonnulls, 'Median'] /= x[nonnulls].Count
    x.Count = np.log1p(x.Count)
    x = x.replace(np.inf, np.nan)
    x = x.replace(-np.inf, np.nan)
    x = x.fillna(x.mean())
    return x[['Count', 'Mean', 'Median', 'Max', 'Min', 'Std']]
def MungeTrain():
    air_visit_data = pd.read_csv('../input/air_visit_data.csv', parse_dates=['visit_date'])
    air_store_info = pd.read_csv('../input/air_store_info.csv')
    hpg_store_info = pd.read_csv('../input/hpg_store_info.csv')
    air_reserve = pd.read_csv('../input/air_reserve.csv', parse_dates=['visit_datetime'])
    air_reserve['visit_date'] = air_reserve.visit_datetime.apply(lambda df: datetime.datetime(year=df.year, month=df.month, day=df.day))
    hpg_reserve = pd.read_csv('../input/hpg_reserve.csv', parse_dates=['visit_datetime'])
    hpg_reserve['visit_date'] = hpg_reserve.visit_datetime.apply(lambda df: datetime.datetime(year=df.year, month=df.month, day=df.day))
    store_id_relation = pd.read_csv('../input/store_id_relation.csv')
    date_info = pd.read_csv('../input/date_info.csv', parse_dates=['calendar_date']).rename(columns={'calendar_date': 'visit_date'})
    air_reserve_by_date = air_reserve.groupby(['air_store_id', 'visit_date']).reserve_visitors.sum().reset_index(drop=False)
    hpg_reserve_by_date = hpg_reserve.groupby(['hpg_store_id', 'visit_date']).reserve_visitors.sum().reset_index(drop=False)
    hpg_store_info.drop(['latitude', 'longitude'], inplace=True, axis=1)
    train = air_visit_data.merge(air_store_info, on='air_store_id')
    train = train.merge(air_reserve_by_date, on=['air_store_id', 'visit_date'], how='left')
    train = train.merge(store_id_relation, on='air_store_id', how='left')
    train = train.merge(hpg_store_info, on='hpg_store_id', how='left')
    train = train.merge(hpg_reserve_by_date, on=['hpg_store_id', 'visit_date'], how='left')
    train = train.merge(date_info, on='visit_date', how='left')
    train['year'] = train.visit_date.dt.year
    train['month'] = train.visit_date.dt.month
    train.reserve_visitors_x = train.reserve_visitors_x.fillna(0)
    train.reserve_visitors_y = train.reserve_visitors_y.fillna(0)
    train.reserve_visitors_x = np.log1p(train.reserve_visitors_x)
    train.reserve_visitors_y = np.log1p(train.reserve_visitors_y)
    train.visitors = np.log1p(train.visitors)
    train.drop(['latitude', 'longitude'], inplace=True, axis=1)
    train = train.fillna(-1)
    train = train.sort_values(by='visit_date')
    return train
def MungeTest(columns):
    air_visit_data = pd.read_csv('../input/sample_submission.csv')
    air_visit_data['visit_date'] = air_visit_data.id.apply(lambda x: datetime.datetime(year=int(x[-10:-6]), month=int(x[-5:-3]), day=int(x[-2:])))
    air_visit_data['air_store_id'] = air_visit_data.id.apply(lambda x: x[:-11])
    air_store_info = pd.read_csv('../input/air_store_info.csv')
    hpg_store_info = pd.read_csv('../input/hpg_store_info.csv')
    air_reserve = pd.read_csv('../input/air_reserve.csv', parse_dates=['visit_datetime'])
    air_reserve['visit_date'] = air_reserve.visit_datetime.apply(lambda df: datetime.datetime(year=df.year, month=df.month, day=df.day))
    hpg_reserve = pd.read_csv('../input/hpg_reserve.csv', parse_dates=['visit_datetime'])
    hpg_reserve['visit_date'] = hpg_reserve.visit_datetime.apply(lambda df: datetime.datetime(year=df.year, month=df.month, day=df.day))
    store_id_relation = pd.read_csv('../input/store_id_relation.csv')
    date_info = pd.read_csv('../input/date_info.csv', parse_dates=['calendar_date']).rename(columns={'calendar_date': 'visit_date'})
    air_reserve_by_date = air_reserve.groupby(['air_store_id', 'visit_date']).reserve_visitors.sum().reset_index(drop=False)
    hpg_reserve_by_date = hpg_reserve.groupby(['hpg_store_id', 'visit_date']).reserve_visitors.sum().reset_index(drop=False)
    hpg_store_info.drop(['latitude', 'longitude'], inplace=True, axis=1)
    test = air_visit_data.merge(air_store_info, on='air_store_id')
    test = test.merge(air_reserve_by_date, on=['air_store_id', 'visit_date'], how='left')
    test = test.merge(store_id_relation, on='air_store_id', how='left')
    test = test.merge(hpg_store_info, on='hpg_store_id', how='left')
    test = test.merge(hpg_reserve_by_date, on=['hpg_store_id', 'visit_date'], how='left')
    test = test.merge(date_info, on='visit_date', how='left')
    test['year'] = test.visit_date.dt.year
    test['month'] = test.visit_date.dt.month
    test.reserve_visitors_x = test.reserve_visitors_x.fillna(0)
    test.reserve_visitors_y = test.reserve_visitors_y.fillna(0)
    test.reserve_visitors_x = np.log1p(test.reserve_visitors_x)
    test.reserve_visitors_y = np.log1p(test.reserve_visitors_y)
    test = test.fillna(-1)
    test = test.sort_values(by='visit_date')
    test.visitors = np.log1p(test.visitors)
    return test[list(['id']) + list(columns)]

train = MungeTrain()
test = MungeTest(train.columns)

test.head()