text_html_output_1.png
from itertools import product

from sklearn.preprocessing import LabelEncoder

import matplotlib.pyplot as plt

import numpy as np

import pandas as pd

sales = pd.read_csv('../input/competitive-data-science-predict-future-sales/sales_train.csv')
test_data = pd.read_csv('../input/competitive-data-science-predict-future-sales/test.csv')
items = pd.read_csv('../input/competitive-data-science-predict-future-sales/items.csv')
item_category = pd.read_csv('../input/competitive-data-science-predict-future-sales/item_categories.csv')
shops = pd.read_csv('../input/competitive-data-science-predict-future-sales/shops.csv')

def downcast_dtypes(df):
    """
        Changes column types in the dataframe: 
                `float64` type to `float32`
                `int64`   type to `int32`
    """
    float_cols = [c for c in df if df[c].dtype == 'float64']
    int_cols = [c for c in df if df[c].dtype == 'int64']
    df[float_cols] = df[float_cols].astype(np.float32)
    df[int_cols] = df[int_cols].astype(np.int32)
    return df

def lag_feature(all_data, list_lags, index_cols, cols_to_rename):
    shift_range = list_lags
    for month_shift in tqdm_notebook(shift_range):
        train_shift = all_data[index_cols + cols_to_rename].copy()
        train_shift['date_block_num'] = train_shift['date_block_num'] + month_shift
        foo = lambda x: '{}_lag_{}'.format(x, month_shift) if x in cols_to_rename else x
        train_shift = train_shift.rename(columns=foo)
        all_data = pd.merge(all_data, train_shift, on=index_cols, how='left').fillna(0)
    del train_shift
    return all_data

Monthly_sales = sales.groupby(["date_block_num", "shop_id"])['item_cnt_day'].sum().reset_index(name = 'item_cnt_month')

fig, axs = plt.subplots(10, 6)

for i in range(60):
  shop_sale_per_month = Monthly_sales.loc[Monthly_sales['shop_id']==i]
  axs[i//6,i%6].tick_params(axis='both', which='both', bottom=False, top= False, labelbottom=False, right=False, left=False, labelleft=False)
  axs[i//6,i%6].plot(shop_sale_per_month['date_block_num'], shop_sale_per_month['item_cnt_month'])

del Monthly_sales, shop_sale_per_month

sales.loc[sales.shop_id == 0, 'shop_id'] = 57
test_data.loc[test_data.shop_id == 0, 'shop_id'] = 57
sales.loc[sales.shop_id == 1, 'shop_id'] = 58
test_data.loc[test_data.shop_id == 1, 'shop_id'] = 58
sales.loc[sales.shop_id == 10, 'shop_id'] = 11
test_data.loc[test_data.shop_id == 10, 'shop_id'] = 11
sales = sales[sales.item_cnt_day < 1001]

temp_df = pd.merge(test_data[['shop_id', 'item_id']], sales[['shop_id', 'item_id']], on=['shop_id', 'item_id'], how='left', indicator='Exist')
temp_var = (temp_df['Exist'] == 'left_only').sum()

Leakage_Percentage = (test_data.shape[0] - temp_var) / test_data.shape[0] * 100

index_cols = ['shop_id', 'item_id', 'date_block_num']
grid = []
for block_num in sales['date_block_num'].unique():
    cur_shops = sales[sales['date_block_num'] == block_num]['shop_id'].unique()
    cur_items = sales[sales['date_block_num'] == block_num]['item_id'].unique()
    grid.append(np.array(list(product(*[cur_shops, cur_items, [block_num]])), dtype='int32'))
grid = pd.DataFrame(np.vstack(grid), columns=index_cols, dtype=np.int32)
gb = sales.groupby(index_cols, as_index=False).agg({'item_cnt_day': ['sum']})
gb.rename(columns={'sum': 'target'}, inplace=True)
gb.columns = [col[0] if col[-1] == '' else col[-1] for col in gb.columns.values]
all_data = pd.merge(grid, gb, how='left', on=index_cols).fillna(0)
all_data.sort_values(['date_block_num', 'shop_id', 'item_id'], inplace=True)
all_data['target'] = all_data['target'].fillna(0).clip(0, 20)
all_data = all_data[all_data['date_block_num'] >= 12]
all_data

shops['city'] = shops.shop_name.apply(lambda x: str.replace(x, '!', '')).apply(lambda x: x.split(' ')[0])
shops['city_enc'] = LabelEncoder().fit_transform(shops['city'])
shops_data = shops[['shop_id', 'city_enc']]
all_data = pd.merge(all_data, shops_data, how='left', on=['shop_id'])
all_data = pd.merge(all_data, items, how='left', on=['item_id'])
all_data = all_data.drop('item_name', axis=1)
item_category['basket'] = item_category['item_category_name'].apply(lambda x: str(x).split(' ')[0])
item_category['basket_enc'] = LabelEncoder().fit_transform(item_category['basket'])
item_category = item_category[['item_category_id', 'basket_enc']]
all_data = pd.merge(all_data, item_category, how='left', on=['item_category_id'])
all_data

all_data = pd.concat([all_data, test_data], ignore_index=True, sort=False, keys=['date_block_num', 'shop_id', 'item_id', 'city_enc', 'item_category_id', 'basket_enc', 'target'])
all_data = downcast_dtypes(all_data)
all_data