text_html_output_1.png
from sklearn.metrics import mean_squared_error

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import seaborn as sns

import time

import numpy as np
import pandas as pd
import os
import seaborn as sns
import plotly.express as px
import time
import matplotlib.pyplot as plt
from sklearn.metrics import mean_squared_error
mse = mean_squared_error
def downcast(df: pd.DataFrame) -> pd.DataFrame:
    float_cols = [c for c in df if df[c].dtype in ['float64']]
    int_cols = [c for c in df if df[c].dtype in ['int64']]
    df[float_cols] = df[float_cols].astype('float32')
    df[int_cols] = df[int_cols].astype('int16')
    return df
def lag_feature(df: pd.DataFrame, lag: int, col: str, merge_cols, fill_value=-10, suffix=''):
    temp = df[merge_cols + [col]]
    temp = temp.groupby(merge_cols).agg({f'{col}': 'mean'}).reset_index()
    new_col_name = f'{col}{suffix}_lag{lag}'
    temp.columns = merge_cols + [new_col_name]
    temp['date_block_num'] += lag
    if new_col_name not in df.columns:
        df = pd.merge(df, temp, on=merge_cols, how='left')
    temp = None
    df[new_col_name] = df[new_col_name].fillna(fill_value).astype('float32')
    return (df, new_col_name)

items = pd.read_csv('../input/data-preprocessing/items.csv')
shops = pd.read_csv('../input/data-preprocessing/shops.csv')
cats = pd.read_csv('../input/data-preprocessing/item_categories.csv')
train = pd.read_csv('../input/data-preprocessing/sales_train.csv')
test = pd.read_csv('../input/data-preprocessing/test.csv').set_index('ID')
dataframes = [train, shops, items, cats]
for d in dataframes:
    d = downcast(d)

tme = pd.read_csv('../input/data-preprocessing/train_monthly_extended.csv')
tme = downcast(tme)
cast_cols = ['item_cnt_month', 'days_with_sales', 'date_block_num']
tme[cast_cols] = tme[cast_cols].astype(int)
tme['item_cnt_month'] = tme['item_cnt_month'].clip(0, 50)
tme.sample(4)

cols = []
for lag in [1, 2, 3, 4, 6, 12]:
    t = time.process_time()
    tme, new_col = lag_feature(tme, lag, 'item_cnt_month', ['date_block_num', 'shop_id', 'item_id'], fill_value=np.nan)
    elapsed_time = time.process_time() - t
    cols.append(new_col)
tme, new_col = lag_feature(tme, -1, 'item_cnt_month', ['date_block_num', 'shop_id', 'item_id'], fill_value=0)
tme = tme.rename(columns={new_col: 'target'})

for lag in [1, 2, 3, 4, 12]:
    t = time.process_time()
    tme, new_col = lag_feature(tme, lag, 'item_cnt_month', ['date_block_num', 'shop_id'], fill_value=np.nan, suffix='s')
    elapsed_time = time.process_time() - t
    cols.append(new_col)

for lag in [1, 2, 3, 4, 12]:
    t = time.process_time()
    tme, new_col = lag_feature(tme, lag, 'item_cnt_month', ['date_block_num', 'item_id'], fill_value=np.nan, suffix='i')
    elapsed_time = time.process_time() - t
    cols.append(new_col)

tme['avg_item_3mo'] = ((tme['item_cnt_month_lag1'] + tme['item_cnt_month_lag2'] + tme['item_cnt_month_lag3']) / 3).astype(np.float16)
tme['diff_1yr'] = (tme['item_cnt_month_lag1'] - tme['item_cnt_month_lag12']).astype(np.float16)
tme['roc_1_2'] = tme['item_cnt_month_lag1'] / tme['item_cnt_month_lag2']
tme['roc_2_3'] = tme['item_cnt_month_lag1'] / tme['item_cnt_month_lag3']
tme['roc_1_4'] = tme['item_cnt_month_lag1'] / tme['item_cnt_month_lag4']
tme['roc_1_12'] = tme['item_cnt_month_lag1'] / tme['item_cnt_month_lag12']
tme['diff_12_34'] = tme['item_cnt_month_lag1'] + tme['item_cnt_month_lag2'] - tme['item_cnt_month_lag3'] - tme['item_cnt_month_lag4']
tme['month_num'] = (1 + tme['date_block_num'] % 12).astype(np.uint8)
tme['daydiff'] = (tme['days_no_sales_beginning'] - tme['days_with_sales']).astype(np.uint8)

print('How long does a given item stay in a shop?')
tme['item_age'] = (tme['date_block_num'] - tme.groupby('item_id')['date_block_num'].transform('min')).astype('int8')
tme['item_age'] = tme['item_age'].clip(0, 25)
tme['item_age_in_shop'] = (tme['date_block_num'] - tme.groupby(['item_id', 'shop_id'])['date_block_num'].transform('min')).astype('int8').clip(0, 25)
tme['price_greater_than80'] = (tme['item_price_avg'] > 1000).astype(int)
tme['price_var_within_month'] = (tme['item_price_max'] - tme['item_price_min']) / tme['item_price_avg']
tme['price_max_avg_within_month'] = tme['item_price_max'] - tme['item_price_avg']
for lag in [1, 2, 3]:
    t = time.process_time()
    print(f'Processing lag {lag} - filling strategy is for decision trees')
    tme, new_col = lag_feature(tme, lag, 'price_var_within_month', ['date_block_num', 'item_id'], fill_value=np.nan, suffix='p')
    elapsed_time = time.process_time() - t
    print(f'   -- {new_col} took {round(elapsed_time, 1)}')
    cols.append(new_col)
tme.sample(4)