text_html_output_1.png
from sklearn.metrics import mean_squared_error

from sklearn.neighbors import KNeighborsRegressor

from sklearn.preprocessing import StandardScaler, MinMaxScaler

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import time

train_set = pd.read_csv('/kaggle/input/preprocessed-sales-data/train_set.csv')
validation_set = pd.read_csv('/kaggle/input/preprocessed-sales-data/validation_set.csv')
test_set = pd.read_csv('/kaggle/input/competitive-data-science-predict-future-sales/test.csv')

train_set.head().T

x_train = train_set.drop(['item_cnt_month', 'date_block_num'], axis=1)
y_train = train_set['item_cnt_month'].astype(int)
x_val = validation_set.drop(['item_cnt_month', 'date_block_num'], axis=1)
y_val = validation_set['item_cnt_month'].astype(int)

latest_records = pd.concat([train_set, validation_set]).drop_duplicates(subset=['shop_id', 'item_id'], keep='last')
x_test = pd.merge(test_set, latest_records, on=['shop_id', 'item_id'], how='left', suffixes=['', '_'])
x_test['year'] = 2015
x_test['month'] = 9
x_test.drop('item_cnt_month', axis=1, inplace=True)
x_test = x_test[x_train.columns]

ts = time.time()
sets = [x_train, x_val, x_test]
for dataset in sets:
    for shop_id in dataset['shop_id'].unique():
        for column in dataset.columns:
            shop_median = dataset[dataset['shop_id'] == shop_id][column].median()
            dataset.loc[dataset[column].isnull() & (dataset['shop_id'] == shop_id), column] = shop_median
x_test.fillna(x_test.mean(), inplace=True)

x_test.head().T

all_f = ['shop_id', 'item_id', 'item_cnt', 'mean_item_cnt', 'transactions', 'year', 'month', 'item_cnt_mean', 'item_cnt_std', 'item_cnt_shifted1', 'item_cnt_shifted2', 'item_cnt_shifted3', 'item_trend', 'shop_mean', 'item_mean', 'shop_item_mean', 'year_mean', 'month_mean']

x_tr = x_train[all_f]
x_va = x_val[all_f]
x_te = x_test[all_f]

def models(model, x_tr, y_train, x_va, x_te):
    model.fit(x_tr, y_train)
    train_pred = model.predict(x_tr)
    val_pred = model.predict(x_va)
    test_pred = model.predict(x_te)
    return (train_pred, val_pred, test_pred)

x_t_knn = x_tr[:100000]
y_t_knn = y_train[:100000]
scaler = MinMaxScaler()
scaler.fit(x_t_knn)
scaled_x_t_knn = scaler.transform(x_t_knn)
scaled_x_v = scaler.transform(x_v)
KN = KNeighborsRegressor(n_neighbors=20, leaf_size=15, n_jobs=-1)
M2_train, M2_val = models(KN, scaled_x_t_knn, y_t_knn, scaled_x_v, x_te)

print('Train rmse:', np.sqrt(mean_squared_error(y_train, M3_train)))
print('Validation rmse:', np.sqrt(mean_squared_error(y_val, M3_val)))