text_plain_output_1.png image_output_1.png
from nltk.stem import WordNetLemmatizer, SnowballStemmer

from nltk.tokenize import RegexpTokenizer

from sklearn.feature_extraction.text import CountVectorizer

from sklearn.metrics import mean_squared_error

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import numpy as np
import pandas as pd
import os
import seaborn as sns
import plotly.express as px
import time
import matplotlib.pyplot as plt
from sklearn.metrics import mean_squared_error
mse = mean_squared_error
def downcast(df: pd.DataFrame) -> pd.DataFrame:
    float_cols = [c for c in df if df[c].dtype in ['float64']]
    int_cols = [c for c in df if df[c].dtype in ['int64']]
    df[float_cols] = df[float_cols].astype('float32')
    df[int_cols] = df[int_cols].astype('int16')
    return df
def lag_feature(df: pd.DataFrame, lag: int, col: str, merge_cols, fill_value=-10, suffix=''):
    temp = df[merge_cols + [col]]
    temp = temp.groupby(merge_cols).agg({f'{col}': 'mean'}).reset_index()
    new_col_name = f'{col}{suffix}_lag{lag}'
    temp.columns = merge_cols + [new_col_name]
    temp['date_block_num'] += lag
    if new_col_name not in df.columns:
        df = pd.merge(df, temp, on=merge_cols, how='left')
    temp = None
    df[new_col_name] = df[new_col_name].fillna(fill_value).astype('float32')
    return (df, new_col_name)

items = pd.read_csv('../input/data-preprocessing/items.csv')
shops = pd.read_csv('../input/data-preprocessing/shops.csv')
cats = pd.read_csv('../input/data-preprocessing/item_categories.csv')
train = pd.read_csv('../input/data-preprocessing/sales_train.csv')
test = pd.read_csv('../input/data-preprocessing/test.csv').set_index('ID')
dataframes = [train, shops, items, cats]
for d in dataframes:
    d = downcast(d)

tme = pd.read_csv('../input/data-preprocessing/train_monthly_extended.csv')
tme = downcast(tme)
cast_cols = ['item_cnt_month', 'days_with_sales', 'date_block_num']
tme[cast_cols] = tme[cast_cols].astype(int)
tme['item_cnt_month'] = tme['item_cnt_month'].clip(0, 50)
tme.sample(4)

import nltk
from nltk.stem import WordNetLemmatizer, SnowballStemmer
from nltk.tokenize import RegexpTokenizer
from nltk.corpus import stopwords
stop_words = ['per', 'I', 'me', 'the', 'what', 'which', 'having', 'for', 'with', 'of', 'about', 'but', 'if', 'both', 'each', 'any', 'a']
stemmer = SnowballStemmer('english')
custom_tokenizer = RegexpTokenizer('\\w+')
def manipulate_str(a):
    a = a.lower()
    word_list = custom_tokenizer.tokenize(a)
    stemmed_words = list()
    for w in word_list:
        sw = stemmer.stem(w)
        if w not in stop_words and len(sw) > 2:
            stemmed_words.append(sw)
    return ' '.join(set(stemmed_words))
items['item_name_en_tokenized'] = items.item_name_en.apply(lambda x: manipulate_str(x))

from sklearn.feature_extraction.text import CountVectorizer
vectorizer = CountVectorizer(min_df=15, max_features=40, stop_words='english')
vectorizer.fit(items['item_name_en_tokenized'])
text_features = vectorizer.transform(items['item_name_en_tokenized'])
text_features.shape

col_names = [f'txt_{c}' for c in vectorizer.get_feature_names_out()]
if type(text_features) is not pd.DataFrame:
    text_features = pd.DataFrame.sparse.from_spmatrix(text_features, columns=col_names)

text_features.mean()