application_vnd.jupyter.stderr_output_1.png
from scipy.sparse import csr_matrix, hstack

from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer

from sklearn.preprocessing import LabelEncoder, MinMaxScaler, StandardScaler,LabelBinarizer

import pandas as pd

train = pd.read_csv('../input/train.tsv', sep='\t')
test = pd.read_csv('../input/test.tsv', sep='\t')

def handle_missing(dataset):
    dataset['category_name'].fillna(value='NA/NA/NA', inplace=True)
    dataset.brand_name.fillna(value='missing', inplace=True)
    dataset.item_description.fillna(value='missing', inplace=True)
    return dataset
def split_cat(dataset):
    dataset['cat1'], dataset['cat2'], dataset['cat3'] = zip(*dataset['category_name'].str.split('/', 2))
    return dataset
def label_maker(dataset):
    lb = LabelBinarizer(sparse_output=True)
    cat1 = lb.fit_transform(dataset['cat1'])
    cat2 = lb.fit_transform(dataset['cat2'])
    cat3 = lb.fit_transform(dataset['cat3'])
    brand_name = lb.fit_transform(dataset['brand_name'])
    del lb
    return (cat1, cat2, cat3, brand_name)
def get_dums(dataset):
    X_dummies = csr_matrix(pd.get_dummies(dataset[['item_condition_id', 'shipping']], sparse=True).values)
    return X_dummies
def text_processing(dataset):
    cv = CountVectorizer()
    name = cv.fit_transform(dataset['name'])
    tv = TfidfVectorizer(max_features=51000, ngram_range=(1, 3), stop_words='english')
    description = tv.fit_transform(dataset['item_description'])
    del cv, tv
    return (name, description)

nrow_train = train.shape[0]
merge: pd.DataFrame = pd.concat([train, test])
submission: pd.DataFrame = test[['test_id']]