application_vnd.jupyter.stderr_output_1.png
from scipy.sparse import csr_matrix, hstack

from sklearn.preprocessing import LabelEncoder

import numpy as np

import os

import pandas as pd

datadir = '../input'
gatrain = pd.read_csv(os.path.join(datadir, 'gender_age_train.csv'), index_col='device_id')
gatest = pd.read_csv(os.path.join(datadir, 'gender_age_test.csv'), index_col='device_id')
phone = pd.read_csv(os.path.join(datadir, 'phone_brand_device_model.csv'))
phone = phone.drop_duplicates('device_id', keep='first').set_index('device_id')
events = pd.read_csv(os.path.join(datadir, 'events.csv'), parse_dates=['timestamp'], index_col='event_id')
appevents = pd.read_csv(os.path.join(datadir, 'app_events.csv'), usecols=['event_id', 'app_id', 'is_active'], dtype={'is_active': bool})
applabels = pd.read_csv(os.path.join(datadir, 'app_labels.csv'))
labelcats = pd.read_csv(os.path.join(datadir, 'label_categories.csv'), index_col='label_id', squeeze=True)

gatrain['trainrow'] = np.arange(gatrain.shape[0])
gatest['testrow'] = np.arange(gatest.shape[0])

brandencoder = LabelEncoder().fit(phone.phone_brand)
phone['brand'] = brandencoder.transform(phone['phone_brand'])
gatrain['brand'] = phone['brand']
gatest['brand'] = phone['brand']
Xtr_brand = csr_matrix((np.ones(gatrain.shape[0]), (gatrain.trainrow, gatrain.brand)))
Xte_brand = csr_matrix((np.ones(gatest.shape[0]), (gatest.testrow, gatest.brand)))

m = phone.phone_brand.str.cat(phone.device_model)
modelencoder = LabelEncoder().fit(m)
phone['model'] = modelencoder.transform(m)
gatrain['model'] = phone['model']
gatest['model'] = phone['model']
Xtr_model = csr_matrix((np.ones(gatrain.shape[0]), (gatrain.trainrow, gatrain.model)))
Xte_model = csr_matrix((np.ones(gatest.shape[0]), (gatest.testrow, gatest.model)))

appencoder = LabelEncoder().fit(appevents.app_id)
appevents['app'] = appencoder.transform(appevents.app_id)
napps = len(appencoder.classes_)
deviceapps = appevents.merge(events[['device_id']], how='left', left_on='event_id', right_index=True).groupby(['device_id', 'app'])['app'].agg(['size']).merge(gatrain[['trainrow']], how='left', left_index=True, right_index=True).merge(gatest[['testrow']], how='left', left_index=True, right_index=True).reset_index()

d = deviceapps.dropna(subset=['trainrow'])
Xtr_app = csr_matrix((np.ones(d.shape[0]), (d.trainrow, d.app)), shape=(gatrain.shape[0], napps))
d = deviceapps.dropna(subset=['testrow'])
Xte_app = csr_matrix((np.ones(d.shape[0]), (d.testrow, d.app)), shape=(gatest.shape[0], napps))
print('Apps data: train shape {}, test shape {}'.format(Xtr_app.shape, Xte_app.shape))