application_vnd.jupyter.stderr_output_2.png text_plain_output_1.png
from subprocess import check_output

from subprocess import check_output

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import numpy as np
import pandas as pd
from subprocess import check_output
from subprocess import check_output
directory = '../input/'
train = pd.read_csv(directory + 'train.csv')
test = pd.read_csv(directory + 'test.csv')
numeric_feats = [x for x in train.columns[1:-1] if 'cont' in x]
categorical_feats = [x for x in train.columns[1:-1] if 'cat' in x]
catwithdummies = pd.get_dummies(train)

catwithdummies = pd.get_dummies(categorical_feats)

index = list(train.index)
print(index[0:10])
np.random.shuffle(index)
print(index[0:10])
train = train.iloc[index]
'train = train.iloc[np.random.permutation(len(train))]'
test['loss'] = np.nan
y = np.log(train['loss'].values + 200)
id_train = train['id'].values
id_test = test['id'].values
ntrain = train.shape[0]
tr_te = pd.concat((train, test), axis=0)
sparse_data = []
f_cat = [f for f in tr_te.columns if 'cat' in f]
for f in f_cat:
    dummy = pd.get_dummies(tr_te[f].astype('category'))
    tmp = csr_matrix(dummy)
    sparse_data.append(tmp)
f_num = [f for f in tr_te.columns if 'cont' in f]
scaler = StandardScaler()
tmp = csr_matrix(scaler.fit_transform(tr_te[f_num]))
sparse_data.append(tmp)
del (tr_te, train, test)
xtr_te = hstack(sparse_data, format='csr')
xtrain = xtr_te[:ntrain, :]
xtest = xtr_te[ntrain:, :]
print('Dim train', xtrain.shape)
print('Dim test', xtest.shape)