text_plain_output_1.png
from scipy import sparse

from sklearn import model_selection, preprocessing, ensemble

from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer

import numpy as np

import pandas as pd

import xgboost as xgb

def runXGB(train_X, train_y, test_X, test_y=None, feature_names=None, seed_val=0, num_rounds=1000):
    param = {}
    param['objective'] = 'multi:softprob'
    param['eta'] = 0.1
    param['max_depth'] = 6
    param['silent'] = 1
    param['num_class'] = 3
    param['eval_metric'] = 'mlogloss'
    param['min_child_weight'] = 1
    param['subsample'] = 0.7
    param['colsample_bytree'] = 0.7
    param['seed'] = 8088
    num_rounds = num_rounds
    plst = list(param.items())
    xgtrain = xgb.DMatrix(train_X, label=train_y)
    if test_y is not None:
        xgtest = xgb.DMatrix(test_X, label=test_y)
        watchlist = [(xgtrain, 'train'), (xgtest, 'test')]
        model = xgb.train(plst, xgtrain, num_rounds, watchlist, early_stopping_rounds=20, verbose_eval=25)
    else:
        xgtest = xgb.DMatrix(test_X)
        model = xgb.train(plst, xgtrain, num_rounds)
    pred_test_y = model.predict(xgtest)
    return (pred_test_y, model)

data_path = '../input/'
train_file = data_path + 'train.json'
test_file = data_path + 'test.json'
train_df = pd.read_json(train_file)
test_df = pd.read_json(test_file)

features_to_use = ['bathrooms', 'bedrooms', 'latitude', 'longitude', 'price']

train_df['num_photos'] = train_df['photos'].apply(len)
test_df['num_photos'] = test_df['photos'].apply(len)
train_df['num_features'] = train_df['features'].apply(len)
test_df['num_features'] = test_df['features'].apply(len)
train_df['num_description_words'] = train_df['description'].apply(lambda x: len(x.split(' ')))
test_df['num_description_words'] = test_df['description'].apply(lambda x: len(x.split(' ')))
train_df['created'] = pd.to_datetime(train_df['created'])
test_df['created'] = pd.to_datetime(test_df['created'])
train_df['created_year'] = train_df['created'].dt.year
test_df['created_year'] = test_df['created'].dt.year
train_df['created_month'] = train_df['created'].dt.month
test_df['created_month'] = test_df['created'].dt.month
train_df['created_day'] = train_df['created'].dt.day
test_df['created_day'] = test_df['created'].dt.day
train_df['created_hour'] = train_df['created'].dt.hour
test_df['created_hour'] = test_df['created'].dt.hour
train_df['price_per_bed'] = train_df['price'] / train_df['bedrooms'].clip(lower=1)
test_df['price_per_bed'] = test_df['price'] / test_df['bedrooms'].clip(lower=1)
train_df['created_date'] = np.array(train_df.created.values, dtype='datetime64[D]').astype(np.float32)
test_df['created_date'] = np.array(test_df.created.values, dtype='datetime64[D]').astype(np.float32)
features_to_use.extend(['num_photos', 'num_features', 'num_description_words', 'created_year', 'created_month', 'created_day', 'listing_id', 'created_hour', 'price_per_bed', 'created_date'])

categorical = ['display_address', 'manager_id', 'building_id', 'street_address']
for f in categorical:
    if train_df[f].dtype == 'object':
        lbl = preprocessing.LabelEncoder()
        lbl.fit(list(train_df[f].values) + list(test_df[f].values))
        train_df[f] = lbl.transform(list(train_df[f].values))
        test_df[f] = lbl.transform(list(test_df[f].values))
        features_to_use.append(f)

train_df['features'] = train_df['features'].apply(lambda x: ' '.join(['_'.join(i.split(' ')) for i in x]))
test_df['features'] = test_df['features'].apply(lambda x: ' '.join(['_'.join(i.split(' ')) for i in x]))
tfidf = CountVectorizer(stop_words='english', max_features=200)
tr_sparse = tfidf.fit_transform(train_df['features'])
te_sparse = tfidf.transform(test_df['features'])

features_to_use = [i for i in features_to_use if 'manager_id' not in i]

train_X = sparse.hstack([train_df[features_to_use], tr_sparse]).tocsr()
test_X = sparse.hstack([test_df[features_to_use], te_sparse]).tocsr()
features_sparse = ['features_' + str(i) for i in range(tr_sparse.shape[1])]
target_num_map = {'high': 0, 'medium': 1, 'low': 2}
train_y = np.array(train_df['interest_level'].apply(lambda x: target_num_map[x]))

fscores = model.get_fscore()
df_features = pd.DataFrame(fscores, index=['score']).T.reset_index()
df_features['f_index'] = df_features['index'].apply(lambda x: int(x[1:]))
df_features['f_name'] = np.array(features_to_use + features_sparse)[df_features['f_index'].values]
pd.Series(index=df_features['f_name'].values, data=df_features['score'].values).sort_values()[-30:].plot(kind='bar')