text_plain_output_2.png text_plain_output_1.png
from sklearn import preprocessing

from sklearn.feature_extraction.text import TfidfVectorizer

from subprocess import check_output

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import numpy as np
import pandas as pd
import scipy
from sklearn.feature_extraction.text import HashingVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD
from sklearn.naive_bayes import BernoulliNB, MultinomialNB
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import log_loss
from sklearn.model_selection import train_test_split
from sklearn import preprocessing
from subprocess import check_output
print(check_output(['ls', '../input']).decode('utf8'))
df = pd.read_json('../input/train.json')
df['priceperbed'] = df['price'].clip(upper=7000) / df['bedrooms'].clip(lower=1)
df['created'] = df['created'].astype(np.datetime64)
df['created_day'] = np.array(df.created.values, dtype='datetime64[D]').astype(np.float32) % 7
df['created_week'] = np.array(df.created.values, dtype='datetime64[W]').astype(np.float32)
df['created_hour'] = np.array(df.created.values, dtype='datetime64[h]').astype(np.float32) % 24
df['desc_count'] = df.description.apply(lambda x: len(x.split())).clip(upper=150)
df['features_count'] = df.features.apply(lambda x: len(x))
df['photos_count'] = df.photos.apply(lambda x: len(x))
lbl = preprocessing.LabelEncoder()
lbl.fit(list(df['manager_id'].values))
df['manager_id'] = lbl.transform(list(df['manager_id'].values))
feature_list = ['no fee', 'hardwood floors', 'laundry in building']
df['features'] = df['features'].apply(lambda x: list(map(str.lower, x)))
for feature in feature_list:
    df[feature] = df['features'].apply(lambda x: feature in x)
vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.5, stop_words='english')
vectorizer.fit(df.description.values)