text_plain_output_1.png
from catboost import CatBoostRegressor

from nltk.corpus import stopwords

from nltk.stem import WordNetLemmatizer, SnowballStemmer

from nltk.tokenize import RegexpTokenizer

from sklearn.compose import ColumnTransformer

from sklearn.pipeline import Pipeline

from sklearn.preprocessing import FunctionTransformer

from sklearn.preprocessing import OneHotEncoder

import gensim

import nltk

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

df = pd.read_csv('/kaggle/input/airbnbdm/train.csv')
test = pd.read_csv('/kaggle/input/airbnbdm/test.csv')
submission = pd.DataFrame()
submission['Id'] = test['id'].copy()

df.columns

verification_methods = ['phone', 'email', 'reviews', 'government_id', 'jumio', 'offline_government_id', 'kba', 'facebook', 'selfie', 'work_email', 'identity_manual', 'google', 'manual_offline', 'manual_online', 'sent_id', 'None', 'weibo', 'zhima_selfie', 'sesame_offline', 'sesame']

def clean(df, test):
    cols_to_drop = ['id', 'name', 'summary', 'space', 'description', 'experiences_offered', 'neighborhood_overview', 'notes', 'access', 'interaction', 'house_rules', 'host_id', 'host_name', 'host_location', 'host_about', 'host_neighbourhood', 'neighbourhood_group_cleansed', 'city', 'state', 'zipcode', 'country_code', 'square_feet', 'host_listings_count', 'reviews_per_month', 'first_review', 'last_review', 'beds', 'host_verifications']
    df['extra_people'] = df['extra_people'].str.strip('$').apply(lambda x: float(x))
    test['extra_people'] = test['extra_people'].str.strip('$').apply(lambda x: float(x))
    df['host_acceptance_rate'] = df['host_acceptance_rate'].fillna('0%').str.strip('%').apply(lambda x: float(x))
    test['host_acceptance_rate'] = test['host_acceptance_rate'].fillna('0%').str.strip('%').apply(lambda x: float(x))
    df['host_response_rate'] = df['host_response_rate'].fillna('0%').str.strip('%').apply(lambda x: float(x))
    test['host_response_rate'] = test['host_response_rate'].fillna('0%').str.strip('%').apply(lambda x: float(x))
    df['host_in_neighbourhood'] = (df['host_neighbourhood'] == df['neighbourhood_group_cleansed']).fillna(False)
    test['host_in_neighbourhood'] = (test['host_neighbourhood'] == test['neighbourhood_group_cleansed']).fillna(False)
    df['host_verifications'].str.strip('[').str.strip(']')[df['host_verifications'].str.strip('[').str.strip(']') == ''] = '[None]'
    test['host_verifications'].str.strip('[').str.strip(']')[test['host_verifications'].str.strip('[').str.strip(']') == ''] = '[None]'
    for method in verification_methods:
        df['is_' + method] = df['host_verifications'].str.contains(method)
        test['is_' + method] = test['host_verifications'].str.contains(method)
    def helper(x):
        if pd.isnull(x):
            return 0
        if x == 'within an hour':
            return 4
        if x == 'within a few hours':
            return 3
        if x == 'within a day':
            return 2
        if x == 'a few days or more':
            return 1
        return 0
    df['host_response_time'] = df['host_response_time'].apply(lambda x: helper(x))
    test['host_response_time'] = test['host_response_time'].apply(lambda x: helper(x))
    stored_ix = df[~df['description'].isnull()].index
    stored_ix2 = test[~test['description'].isnull()].index
    num_non_null_df = df[~df['description'].isnull()].shape[0]
    stemmer = SnowballStemmer('english')
    def preprocess_text(corpus):
        """Takes a corpus in list format and applies basic preprocessing steps of word tokenization,
         removing of english stop words, lower case and lemmatization."""
        processed_corpus = []
        english_words = set(nltk.corpus.words.words())
        english_stopwords = set(stopwords.words('english'))
        wordnet_lemmatizer = WordNetLemmatizer()
        tokenizer = RegexpTokenizer('[\\w|!]+')
        for row in corpus:
            word_tokens = tokenizer.tokenize(row)
            word_tokens_lower = [t.lower() for t in word_tokens]
            word_tokens_lower_english = [t for t in word_tokens_lower if t in english_words or not t.isalpha()]
            word_tokens_no_stops = [t for t in word_tokens_lower_english if not t in english_stopwords]
            word_tokens_no_stops_lemmatized = [wordnet_lemmatizer.lemmatize(t) for t in word_tokens_no_stops]
            processed_corpus.append(word_tokens_no_stops_lemmatized)
        return processed_corpus
    stemmed_stopped = preprocess_text(df['description'].dropna().append(test['description'].dropna(), ignore_index=True))
    dictionary = gensim.corpora.Dictionary(stemmed_stopped)
    dictionary.filter_extremes(no_below=15, keep_n=100000)
    bow_corpus = [dictionary.doc2bow(doc) for doc in stemmed_stopped]
    lda_model = gensim.models.LdaMulticore(bow_corpus, num_topics=3, id2word=dictionary, passes=2, workers=4)
    results = {0: [], 1: [], 2: []}
    for doc in bow_corpus[0:num_non_null_df]:
        done = []
        for i in lda_model[doc]:
            results[i[0]].append(i[1])
            done.append(i[0])
        if 0 not in done:
            results[0].append(0)
        if 1 not in done:
            results[1].append(0)
        if 2 not in done:
            results[2].append(0)
    labels = pd.DataFrame(results)
    labels.index = stored_ix
    labels.columns = ['Cluster0', 'Cluster1', 'Cluster2']
    df = df.join(labels)
    results = {0: [], 1: [], 2: []}
    for doc in bow_corpus[num_non_null_df:]:
        done = []
        for i in lda_model[doc]:
            results[i[0]].append(i[1])
            done.append(i[0])
        if 0 not in done:
            results[0].append(0)
        if 1 not in done:
            results[1].append(0)
        if 2 not in done:
            results[2].append(0)
    labels = pd.DataFrame(results)
    labels.index = stored_ix2
    labels.columns = ['Cluster0', 'Cluster1', 'Cluster2']
    test = test.join(labels)
    df['listing_time'] = df['number_of_reviews'] / df['reviews_per_month']
    df['listing_time'] = df['listing_time'].fillna(0)
    df = df.drop(cols_to_drop, axis=1)
    df['is_train'] = df['transit'].str.lower().str.contains('train')
    df['is_bus'] = df['transit'].str.lower().str.contains('bus')
    df['is_subway'] = df['transit'].str.lower().str.contains('subway')
    df['is_cab'] = df['transit'].str.lower().str.contains('cab') | df['transit'].str.lower().str.contains('car') | df['transit'].str.lower().str.contains('uber') | df['transit'].str.lower().str.contains('taxi')
    df['is_metro'] = df['transit'].str.lower().str.contains('metro')
    df['is_walk'] = df['transit'].str.lower().str.contains('walk')
    df['is_wifi'] = df['amenities'].str.lower().str.contains('wifi') | df['amenities'].str.lower().str.contains('internet')
    df['is_kitchen'] = df['amenities'].str.lower().str.contains('kitchen')
    df['is_heating'] = df['amenities'].str.lower().str.contains('heat')
    df['is_ac'] = df['amenities'].str.lower().str.contains('air conditioning')
    df['is_washer'] = df['amenities'].str.lower().str.contains('washer') | df['amenities'].str.lower().str.contains('dryer') | df['amenities'].str.lower().str.contains('dishwasher')
    df['is_tv'] = df['amenities'].str.lower().str.contains('tv')
    df['is_gym'] = df['amenities'].str.lower().str.contains('gym')
    df['is_pets'] = df['amenities'].str.lower().str.contains('pet')
    df['is_balcony'] = df['amenities'].str.lower().str.contains('balcony')
    df['is_linen'] = df['amenities'].str.lower().str.contains('linen')
    df['is_breakfast'] = df['amenities'].str.lower().str.contains('breakfast')
    df['is_coffee'] = df['amenities'].str.lower().str.contains('coffee')
    df['is_cooking'] = df['amenities'].str.lower().str.contains('cooking')
    df['is_pool'] = df['amenities'].str.lower().str.contains('pool')
    df['amenities'] = df['amenities'].str.strip('{').str.strip('}').str.split(',').apply(lambda x: len(x) if x[0] != '' else 0)
    df['host_is_superhost'] = df['host_is_superhost'].replace({'f': False, 't': True})
    df = df.drop(['transit'], axis=1)
    for col in df[pd.Series(df.columns)[pd.Series(df.columns).str.contains('review')].to_list()].columns:
        to_fill = pd.Series(df[col].sample(df[col].isnull().sum()))
        to_fill.index = df[df[col].isnull()].index
        df[col] = df[col].fillna(to_fill)
    df[df.columns[df.columns.str.contains('^is')]] = df[df.columns[df.columns.str.contains('^is')]].fillna('False')
    df = df.fillna(0)
    df = df.reset_index(drop=True)
    test['listing_time'] = test['number_of_reviews'] / test['reviews_per_month']
    test['listing_time'] = test['listing_time'].fillna(0)
    test = test.drop(cols_to_drop, axis=1)
    test['is_train'] = test['transit'].str.lower().str.contains('train')
    test['is_bus'] = test['transit'].str.lower().str.contains('bus')
    test['is_subway'] = test['transit'].str.lower().str.contains('subway')
    test['is_cab'] = test['transit'].str.lower().str.contains('cab') | test['transit'].str.lower().str.contains('car') | test['transit'].str.lower().str.contains('uber') | test['transit'].str.lower().str.contains('taxi')
    test['is_metro'] = test['transit'].str.lower().str.contains('metro')
    test['is_walk'] = test['transit'].str.lower().str.contains('walk')
    test['is_wifi'] = test['amenities'].str.lower().str.contains('wifi') | test['amenities'].str.lower().str.contains('internet')
    test['is_kitchen'] = test['amenities'].str.lower().str.contains('kitchen')
    test['is_heating'] = test['amenities'].str.lower().str.contains('heat')
    test['is_ac'] = test['amenities'].str.lower().str.contains('air conditioning')
    test['is_washer'] = test['amenities'].str.lower().str.contains('washer') | test['amenities'].str.lower().str.contains('dryer') | test['amenities'].str.lower().str.contains('dishwasher')
    test['is_tv'] = test['amenities'].str.lower().str.contains('tv')
    test['is_gym'] = test['amenities'].str.lower().str.contains('gym')
    test['is_pets'] = test['amenities'].str.lower().str.contains('pet')
    test['is_balcony'] = test['amenities'].str.lower().str.contains('balcony')
    test['is_linen'] = test['amenities'].str.lower().str.contains('linen')
    test['is_breakfast'] = test['amenities'].str.lower().str.contains('breakfast')
    test['is_coffee'] = test['amenities'].str.lower().str.contains('coffee')
    test['is_cooking'] = test['amenities'].str.lower().str.contains('cooking')
    test['is_pool'] = test['amenities'].str.lower().str.contains('pool')
    test['amenities'] = test['amenities'].str.strip('{').str.strip('}').str.split(',').apply(lambda x: len(x) if x[0] != '' else 0)
    test['host_is_superhost'] = test['host_is_superhost'].replace({'f': False, 't': True})
    test = test.drop(['transit'], axis=1)
    for col in test[pd.Series(test.columns)[pd.Series(test.columns).str.contains('review')].to_list()].columns:
        to_fill = pd.Series(test[col].sample(test[col].isnull().sum()))
        to_fill.index = test[test[col].isnull()].index
        test[col] = test[col].fillna(to_fill)
    test[test.columns[test.columns.str.contains('^is')]] = test[test.columns[test.columns.str.contains('^is')]].fillna('False')
    test = test.fillna(0)
    df['bedrooms'] = df['bedrooms'].apply(lambda x: str(x))
    df['bathrooms'] = df['bathrooms'].apply(lambda x: str(x))
    test['bedrooms'] = test['bedrooms'].apply(lambda x: str(x))
    test['bathrooms'] = test['bathrooms'].apply(lambda x: str(x))
    test = test.reset_index(drop=True)
    return (df.replace({False: 0, True: 1, 'False': 0, 'True': 1, 'f': 0, 't': 1}), test.replace({False: 0, True: 1, 'False': 0, 'True': 1, 'f': 0, 't': 1}))

df.columns

df.shape

scores = []
for i in range(20):
    time_features = ['host_since']
    time_converter = Pipeline(steps=[('ft', FunctionTransformer(lambda x: (2020 - pd.to_datetime(x['host_since']).apply(lambda u: u.year)).values.reshape(-1, 1)))])
    ohe_features = ['neighbourhood_cleansed', 'property_type', 'room_type', 'bed_type', 'cancellation_policy', 'bathrooms', 'bedrooms', 'market', 'country']
    ohe_converter = Pipeline(steps=[('ohe', OneHotEncoder(handle_unknown='ignore'))])
    preproc = ColumnTransformer(transformers=[('time', time_converter, time_features)], remainder='passthrough')
    catt_features = [df.drop(['price'], axis=1).columns.get_loc(col) for col in ohe_features]
    pl = Pipeline(steps=[('preprocessor', preproc), ('regressor', CatBoostRegressor(cat_features=catt_features, silent=True))])
    break

submission['Predicted'] = pd.Series(pl.predict(test))

preprocessed = pl['preprocessor'].fit_transform(df.drop('price', axis=1))
preprocessed = preprocessed

preprocessed