text_plain_output_1.png
from nltk.corpus import stopwords

from nltk.stem import WordNetLemmatizer, SnowballStemmer

from nltk.tokenize import RegexpTokenizer

import gensim

import nltk

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

df = pd.read_csv('/kaggle/input/airbnbdm/train.csv')
test = pd.read_csv('/kaggle/input/airbnbdm/test.csv')
submission = pd.DataFrame()
submission['Id'] = test['id'].copy()

df.columns

verification_methods = ['phone', 'email', 'reviews', 'government_id', 'jumio', 'offline_government_id', 'kba', 'facebook', 'selfie', 'work_email', 'identity_manual', 'google', 'manual_offline', 'manual_online', 'sent_id', 'None', 'weibo', 'zhima_selfie', 'sesame_offline', 'sesame']

def clean(df, test):
    cols_to_drop = ['id', 'name', 'summary', 'space', 'description', 'experiences_offered', 'neighborhood_overview', 'notes', 'access', 'interaction', 'house_rules', 'host_id', 'host_name', 'host_location', 'host_about', 'host_neighbourhood', 'neighbourhood_group_cleansed', 'city', 'state', 'zipcode', 'country_code', 'square_feet', 'host_listings_count', 'reviews_per_month', 'first_review', 'last_review', 'beds', 'host_verifications']
    df['extra_people'] = df['extra_people'].str.strip('$').apply(lambda x: float(x))
    test['extra_people'] = test['extra_people'].str.strip('$').apply(lambda x: float(x))
    df['host_acceptance_rate'] = df['host_acceptance_rate'].fillna('0%').str.strip('%').apply(lambda x: float(x))
    test['host_acceptance_rate'] = test['host_acceptance_rate'].fillna('0%').str.strip('%').apply(lambda x: float(x))
    df['host_response_rate'] = df['host_response_rate'].fillna('0%').str.strip('%').apply(lambda x: float(x))
    test['host_response_rate'] = test['host_response_rate'].fillna('0%').str.strip('%').apply(lambda x: float(x))
    df['host_in_neighbourhood'] = (df['host_neighbourhood'] == df['neighbourhood_group_cleansed']).fillna(False)
    test['host_in_neighbourhood'] = (test['host_neighbourhood'] == test['neighbourhood_group_cleansed']).fillna(False)
    df['host_verifications'].str.strip('[').str.strip(']')[df['host_verifications'].str.strip('[').str.strip(']') == ''] = '[None]'
    test['host_verifications'].str.strip('[').str.strip(']')[test['host_verifications'].str.strip('[').str.strip(']') == ''] = '[None]'
    for method in verification_methods:
        df['is_' + method] = df['host_verifications'].str.contains(method)
        test['is_' + method] = test['host_verifications'].str.contains(method)
    def helper(x):
        if pd.isnull(x):
            return 0
        if x == 'within an hour':
            return 4
        if x == 'within a few hours':
            return 3
        if x == 'within a day':
            return 2
        if x == 'a few days or more':
            return 1
        return 0
    df['host_response_time'] = df['host_response_time'].apply(lambda x: helper(x))
    test['host_response_time'] = test['host_response_time'].apply(lambda x: helper(x))
    stored_ix = df[~df['description'].isnull()].index
    stored_ix2 = test[~test['description'].isnull()].index
    num_non_null_df = df[~df['description'].isnull()].shape[0]
    stemmer = SnowballStemmer('english')
    def preprocess_text(corpus):
        """Takes a corpus in list format and applies basic preprocessing steps of word tokenization,
         removing of english stop words, lower case and lemmatization."""
        processed_corpus = []
        english_words = set(nltk.corpus.words.words())
        english_stopwords = set(stopwords.words('english'))
        wordnet_lemmatizer = WordNetLemmatizer()
        tokenizer = RegexpTokenizer('[\\w|!]+')
        for row in corpus:
            word_tokens = tokenizer.tokenize(row)
            word_tokens_lower = [t.lower() for t in word_tokens]
            word_tokens_lower_english = [t for t in word_tokens_lower if t in english_words or not t.isalpha()]
            word_tokens_no_stops = [t for t in word_tokens_lower_english if not t in english_stopwords]
            word_tokens_no_stops_lemmatized = [wordnet_lemmatizer.lemmatize(t) for t in word_tokens_no_stops]
            processed_corpus.append(word_tokens_no_stops_lemmatized)
        return processed_corpus
    stemmed_stopped = preprocess_text(df['description'].dropna().append(test['description'].dropna(), ignore_index=True))
    dictionary = gensim.corpora.Dictionary(stemmed_stopped)
    dictionary.filter_extremes(no_below=15, keep_n=100000)
    bow_corpus = [dictionary.doc2bow(doc) for doc in stemmed_stopped]
    lda_model = gensim.models.LdaMulticore(bow_corpus, num_topics=3, id2word=dictionary, passes=2, workers=4)
    results = {0: [], 1: [], 2: []}
    for doc in bow_corpus[0:num_non_null_df]:
        done = []
        for i in lda_model[doc]:
            results[i[0]].append(i[1])
            done.append(i[0])
        if 0 not in done:
            results[0].append(0)
        if 1 not in done:
            results[1].append(0)
        if 2 not in done:
            results[2].append(0)
    labels = pd.DataFrame(results)
    labels.index = stored_ix
    labels.columns = ['Cluster0', 'Cluster1', 'Cluster2']
    df = df.join(labels)
    results = {0: [], 1: [], 2: []}
    for doc in bow_corpus[num_non_null_df:]:
        done = []
        for i in lda_model[doc]:
            results[i[0]].append(i[1])
            done.append(i[0])
        if 0 not in done:
            results[0].append(0)
        if 1 not in done:
            results[1].append(0)
        if 2 not in done:
            results[2].append(0)
    labels = pd.DataFrame(results)
    labels.index = stored_ix2
    labels.columns = ['Cluster0', 'Cluster1', 'Cluster2']
    test = test.join(labels)
    df['listing_time'] = df['number_of_reviews'] / df['reviews_per_month']
    df['listing_time'] = df['listing_time'].fillna(0)
    df = df.drop(cols_to_drop, axis=1)
    df['is_train'] = df['transit'].str.lower().str.contains('train')
    df['is_bus'] = df['transit'].str.lower().str.contains('bus')
    df['is_subway'] = df['transit'].str.lower().str.contains('subway')
    df['is_cab'] = df['transit'].str.lower().str.contains('cab') | df['transit'].str.lower().str.contains('car') | df['transit'].str.lower().str.contains('uber') | df['transit'].str.lower().str.contains('taxi')
    df['is_metro'] = df['transit'].str.lower().str.contains('metro')
    df['is_walk'] = df['transit'].str.lower().str.contains('walk')
    df['is_wifi'] = df['amenities'].str.lower().str.contains('wifi') | df['amenities'].str.lower().str.contains('internet')
    df['is_kitchen'] = df['amenities'].str.lower().str.contains('kitchen')
    df['is_heating'] = df['amenities'].str.lower().str.contains('heat')
    df['is_ac'] = df['amenities'].str.lower().str.contains('air conditioning')
    df['is_washer'] = df['amenities'].str.lower().str.contains('washer') | df['amenities'].str.lower().str.contains('dryer') | df['amenities'].str.lower().str.contains('dishwasher')
    df['is_tv'] = df['amenities'].str.lower().str.contains('tv')
    df['is_gym'] = df['amenities'].str.lower().str.contains('gym')
    df['is_pets'] = df['amenities'].str.lower().str.contains('pet')
    df['is_balcony'] = df['amenities'].str.lower().str.contains('balcony')
    df['is_linen'] = df['amenities'].str.lower().str.contains('linen')
    df['is_breakfast'] = df['amenities'].str.lower().str.contains('breakfast')
    df['is_coffee'] = df['amenities'].str.lower().str.contains('coffee')
    df['is_cooking'] = df['amenities'].str.lower().str.contains('cooking')
    df['is_pool'] = df['amenities'].str.lower().str.contains('pool')
    df['amenities'] = df['amenities'].str.strip('{').str.strip('}').str.split(',').apply(lambda x: len(x) if x[0] != '' else 0)
    df['host_is_superhost'] = df['host_is_superhost'].replace({'f': False, 't': True})
    df = df.drop(['transit'], axis=1)
    for col in df[pd.Series(df.columns)[pd.Series(df.columns).str.contains('review')].to_list()].columns:
        to_fill = pd.Series(df[col].sample(df[col].isnull().sum()))
        to_fill.index = df[df[col].isnull()].index
        df[col] = df[col].fillna(to_fill)
    df[df.columns[df.columns.str.contains('^is')]] = df[df.columns[df.columns.str.contains('^is')]].fillna('False')
    df = df.fillna(0)
    df = df.reset_index(drop=True)
    test['listing_time'] = test['number_of_reviews'] / test['reviews_per_month']
    test['listing_time'] = test['listing_time'].fillna(0)
    test = test.drop(cols_to_drop, axis=1)
    test['is_train'] = test['transit'].str.lower().str.contains('train')
    test['is_bus'] = test['transit'].str.lower().str.contains('bus')
    test['is_subway'] = test['transit'].str.lower().str.contains('subway')
    test['is_cab'] = test['transit'].str.lower().str.contains('cab') | test['transit'].str.lower().str.contains('car') | test['transit'].str.lower().str.contains('uber') | test['transit'].str.lower().str.contains('taxi')
    test['is_metro'] = test['transit'].str.lower().str.contains('metro')
    test['is_walk'] = test['transit'].str.lower().str.contains('walk')
    test['is_wifi'] = test['amenities'].str.lower().str.contains('wifi') | test['amenities'].str.lower().str.contains('internet')
    test['is_kitchen'] = test['amenities'].str.lower().str.contains('kitchen')
    test['is_heating'] = test['amenities'].str.lower().str.contains('heat')
    test['is_ac'] = test['amenities'].str.lower().str.contains('air conditioning')
    test['is_washer'] = test['amenities'].str.lower().str.contains('washer') | test['amenities'].str.lower().str.contains('dryer') | test['amenities'].str.lower().str.contains('dishwasher')
    test['is_tv'] = test['amenities'].str.lower().str.contains('tv')
    test['is_gym'] = test['amenities'].str.lower().str.contains('gym')
    test['is_pets'] = test['amenities'].str.lower().str.contains('pet')
    test['is_balcony'] = test['amenities'].str.lower().str.contains('balcony')
    test['is_linen'] = test['amenities'].str.lower().str.contains('linen')
    test['is_breakfast'] = test['amenities'].str.lower().str.contains('breakfast')
    test['is_coffee'] = test['amenities'].str.lower().str.contains('coffee')
    test['is_cooking'] = test['amenities'].str.lower().str.contains('cooking')
    test['is_pool'] = test['amenities'].str.lower().str.contains('pool')
    test['amenities'] = test['amenities'].str.strip('{').str.strip('}').str.split(',').apply(lambda x: len(x) if x[0] != '' else 0)
    test['host_is_superhost'] = test['host_is_superhost'].replace({'f': False, 't': True})
    test = test.drop(['transit'], axis=1)
    for col in test[pd.Series(test.columns)[pd.Series(test.columns).str.contains('review')].to_list()].columns:
        to_fill = pd.Series(test[col].sample(test[col].isnull().sum()))
        to_fill.index = test[test[col].isnull()].index
        test[col] = test[col].fillna(to_fill)
    test[test.columns[test.columns.str.contains('^is')]] = test[test.columns[test.columns.str.contains('^is')]].fillna('False')
    test = test.fillna(0)
    df['bedrooms'] = df['bedrooms'].apply(lambda x: str(x))
    df['bathrooms'] = df['bathrooms'].apply(lambda x: str(x))
    test['bedrooms'] = test['bedrooms'].apply(lambda x: str(x))
    test['bathrooms'] = test['bathrooms'].apply(lambda x: str(x))
    test = test.reset_index(drop=True)
    return (df.replace({False: 0, True: 1, 'False': 0, 'True': 1, 'f': 0, 't': 1}), test.replace({False: 0, True: 1, 'False': 0, 'True': 1, 'f': 0, 't': 1}))

df, test = clean(df, test)