text_html_output_1.png
import glob

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import pickle

import numpy as np
import pandas as pd
import os
Train = pd.read_csv('../input/house-prices-advanced-regression-techniques/train.csv')
Train = Train.set_index('Id')
Test = pd.read_csv('../input/house-prices-advanced-regression-techniques/test.csv')
Test = Test.set_index('Id')
preprocessing_path = '../input/house-price-eda'
train_scale = pd.read_csv(f'{preprocessing_path}/train_scaled.csv')
train_scale = train_scale.set_index('Id')
test_scale = pd.read_csv(f'{preprocessing_path}/test_scaled.csv')
try:
    test_scale = test_scale.rename(columns={'Unnamed: 0': 'Id'})
except:
    pass
test_scale = test_scale.set_index('Id')
Train = Train.loc[train_scale.index]
import glob
listing = glob.glob(f'{preprocessing_path}/*_one_hot_pickle')
import pickle
for model_file in listing:
    col = model_file.split('_one_')[0]
    col = col.split('/')[-1]
    enc = pickle.load(open(model_file, 'rb'))
    new_cols = pd.DataFrame(enc.transform(Train[[col]]).toarray(), columns=f'{col}_' + enc.categories_[0])
    new_cols.index = Train.index
    Train = pd.concat([Train, new_cols], axis=1)
    Train = Train.drop(col, axis=1)
for log_col in train_scale.columns[train_scale.columns.str.endswith('log')]:
    col = log_col.split('_log')[0]
    Train[log_col] = np.log(Train[col])
    Train = Train.drop(col, axis=1)
Train = Train[train_scale.columns]
for model_file in listing:
    col = model_file.split('_one_')[0]
    col = col.split('/')[-1]
    enc = pickle.load(open(model_file, 'rb'))
    new_cols = pd.DataFrame(enc.transform(Test[[col]]).toarray(), columns=f'{col}_' + enc.categories_[0])
    new_cols.index = Test.index
    Test = pd.concat([Test, new_cols], axis=1)
    Test = Test.drop(col, axis=1)
for log_col in test_scale.columns[test_scale.columns.str.endswith('log')]:
    col = log_col.split('_log')[0]
    Test[log_col] = np.log(Test[col])
    Test = Test.drop(col, axis=1)
Test = Test[test_scale.columns]

for col in Test.columns:
    if len(set(Train[col])) > 2 and '_log' not in col:
        Train[f'{col}_log'] = np.log(Train[col] + 1)
        Train = Train.drop(col, axis=1)
        Test[f'{col}_log'] = np.log(Test[col] + 1)
        Test = Test.drop(col, axis=1)

Test