text_plain_output_1.png
from sklearn.ensemble import RandomForestRegressor

from sklearn.metrics import mean_absolute_error

from sklearn.preprocessing import OneHotEncoder

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

train_data = pd.read_csv('/kaggle/input/titanic/train.csv', index_col='PassengerId')

test_data = pd.read_csv('/kaggle/input/titanic/test.csv', index_col='PassengerId')

cols_with_missing = [col for col in train_data.columns if train_data[col].isnull().any()]
cols_with_missing

cat = train_data.dtypes == 'object'
object_cols = list(cat[cat].index)
object_cols

test_cols_with_missing = [col for col in test_data.columns if test_data[col].isnull().any()]
test_cols_with_missing

test_cat = test_data.dtypes == 'object'
test_object_cols = list(test_cat[test_cat].index)
test_object_cols

object_cols = [col for col in train_data.columns if train_data[col].dtype == 'object']
good_label_cols = [col for col in object_cols if set(train_data[col]) == set(test_data[col])]
bad_label_cols = list(set(object_cols) - set(good_label_cols))

def score_dataset(X_train, X_valid, y_train, y_valid):
    model = RandomForestRegressor(n_estimators=100, random_state=0)
    model.fit(X_train, y_train)
    preds = model.predict(X_valid)
    return mean_absolute_error(y_valid, preds)

cat = X_train.dtypes == 'object'
object_cols = list(cat[cat].index)
object_cols
object_cols = [col for col in X_train.columns if X_train[col].dtype == 'object']
good_label_cols = [col for col in object_cols if set(X_train[col]) == set(X_valid[col])]
bad_label_cols = list(set(object_cols) - set(good_label_cols))

from sklearn.preprocessing import OneHotEncoder
label_X_train = X_train.copy()
label_X_valid = X_valid.copy()
OH_encoder = OneHotEncoder(handle_unknown='ignore', sparse=False)
OH_col_train = pd.DataFrame(OH_encoder.fit_transform(X_train[object_cols]))
OH_col_valid = pd.DataFrame(OH_encoder.fit_transform(X_valid[object_cols]))
OH_col_train.index = X_train.index
OH_col_valid.index = X_valid.index
num_X_train = X_train.drop(object_cols, axis=1)
num_X_valid = X_valid.drop(object_cols, axis=1)
OH_X_train = pd.concat([num_X_train, OH_col_train], axis=1)
OH_X_valid = pd.concat([num_X_valid, OH_col_valid], axis=1)

print('MAE from OHencoder :')
print(score_dataset(OH_X_train, OH_X_valid, y_train, y_valid))