text_plain_output_1.png
import matplotlib.pyplot as plt

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

train = pd.read_csv('../input/train.csv')
holdout = pd.read_csv('../input/test.csv')
train.isnull().sum()

gender_pivot = train.pivot_table(index='Sex', values='Survived')
class_pivot = train.pivot_table(index='Pclass', values='Survived')
family_cols = ['SibSp', 'Parch', 'Survived']
family = train[family_cols].copy()
family['familysize'] = family[['SibSp', 'Parch']].sum(axis=1)
familySize = family[['SibSp', 'Parch']].sum(axis=1)
family['isalone'] = np.where(familySize >= 1, 1, 0)
family_pivot = family.pivot_table(index='familysize', values='Survived')
isalone_pivot = family.pivot_table(index='isalone', values='Survived')

train['Fare'] = train['Fare'].fillna(train['Fare'].mean())
train['Embarked'] = train['Embarked'].fillna('S')
holdout['Fare'] = holdout['Fare'].fillna(train['Fare'].mean())
holdout['Embarked'] = holdout['Embarked'].fillna('S')
train['Age'] = train['Age'].fillna(-0.5)
holdout['Age'] = holdout['Age'].fillna(-0.5)

cuts = [-1, 0, 5, 12, 18, 35, 60, 100]
labels = ['Missing', 'Infant', 'Child', 'Teenager', 'Young Adult', 'Adult', 'Senior']
train['Age_categories'] = pd.cut(train['Age'], cuts, labels=labels)
holdout['Age_categories'] = pd.cut(holdout['Age'], cuts, labels=labels)
fare_cuts = [-1, 12, 50, 100, 1000]
fare_labels = ['0-12', '12-50', '50-100', '100+']
train['Fare_categories'] = pd.cut(train['Fare'], fare_cuts, labels=fare_labels)
holdout['Fare_categories'] = pd.cut(holdout['Fare'], fare_cuts, labels=fare_labels)
train['Cabin_type'] = train['Cabin'].str[0]
train['Cabin_type'] = train['Cabin_type'].fillna('Unknown')
train = train.drop('Cabin', axis=1)
holdout['Cabin_type'] = holdout['Cabin'].str[0]
holdout['Cabin_type'] = holdout['Cabin_type'].fillna('Unknown')
holdout = holdout.drop('Cabin', axis=1)
titles = {'Mr': 'Mr', 'Mme': 'Mrs', 'Ms': 'Mrs', 'Mrs': 'Mrs', 'Master': 'Master', 'Mlle': 'Miss', 'Miss': 'Miss', 'Capt': 'Officer', 'Col': 'Officer', 'Major': 'Officer', 'Dr': 'Officer', 'Rev': 'Officer', 'Jonkheer': 'Royalty', 'Don': 'Royalty', 'Sir': 'Royalty', 'Countess': 'Royalty', 'Dona': 'Royalty', 'Lady': 'Royalty'}
train_titles = train['Name'].str.extract(' ([A-Za-z]+)\\.', expand=False)
train['Title'] = train_titles.map(titles)
holdout_titles = holdout['Name'].str.extract(' ([A-Za-z]+)\\.', expand=False)
holdout['Title'] = holdout_titles.map(titles)
familySize_train = train[['SibSp', 'Parch']].sum(axis=1)
train['isalone'] = np.where(familySize_train >= 1, 1, 0)
familySize_holdout = holdout[['SibSp', 'Parch']].sum(axis=1)
holdout['isalone'] = np.where(familySize_holdout >= 1, 1, 0)
def get_dummies(df, column_name):
    dummies = pd.get_dummies(df[column_name], prefix=column_name)
    df = pd.concat([df, dummies], axis=1)
    return df
columnNames = ['Age_categories', 'Pclass', 'Sex', 'Fare_categories', 'Title', 'Cabin_type', 'Embarked']
for column in columnNames:
    dummies_train = pd.get_dummies(train[column], prefix=column)
    train = pd.concat([train, dummies_train], axis=1)
    dummies_holdout = pd.get_dummies(holdout[column], prefix=column)
    holdout = pd.concat([holdout, dummies_holdout], axis=1)
train.head(5)
holdout.head(5)
print(holdout.columns)