text_html_output_1.png
import numpy as np

import pandas as pd

titanic_train = pd.read_csv('../input/train.csv', index_col='PassengerId')
titanic_test = pd.read_csv('../input/test.csv', index_col='PassengerId')

combined = pd.concat((titanic_train, titanic_test), axis=0)

ages_mean = combined.pivot_table('Age', index=['Title'], columns=['Sex', 'Pclass'], aggfunc='mean')
ages_mean

ages_std = combined.pivot_table('Age', index=['Title'], columns=['Sex', 'Pclass'], aggfunc='std')
ages_std

def age_guesser(person):
    gender = person['Sex']
    mean_age = ages_mean[gender].loc[person['Title'], person['Pclass']]
    std = ages_std[gender].loc[person['Title'], person['Pclass']]
    persons_age = np.random.randint(mean_age - std, mean_age + std)
    return persons_age
unknown_age = combined['Age'].isnull()
people_w_unknown_age = combined.loc[unknown_age, ['Age', 'Title', 'Sex', 'Pclass']]
people_w_unknown_age['Age'] = people_w_unknown_age.apply(age_guesser, axis=1)
known_age = combined['Age'].notnull()
people_w_known_age = combined.loc[known_age, ['Age', 'Title', 'Sex', 'Pclass']]
combined['new_age'] = pd.concat((people_w_known_age['Age'], people_w_unknown_age['Age']))

combined['Embarked'].fillna('S', inplace=True)
combined['Fare'].fillna(value=combined['Fare'].mean(), inplace=True)
combined['kid'] = 0
combined.loc[combined.new_age <= 12, 'kid'] = 1
combined['parent'] = 0
combined.loc[(combined.Parch > 0) & (combined.new_age >= 18), 'parent'] = 1
combined['child'] = 0
combined.loc[(combined.Parch > 0) & (combined.new_age < 18), 'child'] = 1
combined['family'] = combined['SibSp'] + combined['Parch']
combined.loc[combined.family > 0, 'family'] = 1
combined['male'] = (~combined['Sex'].str.contains('fe')).astype(int)

not_needed = ['Age', 'Cabin', 'Name', 'Sex', 'Ticket', 'Parch', 'SibSp']
combined.drop(not_needed, axis=1, inplace=True)

categorical = ['Embarked', 'Title', 'Pclass']
for column in categorical:
    dummy = pd.get_dummies(combined[column], prefix=column).astype(int)
    combined = combined.join(dummy)
    combined.drop(column, axis=1, inplace=True)
combined.head()