application_vnd.jupyter.stderr_output_1.png
from IPython.display import display

import numpy as np

import pandas as pd

import re

def extract_maritial(name):
    """ extract the person's title, and bin it to Mr. Miss. and Mrs.
    assuming a Miss, Lady or Countess has more change to survive than a regular married woman."""
    re_maritial = ' ([A-Za-z]+\\.) '
    found = re.findall(re_maritial, name)[0]
    replace = [['Dr.', 'Sir.'], ['Rev.', 'Sir.'], ['Major.', 'Officer.'], ['Mlle.', 'Miss.'], ['Col.', 'Officer.'], ['Master.', 'Sir.'], ['Jonkheer.', 'Sir.'], ['Sir.', 'Sir.'], ['Don.', 'Sir.'], ['Countess.', 'High.'], ['Capt.', 'Officer.'], ['Ms.', 'High.'], ['Mme.', 'High.'], ['Dona.', 'High.'], ['Lady.', 'High.']]
    for i in range(0, len(replace)):
        if found == replace[i][0]:
            found = replace[i][1]
            break
    return found
def father(sex, age, parch):
    if sex == 'male' and age > 16 and (parch > 0):
        return 1
    else:
        return 0
def mother(sex, age, parch):
    if sex == 'female' and age > 16 and (parch > 0):
        return 1
    else:
        return 0
def parent(sex, age, parch):
    if mother(sex, age, parch) == 1 or father(sex, age, parch) == 1:
        return 1
    else:
        return 0
def extract_cabin_nr(cabin):
    """ Extracts the cabin number.  If there no number found, return NaN """
    if not pd.isnull(cabin):
        cabin = cabin.split(' ')[-1]
        re_numb = '[A-Z]([0-9]+)'
        try:
            number = int(re.findall(re_numb, cabin)[0])
            return number
        except:
            return np.nan
    else:
        return np.nan
def extract_cabin_letter(cabin):
    """ Extracts the cabin letter.  If there no letter found, return NaN """
    if not pd.isnull(cabin):
        cabin = cabin.split(' ')[-1]
        re_char = '([A-Z])[0-9]+'
        try:
            character = re.findall(re_char, cabin)[0]
            return character
        except:
            return np.nan
    else:
        return np.nan
def expand_sex(sex, age):
    """ this expands male/female with kid.  Cause below 14 years old, male or female is irrelevant"""
    if age < 14:
        return 'kid'
    else:
        return sex

def feat_eng(data):
    data['Title'] = list(map(extract_maritial, data['Name']))
    data['Cabin_char'] = list(map(extract_cabin_letter, data['Cabin']))
    data['Cabin_nr'] = list(map(extract_cabin_nr, data['Cabin']))
    data['Cabin_nr_odd'] = data.Cabin_nr.apply(lambda x: np.nan if x == np.nan else x % 2)
    data['Father'] = list(map(father, data.Sex, data.Age, data.Parch))
    data['Mother'] = list(map(mother, data.Sex, data.Age, data.Parch))
    data['Parent'] = list(map(parent, data.Sex, data.Age, data.Parch))
    data['has_parents_or_kids'] = data.Parch.apply(lambda x: 1 if x > 0 else 0)
    data['FamilySize'] = data.SibSp + data.Parch
    data['Sex'] = list(map(expand_sex, data['Sex'], data['Age']))
    data['FareBin'] = pd.cut(data.Fare, bins=(-1000, 0, 8.67, 16.11, 32, 350, 1000))
    data['AgeBin'] = pd.cut(data.Age, bins=(0, 15, 25, 60, 90))
    return data

def missing(data):
    data.loc[data.Age.isnull() & (data.Title == 'Sir.'), 'Age'] = data.loc[data.Title == 'Sir.', 'Age'].median()
    data.loc[data.Age.isnull() & (data.Title == 'Officer.'), 'Age'] = data.loc[data.Title == 'Officer.', 'Age'].median()
    data.loc[data.Age.isnull() & (data.Title == 'Miss.'), 'Age'] = data.loc[data.Title == 'Miss.', 'Age'].median()
    data.loc[data.Age.isnull() & (data.Title == 'High.'), 'Age'] = data.loc[data.Title == 'High.', 'Age'].median()
    data.loc[data.Age.isnull() & (data.Title == 'Mrs.'), 'Age'] = data.loc[data.Title == 'Mrs.', 'Age'].median()
    data.loc[data.Age.isnull() & (data.Title == 'Mr.'), 'Age'] = data.loc[data.Title == 'Mr.', 'Age'].median()
    median_fare = data['Fare'].median()
    data['Fare'].fillna(value=median_fare, inplace=True)
    mode_embarked = data['Embarked'].mode()[0]
    data['Embarked'].fillna(value=mode_embarked, inplace=True)
    data['Cabin_char'].fillna(value=-9999, inplace=True)
    data['Cabin_nr'].fillna(value=-9999, inplace=True)
    data['Cabin_nr_odd'].fillna(value=-9999, inplace=True)
    data = data.drop(['Name', 'Cabin', 'Fare', 'Age', 'Ticket'], 1)
    return data

train = pd.read_csv('../input/train.csv')
display('Unaltered training set:')
display(train.head(8))
train = feat_eng(train)
display('After feature engineering:')
display(train.head(8))
train = missing(train)
display('After handling missing values:')
display(train.head(8))
train = pd.get_dummies(train, drop_first=True)
display('After handling categorical values:')
display(train.head(8))