text_plain_output_1.png
import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

train = pd.read_csv('../input/titanic/train.csv')
test = pd.read_csv('../input/titanic/test.csv')

train.describe().columns

df_num = train[['Age', 'SibSp', 'Parch', 'Fare']]
df_cat = train[['Survived', 'Pclass', 'Sex', 'Ticket', 'Cabin', 'Embarked']]

df_cat.Cabin
train['cabin_multiple'] = train.Cabin.apply(lambda x: 0 if pd.isna(x) else len(x.split(' ')))
train['cabin_multiple'].value_counts()

pd.pivot_table(train, index='Survived', columns='cabin_multiple', values='Ticket', aggfunc='count')

train['cabin_adv'] = train.Cabin.apply(lambda x: str(x)[0])

train['numeric_ticket'] = train.Ticket.apply(lambda x: 1 if x.isnumeric() else 0)
train['ticket_letters'] = train.Ticket.apply(lambda x: ''.join(x.split(' ')[:-1]).replace('.', '').replace('/', '').lower() if len(x.split(' ')[:-1]) > 0 else 0)
train['numeric_ticket'].value_counts()

pd.set_option('max_rows', None)
train['ticket_letters'].value_counts()

pd.pivot_table(train, index='Survived', columns='numeric_ticket', values='Ticket', aggfunc='count')

pd.pivot_table(train, index='Survived', columns='ticket_letters', values='Ticket', aggfunc='count')

train['name_title'] = train.Name.apply(lambda x: x.split(',')[1].split('.')[0].strip())
train['name_title'].value_counts()

data['cabin_multiple'] = data.Cabin.apply(lambda x: 0 if pd.isna(x) else len(x.split(' ')))
data['cabin_adv'] = data.Cabin.apply(lambda x: str(x)[0])
data['numeric_ticket'] = data.Ticket.apply(lambda x: 1 if x.isnumeric() else 0)
data['ticket_letters'] = data.Ticket.apply(lambda x: ''.join(x.split(' ')[:-1]).replace('.', '').replace('/', '').lower() if len(x.split(' ')[:-1]) > 0 else 0)
data['name_title'] = data.Name.apply(lambda x: x.split(',')[1].split('.')[0].strip())
data.dropna(subset=['Embarked'], inplace=True)
original_age = data.Age
original_fare = data.Fare
data['norm_sibsp'] = np.log(data.SibSp + 1)
data['norm_sibsp'].hist()
data['norm_fare'] = np.log(data.Fare + 1)
data['norm_fare'].hist()
data.Pclass = data.Pclass.astype(str)
all_dummies = pd.get_dummies(data[['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'norm_fare', 'Embarked', 'cabin_adv', 'cabin_multiple', 'numeric_ticket', 'name_title', 'train_test']])
X_train = all_dummies[all_dummies.train_test == 1].drop(['train_test'], axis=1)
X_test = all_dummies[all_dummies.train_test == 0].drop(['train_test'], axis=1)
y_train = data[data.train_test == 1].Survived
y_train.shape