text_plain_output_1.png
import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

all_train_data = pd.read_csv('/kaggle/input/titanic/train.csv')
val_data = all_train_data[:int(0.2 * len(all_train_data))]
train_data = all_train_data[int(0.2 * len(all_train_data)):].reset_index()
test_data = pd.read_csv('/kaggle/input/titanic/test.csv')

def todummy(x, categories=None):
    X = np.zeros((len(x), len(categories) + 1), dtype=int)
    for i, cat in enumerate(categories):
        X[:, i] = x == cat
    X[:, -1] = 1 - X[:, :-1].sum(axis=1)
    return X

def majorcategories(x: pd.Series, cutoff_categorynum=10, cutoff_percentile=0.95, verbose=True) -> list:
    xcopy = x[~x.isnull()]
    nullpdf = sum(x.isnull()) / len(x)
    category = np.unique(xcopy)
    valuepdf = {}
    for cat in category:
        valuepdf[cat] = sum(xcopy == cat) / len(x)
    sortvaluepdf = sorted(valuepdf.items(), key=lambda x: x[1], reverse=True)
    cdf = np.cumsum([x[1] for x in sortvaluepdf])
    if len(sortvaluepdf) <= cutoff_categorynum:
        return [x[0] for x in sortvaluepdf]
    else:
        majorcategory = []
        i = 0
        while i < cutoff_categorynum and i < len(sortvaluepdf):
            p = cdf[i]
            if p < cutoff_percentile:
                majorcategory.append(sortvaluepdf[i][0])
            else:
                break
            i += 1
        return majorcategory

trained_prefix = all_train_data['Name'].apply(lambda x: x.split(', ')[-1].split('. ')[0])
trained_prefix_category = majorcategories(trained_prefix)
trained_ticketstr = []
for i in range(len(all_train_data)):
    ticket = all_train_data.loc[i, 'Ticket']
    if ' ' in ticket:
        rev_ticket = ticket[::-1]
        num_ticket = rev_ticket[:rev_ticket.index(' ')]
        string = ticket.replace(num_ticket[::-1], '')
    elif any([x in [str(j) for j in range(10)] for x in ticket]):
        string = ''
    else:
        string = ticket
    string = ''.join([x for x in string if x not in ['.', '/', ' ']])
    if len(string) > 0:
        trained_ticketstr.append(string)
trained_ticketstr_category = majorcategories(pd.Series(trained_ticketstr), cutoff_categorynum=100)
trained_ticketstr_category += ['NA']
trained_cabinstr = []
for i in range(len(all_train_data)):
    cabin = all_train_data.loc[i, 'Cabin']
    if type(cabin) is str:
        cabin = ''.join([x for x in cabin if x not in [str(j) for j in range(10)]])
        cabin_split = np.array(cabin.split(' '))
        _, idx = np.unique(cabin_split, return_index=True)
        trained_cabinstr.append(' '.join(cabin_split[np.sort(idx)]))
trained_cabinstr_category = majorcategories(pd.Series(trained_cabinstr), cutoff_categorynum=100)
trained_cabinstr_category += ['NA']
trained_embarked_category = majorcategories(all_train_data['Embarked'])

def preprocessing(data):
    X = {}
    if 'Survived' in data:
        y = data['Survived']
    else:
        y = None
    X['Pclass'] = data['Pclass']
    prefix = data['Name'].apply(lambda x: x.split(', ')[-1].split('. ')[0])
    dummy = todummy(prefix, trained_prefix_category)
    for i, cat in enumerate(trained_prefix_category):
        X['Prefix_' + cat] = dummy[:, i]
    dummy = todummy(data['Sex'], ['male'])
    for i, cat in enumerate(['male', 'female']):
        X['Sex_' + cat] = dummy[:, i]
    X['Age'] = [0.01 if np.isnan(x) else x for x in data['Age']]
    X['Age0'] = [x > 0 and x <= 10 for x in X['Age']]
    X['Age1'] = [x > 10 and x <= 20 for x in X['Age']]
    X['Age2'] = [x > 20 and x <= 40 for x in X['Age']]
    X['Age3'] = [x > 40 and x <= 60 for x in X['Age']]
    X['Age4'] = [x > 60 for x in X['Age']]
    X['Age'] = np.log(X['Age'])
    X['SibSp'] = data['SibSp']
    X['Parch'] = data['Parch']
    X['Ticketnum'] = []
    strings = []
    for i in range(len(data)):
        ticket = train_data.loc[i, 'Ticket']
        if ' ' in ticket:
            rev_ticket = ticket[::-1]
            num_ticket = rev_ticket[:rev_ticket.index(' ')]
            string = ticket.replace(num_ticket[::-1], '')
            string = ''.join([x for x in string if x not in ['.', '/', ' ']])
            strings.append(string)
            X['Ticketnum'].append(int(num_ticket[::-1]))
        elif any([x in [str(j) for j in range(10)] for x in ticket]):
            strings.append('NA')
            X['Ticketnum'].append(np.log(int(ticket)))
        else:
            strings.append(ticket)
            X['Ticketnum'].append(0)
    dummy = todummy(strings, trained_ticketstr_category)
    for i, cat in enumerate(trained_ticketstr_category):
        X['Ticketstr_' + cat] = dummy[:, i]
    X['Fare'] = data['Fare'].copy()
    X['Fare'].at[data['Fare'].isnull()] = 0.0
    X['Cabinnum'] = []
    X['Travellernum'] = []
    strings = []
    for i in range(len(data)):
        cabin = data.loc[i, 'Cabin']
        if type(cabin) is float:
            strings.append('NA')
            if np.isnan(data.loc[i, 'Cabin']):
                X['Cabinnum'].append(0.0)
            else:
                X['Cabinnum'].append(float(data.loc[i, 'Cabin']))
            X['Travellernum'].append(1)
        else:
            cabinstr = ''.join([x for x in cabin if x not in [str(j) for j in range(10)]])
            cabin_split = np.array(cabinstr.split(' '))
            _, idx = np.unique(cabin_split, return_index=True)
            string = ' '.join(cabin_split[np.sort(idx)])
            if len(string) > 0:
                strings.append(string)
            else:
                strings.append('NA')
            cabinnum = cabin.replace(string, '')
            if len(cabinnum) > 0:
                cabinnum = [int(x) for x in cabin.replace(string, '').split(' ')]
            else:
                cabinnum = [0]
            X['Travellernum'].append(len(cabinnum))
            X['Cabinnum'].append(np.mean(cabinnum))
    dummy = todummy(strings, trained_cabinstr_category)
    for i, cat in enumerate(trained_cabinstr_category):
        X['Cabinstr_' + cat] = dummy[:, i]
    dummy = todummy(data['Embarked'], trained_embarked_category)
    for i, cat in enumerate(trained_embarked_category):
        X['Embarked_' + cat] = dummy[:, i]
    return (pd.DataFrame(data=X), y)

X_train, y_train = preprocessing(train_data)
X_val, y_val = preprocessing(val_data)
X_test, _ = preprocessing(test_data)
X_train.head()