image_output_1.png
from collections import Counter

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

train = pd.read_csv('../input/train.csv')
test = pd.read_csv('../input/test.csv')
IDtest = test['PassengerId']

def detect_outlier(df, n, features):
    outlier_indices = []
    for col in features:
        Q1 = np.percentile(df[col].dropna(), 25)
        Q3 = np.percentile(df[col].dropna(), 75)
        IQR = Q3 - Q1
        outlier_step = 1.5 * IQR
        outliers_list_col = df[(df[col] < Q1 - outlier_step) | (df[col] > Q3 + outlier_step)].index
        outlier_indices.extend(outliers_list_col)
    outlier_indices = Counter(outlier_indices)
    multiple_outliers = list((k for k, v in outlier_indices.items() if v > n))
    return multiple_outliers

outliers_to_drop = detect_outlier(train, 2, ['Age', 'SibSp', 'Parch', 'Fare'])

train.loc[outliers_to_drop]

train = train.drop(outliers_to_drop, axis=0).reset_index(drop=True)

train_len = len(train)
dataset = pd.concat(objs=[train, test], axis=0, sort=False).reset_index(drop=True)

dataset = dataset.fillna(np.nan)

dataset.isnull().sum()

dataset['Fare'].isnull().sum()