text_html_output_1.png
import matplotlib.pyplot as plt

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import seaborn as sns

import typing

df = pd.read_csv('../input/titanic/train.csv')
test_df = pd.read_csv('../input/titanic/test.csv')

def detect_NaNs(df_temp: pd.DataFrame, name='', silent: bool=False, plot: bool=True):
    """
    Detect NaNs in a provided dataframe and return the columns that NaNs were detected in     
    Parameters
    ----------
    df_temp : pd.DataFrame
        Dataframe to detect NaN values in
    name : str
        Name of the dataframe which helps give a more descriptive read out
    silent : bool
        Whether the print statements should fire
    plot : bool
        Whether to return a plot of the counts of NaNs in the data
    Returns
    -------
    typing.List
        List of columns in the provided dataframe that contain NaN values
    """
    count_nulls = df_temp.isnull().sum().sum()
    columns_with_NaNs = []
    if count_nulls > 0:
        for col in df_temp.columns:
            if df_temp[col].isnull().sum().sum() > 0:
                columns_with_NaNs.append(col)
    return columns_with_NaNs

nan_columns = detect_NaNs(df, 'Training Data')

def fill_nans_create_columns(df_temp: pd.DataFrame, columns: typing.List, value: float=0):
    """
    Fill NaN of provided columns and create columns to signify they weren't there.
    Parameters
    ----------
    df_temp : pd.DataFrame
        Dataframe to modify
    columns : typing.List
        Columns of the provided dataframe to modify
    value : float
        Value to replace the NaN values with
    Returns
    -------
    pd.DataFrame
        Modified Dataframe with NaNs filled and new columns signifying the rows that contained NaNs
    """
    for col in columns:
        df_temp[col + '_was_null'] = df_temp[col].isnull().astype(int)
        df_temp[col] = df_temp[col].fillna(value)
    return df_temp

df = fill_nans_create_columns(df, nan_columns, 0)
test_df = fill_nans_create_columns(test_df, nan_columns, 0)

def detect_duplicates(df_temp: pd.DataFrame):
    """
    Detect duplicates in data and return the columns in which duplicates where detected.
    Parameters
    ----------
    df_temp : pd.DataFrame
        Dataframe to detect duplicates in
    """
    cols_to_use = []
    id_cols = []
    for col in df_temp.columns:
        if len(df_temp[col].unique()) != len(df_temp[col]):
            cols_to_use.append(col)
        else:
            id_cols.append(col)
    df_temp = df_temp.copy()[cols_to_use]
    count_dupes = df_temp.duplicated().sum()

detect_duplicates(df)