text_plain_output_1.png image_output_1.png
import matplotlib as mpl

import matplotlib.pyplot as plt

import numpy as np

import pandas as pd

import seaborn as sns

import sklearn as sk

import tensorflow as tf

import xgboost as xgb

X_full = pd.read_csv('../input/30-days-of-ml/train.csv', index_col=0)
X_test_full = pd.read_csv('../input/30-days-of-ml/test.csv', index_col=0)

X_full.isnull().sum()

categorical_features = [cname for cname in X_full.columns if X_full[cname].nunique() <= 15 and X_full[cname].dtype == 'object']
numeric_features = [cname for cname in X_full.columns if X_full[cname].dtype in ['int64', 'float64']]
my_features = categorical_features + numeric_features
numeric_features.remove('target')

## Correlations
correlations = X_full[my_features].corr()
f, ax = plt.subplots(figsize=(12, 12))
sns.heatmap(correlations, square=True, cbar=True, annot=True, vmax=.9);

## Box Plot for Outliers
fig = plt.figure(figsize=(18,6))
sns.boxplot(data=X_full[numeric_features], orient="h", palette="Set2");
plt.xticks(fontsize= 14)
plt.title('Box plot of numerical columns', fontsize=16);

plt.xticks(fontsize=14)

from scipy import stats
def treatoutliers(df=None, columns=None, factor=1.5, method='IQR', treatment='cap'):
    for column in columns:
        if method == 'STD':
            permissable_std = factor * df[column].std()
            col_mean = df[column].mean()
            floor, ceil = (col_mean - permissable_std, col_mean + permissable_std)
        elif method == 'IQR':
            Q1 = df[column].quantile(0.25)
            Q3 = df[column].quantile(0.75)
            IQR = Q3 - Q1
            floor, ceil = (Q1 - factor * IQR, Q3 + factor * IQR)
        if treatment == 'remove':
            df = df[(df[column] >= floor) & (df[column] <= ceil)]
        elif treatment == 'cap':
            df[column] = df[column].clip(floor, ceil)
    return df
for colName in [['target', 'cont0', 'cont6', 'cont8']]:
    X_full = treatoutliers(df=X_full, columns=colName, treatment='cap')

plt.xticks(fontsize=14)

fig = plt.figure(figsize=(18, 6))
sns.boxplot(data=X_full[numeric_features], orient='h', palette='Set2')
plt.xticks(fontsize=14)
plt.title('Box plot of numerical columns after handling Outliers', fontsize=16)