image_output_1.png
import matplotlib as mpl

import numpy as np

import pandas as pd

import seaborn as sns

import sklearn as sk

import tensorflow as tf

import xgboost as xgb

X_full = pd.read_csv('../input/30-days-of-ml/train.csv', index_col=0)
X_test_full = pd.read_csv('../input/30-days-of-ml/test.csv', index_col=0)

X_full.isnull().sum()

categorical_features = [cname for cname in X_full.columns if X_full[cname].nunique() <= 15 and X_full[cname].dtype == 'object']
numeric_features = [cname for cname in X_full.columns if X_full[cname].dtype in ['int64', 'float64']]
my_features = categorical_features + numeric_features
numeric_features.remove('target')

from scipy import stats
def treatoutliers(df=None, columns=None, factor=1.5, method='IQR', treatment='cap'):
    for column in columns:
        if method == 'STD':
            permissable_std = factor * df[column].std()
            col_mean = df[column].mean()
            floor, ceil = (col_mean - permissable_std, col_mean + permissable_std)
        elif method == 'IQR':
            Q1 = df[column].quantile(0.25)
            Q3 = df[column].quantile(0.75)
            IQR = Q3 - Q1
            floor, ceil = (Q1 - factor * IQR, Q3 + factor * IQR)
        if treatment == 'remove':
            print(treatment, column)
            df = df[(df[column] >= floor) & (df[column] <= ceil)]
        elif treatment == 'cap':
            print(treatment, column)
            df[column] = df[column].clip(floor, ceil)
    return df
for colName in [['target', 'cont0', 'cont6', 'cont8']]:
    X_full = treatoutliers(df=X_full, columns=colName, treatment='cap')
X_full.info()