text_plain_output_1.png image_output_1.png
import matplotlib.pyplot as plt

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import scipy.stats as ss

import seaborn as sns

df = pd.read_csv('../input/travel insurance.csv')
df1 = df

df.fillna('Not Specified', inplace=True)

df.isnull().sum()

df_numerical = df._get_numeric_data()

df.loc[df['Duration'] < 0, 'Duration'] = 49.317

df.loc[df['Net Sales'] == 0.0, 'Commision (in value)'] = 0

def cramers_v(x, y):
    confusion_matrix = pd.crosstab(x, y)
    chi2 = ss.chi2_contingency(confusion_matrix)[0]
    n = confusion_matrix.sum().sum()
    phi2 = chi2 / n
    r, k = confusion_matrix.shape
    phi2corr = max(0, phi2 - (k - 1) * (r - 1) / (n - 1))
    rcorr = r - (r - 1) ** 2 / (n - 1)
    kcorr = k - (k - 1) ** 2 / (n - 1)
    return np.sqrt(phi2corr / min(kcorr - 1, rcorr - 1))

categorical = ['Agency', 'Agency Type', 'Distribution Channel', 'Product Name', 'Destination', 'Gender', 'Claim']
cramers = pd.DataFrame({i: [cramers_v(df[i], df[j]) for j in categorical] for i in categorical})
cramers['column'] = [i for i in categorical if i not in ['memberid']]
cramers.set_index('column', inplace=True)

from scipy.stats import chi2_contingency
class ChiSquare:
    def __init__(self, df):
        self.df = df
        self.p = None
        self.chi2 = None
        self.dof = None
        self.dfObserved = None
        self.dfExpected = None
    def _print_chisquare_result(self, colX, alpha):
        result = ''
        if self.p < alpha:
            result = '{0} is IMPORTANT for Prediction'.format(colX)
        else:
            result = '{0} is NOT an important predictor. (Discard {0} from model)'.format(colX)
    def TestIndependence(self, colX, colY, alpha=0.05):
        X = self.df[colX].astype(str)
        Y = self.df[colY].astype(str)
        self.dfObserved = pd.crosstab(Y, X)
        chi2, p, dof, expected = ss.chi2_contingency(self.dfObserved.values)
        self.p = p
        self.chi2 = chi2
        self.dof = dof
        self.dfExpected = pd.DataFrame(expected, columns=self.dfObserved.columns, index=self.dfObserved.index)

X = df.drop(['Claim'], axis=1)
ct = ChiSquare(df)
for c in X.columns:
    ct.TestIndependence(c, 'Claim')

df.drop(columns=['Distribution Channel', 'Agency Type'], axis=1, inplace=True)

df.info()