text_plain_output_1.png
from sklearn.metrics import confusion_matrix,recall_score, accuracy_score , classification_report, balanced_accuracy_score

from sklearn.preprocessing import OneHotEncoder

from sklearn.tree import DecisionTreeClassifier

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

data = pd.read_csv('/kaggle/input/income/train.csv')
data.shape

all_columns = list(data.columns)
all_columns

data.isna().sum()

cat_columns = ['workclass', 'education', 'marital-status', 'occupation', 'relationship', 'race', 'gender', 'native-country']

def event_rate_analysis(column, target):
    temp = (data.groupby(column)[target].sum() / data[target].sum()).to_frame().reset_index()
    temp['Volume'] = (data.groupby(column)[target].count() / data[target].count()).to_list()
    temp = temp.sort_values(target, ascending=False)

from sklearn.preprocessing import OneHotEncoder
ohe = OneHotEncoder(sparse=False)
data_ohe = ohe.fit_transform(data[['race', 'gender', 'relationship', 'occupation', 'workclass', 'native-country', 'marital-status', 'education']])

cat_columns = []
for cat in ohe.categories_:
    cat_columns.extend(cat)

cat_df = pd.DataFrame(data_ohe, columns=cat_columns)

final_df = pd.concat([data.drop(['race', 'gender', 'relationship', 'occupation', 'workclass', 'native-country', 'marital-status', 'education'], axis=1), cat_df], axis=1)
final_df.shape

X = final_df.drop('income_>50K', axis=1)
y = final_df['income_>50K']

y_train.value_counts(normalize=True)

from sklearn.tree import DecisionTreeClassifier
clf = DecisionTreeClassifier(max_depth=20, criterion='entropy', random_state=42)
clf.fit(X_train, y_train)

from sklearn.metrics import confusion_matrix, recall_score, accuracy_score, classification_report, balanced_accuracy_score
def print_performance(text, y_pred):
    pass

y_pred = clf.predict(X_test)

dt1 = DecisionTreeClassifier(max_depth=20, criterion='entropy', random_state=42)
dt2 = DecisionTreeClassifier(max_depth=20, criterion='entropy', random_state=42)
dt3 = DecisionTreeClassifier(max_depth=20, criterion='entropy', random_state=42)
dt4 = DecisionTreeClassifier(max_depth=20, criterion='entropy', random_state=42)

data1 = X_train[X_train['Husband'] == 1]
data2 = X_train[X_train['Married-civ-spouse'] == 1]
data3 = X_train.loc[(X_train['Sales'] == 1) | (X_train['Prof-specialty'] == 1) | (X_train['Exec-managerial'] == 1)]
data4 = X_train.loc[~((X_train['Husband'] == 1) | (X_train['Married-civ-spouse'] == 1) | ((X_train['Sales'] == 1) | (X_train['Prof-specialty'] == 1) | (X_train['Exec-managerial'] == 1)))]

label1 = y.iloc[list(data1.index)]
label2 = y.iloc[list(data2.index)]
label3 = y.iloc[list(data3.index)]
label4 = y.iloc[list(data4.index)]

dt1.fit(data1, label1)
dt2.fit(data2, label2)
dt3.fit(data3, label3)
dt4.fit(data4, label4)

y_pred = []
for index, row in X_test.iterrows():
    if row['Husband'] == 1:
        pred = dt1.predict([row])[0]
        if pred == 1:
            y_pred.append(pred)
            continue
    if row['Married-civ-spouse'] == 1:
        pred = dt2.predict([row])[0]
        if pred == 1:
            y_pred.append(pred)
            continue
    if row['Sales'] == 1 or row['Exec-managerial'] == 1 or row['Prof-specialty'] == 1:
        pred = dt2.predict([row])[0]
        if pred == 1:
            y_pred.append(pred)
            continue
    pred = dt3.predict([row])[0]
    if pred == 1:
        y_pred.append(pred)
    else:
        y_pred.append(0)

print_performance('Segmented Decision Tree', y_pred)