text_html_output_1.png
from sklearn.impute import SimpleImputer

import matplotlib.pyplot as plt

import numpy as np

import pandas as pd

import seaborn as sns

import time

train = pd.read_csv('../input/tabular-playground-series-sep-2021/train.csv', index_col='id')
test = pd.read_csv('../input/tabular-playground-series-sep-2021/test.csv', index_col='id')
train_df = pd.read_csv('../input/tabular-playground-series-sep-2021/train.csv', index_col='id')

training_missing_val_count_by_column = train.isnull().values.sum()
test_missing_val_count_by_column = test.isnull().values.sum()

pd.set_option('display.max_rows', None)
train.describe().T

train2 = train.dropna(axis='rows')
train3 = train.dropna(axis='columns')

training_missing_val_count_by_column = train.isnull().sum()

imputer = SimpleImputer(strategy='mean')
train_imputed = pd.DataFrame(imputer.fit_transform(train))
train_imputed.columns = train.columns
train = train_imputed

corr = train.corr()
mask = np.triu(np.ones_like(corr, dtype=bool))

corr_matrix = train.corr().abs()
high_corr = np.where(corr_matrix > 0.02)
high_corr = [(corr_matrix.columns[x], corr_matrix.columns[y]) for x, y in zip(*high_corr) if x != y and x < y]
print('high correlation', high_corr)
featuresofinterest = ['f6', 'f15', 'f32', 'f34', 'f36', 'f45', 'f46', 'f51', 'f57', 'f86', 'f90', 'f97', 'f111']
print('\n', time.time())