image_output_1.png
from sklearn.model_selection import train_test_split

import matplotlib.pyplot as plt

import numpy as np

import pandas as pd

import seaborn as sns

df = pd.read_csv('/kaggle/input/playground-series-s3e14/train.csv')

df.isna().sum().sum()

def plot_features(df):
    fig, axs = plt.subplots(figsize=(16, 10), ncols=3, nrows=6)
    for count, col in enumerate(df.columns):

        sns.boxplot(data=df, x=col, ax=axs[count // 3][count % 3])
    plt.tight_layout()
    plt.show()

plot_features(df)

def plot_corr(df):
    plt.figure(figsize=(16, 10))
    corr = df.drop(columns=['id', 'yield']).corr()
    cmap = sns.diverging_palette(230, 20, as_cmap=True)

    sns.heatmap(corr,cmap=cmap, center=0, annot=True,
            square=True, linewidths=.5, cbar_kws={"shrink": .5})
    plt.title('Correlation matrix of features')
    plt.show()
    
    return corr

corr = plot_corr(df)

upper = corr.where(np.triu(np.ones(corr.shape), k=1).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column] > 0.9)]
df.drop(to_drop, axis=1, inplace=True)

X = df.drop(columns=['id', 'yield'])
y = df['yield']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
(len(X_train), len(X_test))