text_plain_output_1.png image_output_1.png
import matplotlib.pyplot as plt

import pandas as pd

import seaborn as sns

train = pd.read_csv('/kaggle/input/playground-series-s3e14/train.csv', index_col='id')
test = pd.read_csv('/kaggle/input/playground-series-s3e14/test.csv', index_col='id')
submission = pd.read_csv('/kaggle/input/playground-series-s3e14/sample_submission.csv')
origin = pd.read_csv('/kaggle/input/wild-blueberry-yield-prediction-dataset/WildBlueberryPollinationSimulationData.csv').drop('Row#', axis=1)

def info(train):
    pass

train.describe().T

cont_col = [i for (i, j) in zip(test.columns, test.dtypes) if j in ["int", "float"]]
fig, axes = plt.subplots(4, 4, figsize=(30, 20))
for i, ax in enumerate(axes.flat):
    sns.kdeplot(
        ax=ax, data=train, x=cont_col[i], color="#F8766D", label="Train", fill=True
    )
    sns.kdeplot(
        ax=ax, data=test, x=cont_col[i], color="#00BFC4", label="Test", fill=True
    )
    ax.set_title(f"{cont_col[i]} distribution")
fig.tight_layout()
plt.legend()

fig, axes = plt.subplots(4, 4, figsize=(30, 20))
for i, ax in enumerate(axes.flat):
    sns.kdeplot(
        ax=ax, data=train, x=cont_col[i], color="#F8766D", label="Train", fill=True
    )
    sns.kdeplot(
        ax=ax, data=origin, x=cont_col[i], color="#00BFC4", label="Original", fill=True
    )
    ax.set_title(f"{cont_col[i]} distribution")
fig.tight_layout()
plt.legend()

train = train.drop(['MinOfUpperTRange', 'AverageOfUpperTRange', 'MaxOfLowerTRange', 'MinOfLowerTRange', 'AverageOfLowerTRange', 'AverageRainingDays'], axis=1)
test = test.drop(['MinOfUpperTRange', 'AverageOfUpperTRange', 'MaxOfLowerTRange', 'MinOfLowerTRange', 'AverageOfLowerTRange', 'AverageRainingDays'], axis=1)

cont_col = [i for (i, j) in zip(test.columns, test.dtypes) if j in ["int", "float"]]
fig, axes = plt.subplots(3, 3, figsize=(30, 20))
for i, ax in enumerate(axes.flat):
    sns.kdeplot(
        ax=ax, data=train, x=cont_col[i], color="#F8766D", label="Train", fill=True
    )
    sns.kdeplot(
        ax=ax, data=test, x=cont_col[i], color="#00BFC4", label="Test", fill=True
    )
    ax.set_title(f"{cont_col[i]} distribution")
fig.tight_layout()
plt.legend()

cat_feat = list()
for col in train.columns:
    if train[col].nunique() < 20:
        cat_feat.append(col)

def outlier_removal(dataframe, features):
    for feature_name in features:
        Q1 = dataframe[feature_name].quantile(0.25)
        Q3 = dataframe[feature_name].quantile(0.75)
        IQR = Q3 - Q1
        dataframe = dataframe[(dataframe[feature_name] >= Q1 - 1.5 * IQR) & (dataframe[feature_name] <= Q3 + 1.5 * IQR)]
    return dataframe

features = train.columns
train = outlier_removal(train, features)
for col in cat_feat:
    q1 = test[col].quantile(0.25)
    q3 = test[col].quantile(0.75)
    iqr = q3 - q1
    lower_bound = q1 - 1.5 * iqr
    upper_bound = q3 + 1.5 * iqr
    mean_value = test[col].mean()
    test.loc[(test[col] < lower_bound) | (test[col] > upper_bound), col] = mean_value