text_plain_output_1.png
from scipy.cluster.hierarchy import dendrogram, ward

import matplotlib.pyplot as plt

import numpy as np

import pandas as pd

import seaborn as sns

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import RepeatedStratifiedKFold, StratifiedKFold, KFold
from scipy.cluster.hierarchy import dendrogram, ward
sns.set_theme(style='white', palette='viridis')
pal = sns.color_palette('viridis')
pd.set_option('display.max_rows', 100)

train = pd.read_csv('../input/playground-series-s3e14/train.csv')
test_1 = pd.read_csv('../input/playground-series-s3e14/test.csv')
orig_train = pd.read_csv('../input/wild-blueberry-yield-prediction-dataset/WildBlueberryPollinationSimulationData.csv')
train.drop('id', axis=1, inplace=True)
test = test_1.drop('id', axis=1)
orig_train.drop('Row#', axis=1, inplace=True)

def heatmap(dataset, label=None):
    corr = dataset.corr()
    mask = np.zeros_like(corr)
    mask[np.triu_indices_from(mask)] = True

def distance(data, label=''):
    distances = data.corr()
    dist_linkage = ward(distances)
    dendro = dendrogram(dist_linkage, labels=data.columns, leaf_rotation=90)

def find_duplicates(data, column, label=''):
    pass

desc = pd.DataFrame(index=orig_train.columns)
desc['count'] = len(orig_train)
desc['nunique'] = orig_train.nunique()
desc['%unique'] = desc['nunique'] / len(orig_train) * 100
desc['null'] = orig_train.isna().sum()
desc['type'] = orig_train.dtypes
desc

feature_set_1 = ['RainingDays', 'AverageRainingDays']
feature_set_2 = ['MaxOfUpperTRange', 'MinOfUpperTRange', 'AverageOfUpperTRange', 'MaxOfLowerTRange', 'MinOfLowerTRange', 'AverageOfLowerTRange']
feature_set_3 = ['fruitset', 'fruitmass', 'seeds']

combo = pd.concat([train.drop('yield', axis=1), test])

desc = pd.DataFrame(index=combo.columns)
desc['count'] = len(combo)
desc['nunique'] = combo.nunique()
desc['%unique'] = desc['nunique'] / len(combo) * 100
desc['null'] = combo.isna().sum()
desc['type'] = combo.dtypes
desc

combo[-combo.duplicated(subset=feature_set_1)].loc[:, feature_set_1]

combo[-combo.duplicated(subset=feature_set_2)].loc[:, feature_set_2]

find_duplicates(combo, feature_set_3, 'competition')