text_plain_output_1.png
import pandas as pd

target = 'yield'
df1 = pd.read_csv(TRAIN_CSV)
df1.rename({'Id': 'id'}, axis=1, inplace=True)
df1['test'] = 0
df1['gen'] = 1
df2 = pd.read_csv(TEST_CSV)
df2.rename({'Id': 'id'}, axis=1, inplace=True)
df2['test'] = 1
df2['gen'] = 1
df3 = pd.read_csv(EXTERNAL_CSV)
df3.rename({'Row#': 'id'}, axis=1, inplace=True)
df3['test'] = 0
df3['gen'] = 0
df = pd.concat([df1, df2, df3])
df.id.fillna(-1, inplace=True)
df.id = df.id.astype(int)
df.reset_index(inplace=True)
df.drop('index', axis=1, inplace=True)

df.columns

def explore_categorical_value(df, col, target='yield'):
    print(f'{col}')
    values = df[col].unique()
    values.sort()
    for value in values:
        select = df.loc[df[col] == value]
        mn = select[target].mean()
        cnt = len(select)
        cnt_test = len(select[select.test == 1])
        print(f'\t{value:16}\t{mn:.3f}\t{cnt}\t{cnt_test}')
cat_columns = ['clonesize', 'honeybee', 'bumbles', 'andrena', 'osmia', 'MaxOfUpperTRange', 'MinOfUpperTRange', 'AverageOfUpperTRange', 'MaxOfLowerTRange', 'MinOfLowerTRange', 'AverageOfLowerTRange', 'RainingDays', 'AverageRainingDays']
print(f'\t                  \tMean \t\tCount\tIn test')
for col in cat_columns:
    explore_categorical_value(df, col)