image_output_1.png
import matplotlib.pyplot as plt

import numpy as np

import pandas as pd

import seaborn as sns

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
pd.options.display.min_rows = 100
pd.options.display.max_rows = 100
plt.style.use('seaborn-whitegrid')
plt.rc('figure', autolayout=True, titlesize=18, titleweight='bold')
plt.rc('axes', labelweight='bold', labelsize='large', titleweight='bold', titlesize=16, titlepad=10)
get_ipython().config.InlineBackend.figure_format = 'retina'

leaderboard = pd.read_csv('../input/feedback-prize-efficiency-data/leaderboard.csv', index_col='EfficiencyRank')
leaderboard.to_csv('full_leaderboard.csv')

def is_pareto_efficient(costs, return_mask=True):
    """
    Find the pareto-efficient points
    :param costs: An (n_points, n_costs) array
    :param return_mask: True to return a mask
    :return: An array of indices of pareto-efficient points.
        If return_mask is True, this will be an (n_points, ) boolean array
        Otherwise it will be a (n_efficient_points, ) integer array of indices.
    """
    is_efficient = np.arange(costs.shape[0])
    n_points = costs.shape[0]
    next_point_index = 0
    while next_point_index < len(costs):
        nondominated_point_mask = np.any(costs < costs[next_point_index], axis=1)
        nondominated_point_mask[next_point_index] = True
        is_efficient = is_efficient[nondominated_point_mask]
        costs = costs[nondominated_point_mask]
        next_point_index = np.sum(nondominated_point_mask[:next_point_index]) + 1
    if return_mask:
        is_efficient_mask = np.zeros(n_points, dtype=bool)
        is_efficient_mask[is_efficient] = True
        return is_efficient_mask
    else:
        return is_efficient

max_runtime = 32400
min_logloss = 0.55435
def efficiency_score(score, scoring_time):
    return score / (np.log(3) - min_logloss) + scoring_time / max_runtime

df_costs = leaderboard.loc[:, ['PrivateScore', 'ScoringTime']]
df_costs['efficient'] = is_pareto_efficient(df_costs.assign(PrivateScore=lambda x: x['PrivateScore']).to_numpy())
df_costs['efficient'] = df_costs['efficient'].astype('category')
fig, ax = plt.subplots(figsize=(7, 7), dpi=120)
x = [min_logloss, np.log(3)]
y = [0, max_runtime]
xx, yy = np.meshgrid(np.linspace(x[0], x[1], 100), np.linspace(y[0], y[1], 100))
zz = efficiency_score(xx, yy)
ax.contourf(xx, yy, zz, levels=25, alpha=0.5, zorder=1, cmap='magma')
df_efficient = df_costs.astype({'efficient': bool}).query('efficient').sort_values(f'PrivateScore')
ax.plot(df_efficient['PrivateScore'], df_efficient['ScoringTime'], '.-', color='C3', zorder=100, linewidth=1)
hue_order = [True, False]
colors = {True: 'C3', False: 'C0'}
sns.scatterplot(x='PrivateScore', y='ScoringTime', hue='efficient', data=df_costs, ax=ax, palette=colors, hue_order=hue_order)
ax.set_xlim(x[0] - 0.001, x[1] + 0.001)
ax.set_ylim(y[0] - 10, y[1] + 10)
ax.set_title('Efficiency Leaderboard')
ax.grid([])