Evaluator Comparison Analysis
Analyzing differences between Arthur's and Anna's evaluations
""")
)
# Display Agreement Section
display(Markdown('## Agreement Overview'))
# Create side-by-side visualizations
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(18, 7))
# Agreement Pie Chart
labels = ['Agreement', 'Disagreement']
sizes = [agree_count, disagree_count]
colors = ['#4CAF50', '#F44336']
explode = (0.1, 0)
ax1.pie(
sizes,
explode=explode,
labels=labels,
colors=colors,
autopct='%1.1f%%',
shadow=True,
startangle=140,
textprops={'fontsize': 12, 'fontweight': 'bold'},
)
ax1.set_title('Evaluator Pass/Fail Agreement', fontsize=16, fontweight='bold')
# Average Scores Bar Chart
avg_df = calculate_average_metrics()
# Set width of bars
bar_width = 0.35
x = np.arange(len(avg_df))
# Create bars
ax2.bar(
x - bar_width / 2,
avg_df['Arthur'],
width=bar_width,
label="Arthur's Avg",
color='#8884d8',
edgecolor='white',
linewidth=1.5,
)
ax2.bar(
x + bar_width / 2, avg_df['Anna'], width=bar_width, label="Anna's Avg", color='#82ca9d', edgecolor='white', linewidth=1.5
)
# Add data labels
for i in range(len(x)):
ax2.text(
x[i] - bar_width / 2,
avg_df['Arthur'][i] + 0.05,
f'{avg_df["Arthur"][i]:.2f}',
ha='center',
va='bottom',
fontweight='bold',
fontsize=10,
)
ax2.text(
x[i] + bar_width / 2,
avg_df['Anna'][i] + 0.05,
f'{avg_df["Anna"][i]:.2f}',
ha='center',
va='bottom',
fontweight='bold',
fontsize=10,
)
# Customize plot
ax2.set_xlabel('Category', fontsize=12, fontweight='bold')
ax2.set_ylabel('Average Score', fontsize=12, fontweight='bold')
ax2.set_title('Average Scores by Evaluator', fontsize=16, fontweight='bold')
ax2.set_xticks(x)
ax2.set_xticklabels(avg_df['Category'], fontsize=12)
ax2.set_ylim(0, 2.2)
ax2.grid(axis='y', linestyle='--', alpha=0.7)
ax2.legend(loc='lower center', bbox_to_anchor=(0.5, -0.25), ncol=2, fontsize=12)
plt.tight_layout()
display(plt.gcf())
plt.close()
# Now show correlation analysis
display(Markdown('## Correlation Analysis'))
# Create correlations chart
fig, ax = plt.subplots(figsize=(10, 6))
metrics = ['Helpfulness', 'Adequate Length', 'Pass Rate']
corr_values = [correlations['helpfulness'], correlations['adequate_length'], correlations['pass_rate']]
bars = ax.bar(metrics, corr_values)
# Colorize bars based on correlation (positive or negative)
for i, bar in enumerate(bars):
if corr_values[i] < 0:
bar.set_color('#F44336') # red for negative correlation
else:
bar.set_color('#4CAF50') # green for positive correlation
# Add correlation values above/below bars
for i, v in enumerate(corr_values):
if v >= 0:
ax.text(i, v + 0.05, f'{v:.2f}', ha='center', fontweight='bold')
else:
ax.text(i, v - 0.1, f'{v:.2f}', ha='center', fontweight='bold')
# Add reference line at y=0
ax.axhline(y=0, color='black', linestyle='-', alpha=0.3)
# Set y-axis limits to show the full range -1 to 1
ax.set_ylim(-1.1, 1.1)
ax.set_title('Evaluator Correlation Analysis', fontsize=14, fontweight='bold')
ax.set_ylabel('Correlation Coefficient', fontsize=12)
ax.text(
1,
-0.9,
'Range: -1 to 1, where 1 is perfect positive correlation,\n-1 is perfect negative correlation, and 0 is no correlation',
fontsize=8,
ha='center',
style='italic',
)
plt.tight_layout()
display(plt.gcf())
plt.close()
# Display scatter plots
display(Markdown('## Score Comparison Scatter Plots'))
# Create a 1x2 grid for helpfulness and adequate length scatter plots
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(18, 7))
# Helpfulness Scatter Plot
helpfulness_data = [item for item in agreement_data if item['category'] == 'Helpfulness']
x1 = [item['arthurValue'] for item in helpfulness_data]
y1 = [item['annaValue'] for item in helpfulness_data]
platforms1 = [item['platform'] for item in helpfulness_data]
scatter1 = ax1.scatter(x1, y1, c='#8884d8', s=100, alpha=0.7)
# Add platform labels
for i, platform in enumerate(platforms1):
ax1.annotate(platform, (x1[i], y1[i]), textcoords='offset points', xytext=(0, 10), ha='center')
# Add axis labels
ax1.set_xlabel("Arthur's Rating", fontsize=12)
ax1.set_ylabel("Anna's Rating", fontsize=12)
ax1.set_title('Helpfulness Correlation', fontsize=14, fontweight='bold')
# Set axis limits
ax1.set_xlim(0, 2)
ax1.set_ylim(0, 2)
# Add perfect correlation line
ax1.plot([0, 2], [0, 2], 'k--', alpha=0.3)
# Add correlation value text
ax1.text(0.1, 1.8, f'Correlation: {correlations["helpfulness"]}', fontsize=12, bbox=dict(facecolor='white', alpha=0.5))
ax1.grid(True, linestyle='--', alpha=0.3)
# Adequate Length Scatter Plot
adequate_length_data = [item for item in agreement_data if item['category'] == 'Adequate Length']
x2 = [item['arthurValue'] for item in adequate_length_data]
y2 = [item['annaValue'] for item in adequate_length_data]
platforms2 = [item['platform'] for item in adequate_length_data]
scatter2 = ax2.scatter(x2, y2, c='#82ca9d', s=100, alpha=0.7)
# Add platform labels
for i, platform in enumerate(platforms2):
ax2.annotate(platform, (x2[i], y2[i]), textcoords='offset points', xytext=(0, 10), ha='center')
# Add axis labels
ax2.set_xlabel("Arthur's Rating", fontsize=12)
ax2.set_ylabel("Anna's Rating", fontsize=12)
ax2.set_title('Adequate Length Correlation', fontsize=14, fontweight='bold')
# Set axis limits
ax2.set_xlim(0, 2)
ax2.set_ylim(0, 2)
# Add perfect correlation line
ax2.plot([0, 2], [0, 2], 'k--', alpha=0.3)
# Add correlation value text
ax2.text(0.1, 1.8, f'Correlation: {correlations["adequate_length"]}', fontsize=12, bbox=dict(facecolor='white', alpha=0.5))
ax2.grid(True, linestyle='--', alpha=0.3)
plt.tight_layout()
display(plt.gcf())
plt.close()
# Pass Rate Correlation Scatter Plot
display(Markdown('## Pass Rate Comparison'))
plt.figure(figsize=(10, 6))
x = [item['arthur'] for item in pass_fail_agreement]
y = [item['anna'] for item in pass_fail_agreement]
platforms = [item['platform'] for item in pass_fail_agreement]
colors = ['#4CAF50' if item['agreement'] == 'Agree' else '#F44336' for item in pass_fail_agreement]
scatter = plt.scatter(x, y, c=colors, s=100, alpha=0.7)
# Add platform labels
for i, platform in enumerate(platforms):
plt.annotate(platform, (x[i], y[i]), textcoords='offset points', xytext=(0, 10), ha='center')
# Add axis labels
plt.xlabel("Arthur's Pass Rate (%)", fontsize=12)
plt.ylabel("Anna's Pass Rate (%)", fontsize=12)
plt.title('Pass Rate Correlation', fontsize=14, fontweight='bold')
# Set axis limits
plt.xlim(30, 105)
plt.ylim(30, 105)
# Add perfect correlation line
plt.plot([30, 105], [30, 105], 'k--', alpha=0.3)
# Add correlation value text
plt.text(35, 95, f'Correlation: {correlations["pass_rate"]}', fontsize=12, bbox=dict(facecolor='white', alpha=0.5))
# Add legend
from matplotlib.lines import Line2D
legend_elements = [
Line2D([0], [0], marker='o', color='w', markerfacecolor='#4CAF50', markersize=10, label='Agreement'),
Line2D([0], [0], marker='o', color='w', markerfacecolor='#F44336', markersize=10, label='Disagreement'),
]
plt.legend(handles=legend_elements, loc='upper left')
plt.grid(True, linestyle='--', alpha=0.3)
plt.tight_layout()
display(plt.gcf())
plt.close()
# Platform-specific differences
display(Markdown('## Platform-specific Evaluator Differences'))
# Calculate platform differences if not already done
if not 'display_df' in globals():
platform_differences = []
for platform in set(item['platform'] for item in agreement_data):
helpfulness = next(
(item for item in agreement_data if item['platform'] == platform and item['category'] == 'Helpfulness'), None
)
adequate_length = next(
(item for item in agreement_data if item['platform'] == platform and item['category'] == 'Adequate Length'), None
)
pass_fail = next((item for item in pass_fail_agreement if item['platform'] == platform), None)
if helpfulness and adequate_length and pass_fail:
helpfulness_diff = helpfulness['arthurValue'] - helpfulness['annaValue']
adequate_length_diff = adequate_length['arthurValue'] - adequate_length['annaValue']
pass_rate_diff = pass_fail['arthur'] - pass_fail['anna']
return platform_differences.append(
{
'Platform': platform,
'Helpfulness Diff': helpfulness_diff,
'Adequate Length Diff': adequate_length_diff,
'Pass Rate Diff': pass_rate_diff,
'Agreement': pass_fail['agreement'],
}
)
platform_differences = []
for platform in set(item['platform'] for item in agreement_data):
helpfulness = next((item for item in agreement_data if item['platform'] == platform and item['category'] == 'Helpfulness'), None)
adequate_length = next(
(item for item in agreement_data if item['platform'] == platform and item['category'] == 'Adequate Length'), None
)
pass_fail = next((item for item in pass_fail_agreement if item['platform'] == platform), None)
if helpfulness and adequate_length and pass_fail:
helpfulness_diff = helpfulness['arthurValue'] - helpfulness['annaValue']
adequate_length_diff = adequate_length['arthurValue'] - adequate_length['annaValue']
pass_rate_diff = pass_fail['arthur'] - pass_fail['anna']
return platform_differences.append(
{
'Platform': platform,
'Helpfulness Diff': helpfulness_diff,
'Adequate Length Diff': adequate_length_diff,
'Pass Rate Diff': pass_rate_diff,
'Agreement': pass_fail['agreement'],
}
)
platform_diff_df = pd.DataFrame(platform_differences)
_()
return
@app.cell
def _(correlations, plt):
# Creating Correlation Analysis Chart
fig, ax = plt.subplots(figsize=(10, 6))
metrics = ['Helpfulness', 'Adequate Length', 'Pass Rate']
corr_values = [correlations['helpfulness'], correlations['adequate_length'], correlations['pass_rate']]
bars = ax.bar(metrics, corr_values, color=['#8884d8', '#82ca9d', '#ff7300'])
# Colorize bars based on correlation (positive or negative)
for i, bar in enumerate(bars):
if corr_values[i] < 0:
bar.set_color('#F44336') # red for negative correlation
else:
bar.set_color('#4CAF50') # green for positive correlation
# Add correlation values above/below bars
for i, v in enumerate(corr_values):
if v >= 0:
ax.text(i, v + 0.05, f'{v:.2f}', ha='center', fontweight='bold')
else:
ax.text(i, v - 0.1, f'{v:.2f}', ha='center', fontweight='bold')
# Add reference line at y=0
ax.axhline(y=0, color='black', linestyle='-', alpha=0.3)
# Set y-axis limits to show the full range -1 to 1
ax.set_ylim(-1.1, 1.1)
# Add labels and title
ax.set_title('Evaluator Correlation Analysis', fontsize=14, fontweight='bold')
ax.set_ylabel('Correlation Coefficient', fontsize=12)
ax.text(
1,
-0.9,
'Range: -1 to 1, where 1 is perfect positive correlation,\n-1 is perfect negative correlation, and 0 is no correlation',
fontsize=8,
ha='center',
style='italic',
)
plt.tight_layout()
return ax, bar, bars, corr_values, fig, i, metrics, v
@app.cell
def _(agreement_data, correlations, plt):
def _():
# Create Helpfulness Correlation Scatter Plot
helpfulness_data = [item for item in agreement_data if item['category'] == 'Helpfulness']
fig, ax = plt.subplots(figsize=(8, 6))
x = [item['arthurValue'] for item in helpfulness_data]
y = [item['annaValue'] for item in helpfulness_data]
platforms = [item['platform'] for item in helpfulness_data]
scatter = ax.scatter(x, y, c='#8884d8', s=100, alpha=0.7)
# Add platform labels
for i, platform in enumerate(platforms):
ax.annotate(platform, (x[i], y[i]), textcoords='offset points', xytext=(0, 10), ha='center')
# Add axis labels
ax.set_xlabel("Arthur's Rating", fontsize=12)
ax.set_ylabel("Anna's Rating", fontsize=12)
ax.set_title('Helpfulness Correlation', fontsize=14, fontweight='bold')
# Set axis limits
ax.set_xlim(0, 2)
ax.set_ylim(0, 2)
# Add perfect correlation line
ax.plot([0, 2], [0, 2], 'k--', alpha=0.3)
# Add correlation value text
ax.text(0.1, 1.8, f'Correlation: {correlations["helpfulness"]}', fontsize=12, bbox=dict(facecolor='white', alpha=0.5))
plt.grid(True, linestyle='--', alpha=0.3)
return plt.tight_layout()
_()
return
@app.cell
def _(agreement_data, pass_fail_agreement, pd):
def _():
# Create a DataFrame to show platform-specific differences
platform_differences = []
for platform in set(item['platform'] for item in agreement_data):
helpfulness = next((item for item in agreement_data if item['platform'] == platform and item['category'] == 'Helpfulness'), None)
adequate_length = next(
(item for item in agreement_data if item['platform'] == platform and item['category'] == 'Adequate Length'), None
)
pass_fail = next((item for item in pass_fail_agreement if item['platform'] == platform), None)
if helpfulness and adequate_length and pass_fail:
helpfulness_diff = helpfulness['arthurValue'] - helpfulness['annaValue']
adequate_length_diff = adequate_length['arthurValue'] - adequate_length['annaValue']
pass_rate_diff = pass_fail['arthur'] - pass_fail['anna']
return platform_differences.append(
{
'Platform': platform,
'Helpfulness Diff': helpfulness_diff,
'Adequate Length Diff': adequate_length_diff,
'Pass Rate Diff': pass_rate_diff,
'Agreement': pass_fail['agreement'],
}
)
platform_diff_df = pd.DataFrame(platform_differences)
# Display platform differences
platform_diff_df['Helpfulness Diff'] = platform_diff_df['Helpfulness Diff'].round(1)
platform_diff_df['Adequate Length Diff'] = platform_diff_df['Adequate Length Diff'].round(1)
platform_diff_df['Pass Rate Diff'] = platform_diff_df['Pass Rate Diff'].astype(int)
def style_diff(val):
if val > 0:
return f'Arthur +{abs(val)}'
elif val < 0:
return f'Anna +{abs(val)}'
else:
return 'Equal'
# Apply styling and display the data
styled_platform_diff = platform_diff_df.copy()
styled_platform_diff['Helpfulness'] = styled_platform_diff['Helpfulness Diff'].apply(style_diff)
styled_platform_diff['Adequate Length'] = styled_platform_diff['Adequate Length Diff'].apply(style_diff)
styled_platform_diff['Pass Rate'] = styled_platform_diff['Pass Rate Diff'].apply(style_diff)
display_cols = ['Platform', 'Helpfulness', 'Adequate Length', 'Pass Rate', 'Agreement']
display_df = styled_platform_diff[display_cols]
_()
return
@app.cell
def _(calculate_average_metrics, np, plt):
def plot_average_scores():
"""Plot the average scores for each category by evaluator"""
# Get average data
avg_df = calculate_average_metrics()
# Set up plot
plt.figure(figsize=(10, 6))
# Set width of bars
bar_width = 0.35
x = np.arange(len(avg_df))
# Create bars
plt.bar(
x - bar_width / 2,
avg_df['Arthur'],
width=bar_width,
label="Arthur's Avg. Score",
color='#8884d8',
alpha=0.8,
edgecolor='white',
linewidth=1.5,
)
plt.bar(
x + bar_width / 2,
avg_df['Anna'],
width=bar_width,
label="Anna's Avg. Score",
color='#82ca9d',
alpha=0.8,
edgecolor='white',
linewidth=1.5,
)
# Add data labels
for i in range(len(x)):
plt.text(
x[i] - bar_width / 2,
avg_df['Arthur'][i] + 0.05,
f'{avg_df["Arthur"][i]:.2f}',
ha='center',
va='bottom',
color='#333',
fontweight='bold',
)
plt.text(
x[i] + bar_width / 2,
avg_df['Anna'][i] + 0.05,
f'{avg_df["Anna"][i]:.2f}',
ha='center',
va='bottom',
color='#333',
fontweight='bold',
)
# Customize plot
plt.xlabel('Evaluation Category', fontsize=12, fontweight='bold')
plt.ylabel('Average Score (0-2 scale)', fontsize=12, fontweight='bold')
plt.title('Average Scores by Evaluator', fontsize=14, fontweight='bold')
plt.xticks(x, avg_df['Category'], fontsize=11)
plt.ylim(0, 2.2) # Set reasonable y-axis limit
plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.15), shadow=True, ncol=2)
plt.grid(axis='y', linestyle='--', alpha=0.7)
# Add a border to the plot
ax = plt.gca()
for spine in ax.spines.values():
spine.set_edgecolor('#dddddd')
spine.set_linewidth(1.5)
plt.tight_layout()
return plt.gca()
return (plot_average_scores,)
@app.cell
def _(agreement_data, pass_fail_agreement, pd):
def calculate_average_metrics():
"""Calculate average metrics for each evaluator and category"""
# Process helpfulness data
helpfulness_data = [item for item in agreement_data if item['category'] == 'Helpfulness']
arthur_helpfulness = sum(item['arthurValue'] for item in helpfulness_data) / len(helpfulness_data)
anna_helpfulness = sum(item['annaValue'] for item in helpfulness_data) / len(helpfulness_data)
# Process adequate length data
adequate_length_data = [item for item in agreement_data if item['category'] == 'Adequate Length']
arthur_adequate = sum(item['arthurValue'] for item in adequate_length_data) / len(adequate_length_data)
anna_adequate = sum(item['annaValue'] for item in adequate_length_data) / len(adequate_length_data)
# Create DataFrame with results
avg_df = pd.DataFrame(
{
'Category': ['Helpfulness', 'Adequate Length'],
'Arthur': [arthur_helpfulness, arthur_adequate],
'Anna': [anna_helpfulness, anna_adequate],
}
)
return avg_df
# Count agreement vs disagreement
agree_count = sum(1 for item in pass_fail_agreement if item['agreement'] == 'Agree')
disagree_count = sum(1 for item in pass_fail_agreement if item['agreement'] == 'Disagree')
return agree_count, calculate_average_metrics, disagree_count
@app.cell
def _(plot_average_scores):
plot_average_scores()
return
@app.cell
def _(agree_count, ax, calculate_average_metrics, disagree_count, np, plt):
def interactive_evaluator_dashboard():
"""Display an interactive dashboard for evaluator analysis"""
from IPython.display import display, Markdown, HTML
# Display header
display(
HTML("""