= Counter(preferences)pref_ratios = {k: v / len(preferences) for k, v in counts.items()}for k, v in pref_ratios.items():    print(f"{name_map.get(k)}: {v:.2%}")    OpenAI Functions Agent: 95.00%    None: 5.00%Estimate Confidence Intervals‚ÄãThe results seem pretty clear, but if you want to have a better sense of how confident we are, that model "A" (the OpenAI Functions Agent) is the preferred model, we can calculate confidence intervals. Below, use the Wilson score to estimate the confidence interval.from math import sqrtdef wilson_score_interval(    preferences: list, which: str = "a", z: float = 1.96) -> tuple:    """Estimate the confidence interval using the Wilson score.    See: https://en.wikipedia.org/wiki/Binomial_proportion_confidence_interval#Wilson_score_interval    for more details, including when to use it and when it should not be used.    """    total_preferences = preferences.count("a") + preferences.count("b")    n_s = preferences.count(which)    if total_preferences == 0:        return (0, 0)    p_hat = n_s / total_preferences    denominator = 1 + (z**2) / total_preferences    adjustment = (z / denominator) * sqrt(        p_hat * (1 - p_hat) / total_preferences        + (z**2) / (4 * total_preferences * total_preferences)    )    center = (p_hat + (z**2) / (2 * total_preferences)) / denominator    lower_bound = min(max(center - adjustment, 0.0), 1.0)    upper_bound = min(max(center + adjustment, 0.0), 1.0)    return (lower_bound, upper_bound)for which_, name in name_map.items():    low, high = wilson_score_interval(preferences, which=which_)    print(        f'The "{name}" would be preferred between {low:.2%} and {high:.2%} percent of the time (with 95% confidence).'    )    The "OpenAI Functions Agent" would be preferred between 83.18% and 100.00% percent of the time (with 95% confidence).    The "Structured Chat Agent" would be preferred between 0.00% and 16.82% percent of the time (with 95% confidence).Print out the p-value.from scipy import
Open In Colab
Open In Colab ->: = Counter(preferences)pref_ratios = {k: v / len(preferences) for k, v in counts.items()}for k, v in pref_ratios.items():    print(f"{name_map.get(k)}: {v:.2%}")    OpenAI Functions Agent: 95.00%    None: 5.00%Estimate Confidence Intervals‚ÄãThe results seem pretty clear, but if you want to have a better sense of how confident we are, that model "A" (the OpenAI Functions Agent) is the preferred model, we can calculate confidence intervals. Below, use the Wilson score to estimate the confidence interval.from math import sqrtdef wilson_score_interval(    preferences: list, which: str = "a", z: float = 1.96) -> tuple:    """Estimate the confidence interval using the Wilson score.    See: https://en.wikipedia.org/wiki/Binomial_proportion_confidence_interval#Wilson_score_interval    for more details, including when to use it and when it should not be used.    """    total_preferences = preferences.count("a") + preferences.count("b")    n_s = preferences.count(which)    if total_preferences == 0:        return (0, 0)    p_hat = n_s / total_preferences    denominator = 1 + (z**2) / total_preferences    adjustment = (z / denominator) * sqrt(        p_hat * (1 - p_hat) / total_preferences        + (z**2) / (4 * total_preferences * total_preferences)    )    center = (p_hat + (z**2) / (2 * total_preferences)) / denominator    lower_bound = min(max(center - adjustment, 0.0), 1.0)    upper_bound = min(max(center + adjustment, 0.0), 1.0)    return (lower_bound, upper_bound)for which_, name in name_map.items():    low, high = wilson_score_interval(preferences, which=which_)    print(        f'The "{name}" would be preferred between {low:.2%} and {high:.2%} percent of the time (with 95% confidence).'    )    The "OpenAI Functions Agent" would be preferred between 83.18% and 100.00% percent of the time (with 95% confidence).    The "Structured Chat Agent" would be preferred between 0.00% and 16.82% percent of the time (with 95% confidence).Print out the p-value.from scipy import