to speed things up# This will lead to larger confidence intervals downstream.batch = []for example in tqdm(dataset[:20]):    batch.extend([agent.acall(example["inputs"]) for agent in agents])    if len(batch) >= concurrency_level:        batch_results = await asyncio.gather(*batch, return_exceptions=True)        results.extend(list(zip(*[iter(batch_results)] * 2)))        batch = []if batch:    batch_results = await asyncio.gather(*batch, return_exceptions=True)    results.extend(list(zip(*[iter(batch_results)] * 2)))      0%|          | 0/20 [00:00<?, ?it/s]Step 5. Evaluate Pairs‚ÄãNow it's time to evaluate the results. For each agent response, run the evaluation chain to select which output is preferred (or return a tie).Randomly select the input order to reduce the likelihood that one model will be preferred just because it is presented first.import randomdef predict_preferences(dataset, results) -> list:    preferences = []    for example, (res_a, res_b) in zip(dataset, results):        input_ = example["inputs"]        # Flip a coin to reduce persistent position bias        if random.random() < 0.5:            pred_a, pred_b = res_a, res_b            a, b = "a", "b"        else:            pred_a, pred_b = res_b, res_a            a, b = "b", "a"        eval_res = eval_chain.evaluate_string_pairs(            prediction=pred_a["output"] if isinstance(pred_a, dict) else str(pred_a),            prediction_b=pred_b["output"] if isinstance(pred_b, dict) else str(pred_b),            input=input_,        )        if eval_res["value"] == "A":            preferences.append(a)        elif eval_res["value"] == "B":            preferences.append(b)        else:            preferences.append(None)  # No preference    return preferencespreferences = predict_preferences(dataset, results)Print out the ratio of preferences.from collections import Countername_map = {    "a": "OpenAI Functions Agent",    "b": "Structured Chat Agent",}counts = Counter(preferences)pref_ratios = {k:
Open In Colab
Open In Colab ->: to speed things up# This will lead to larger confidence intervals downstream.batch = []for example in tqdm(dataset[:20]):    batch.extend([agent.acall(example["inputs"]) for agent in agents])    if len(batch) >= concurrency_level:        batch_results = await asyncio.gather(*batch, return_exceptions=True)        results.extend(list(zip(*[iter(batch_results)] * 2)))        batch = []if batch:    batch_results = await asyncio.gather(*batch, return_exceptions=True)    results.extend(list(zip(*[iter(batch_results)] * 2)))      0%|          | 0/20 [00:00<?, ?it/s]Step 5. Evaluate Pairs‚ÄãNow it's time to evaluate the results. For each agent response, run the evaluation chain to select which output is preferred (or return a tie).Randomly select the input order to reduce the likelihood that one model will be preferred just because it is presented first.import randomdef predict_preferences(dataset, results) -> list:    preferences = []    for example, (res_a, res_b) in zip(dataset, results):        input_ = example["inputs"]        # Flip a coin to reduce persistent position bias        if random.random() < 0.5:            pred_a, pred_b = res_a, res_b            a, b = "a", "b"        else:            pred_a, pred_b = res_b, res_a            a, b = "b", "a"        eval_res = eval_chain.evaluate_string_pairs(            prediction=pred_a["output"] if isinstance(pred_a, dict) else str(pred_a),            prediction_b=pred_b["output"] if isinstance(pred_b, dict) else str(pred_b),            input=input_,        )        if eval_res["value"] == "A":            preferences.append(a)        elif eval_res["value"] == "B":            preferences.append(b)        else:            preferences.append(None)  # No preference    return preferencespreferences = predict_preferences(dataset, results)Print out the ratio of preferences.from collections import Countername_map = {    "a": "OpenAI Functions Agent",    "b": "Structured Chat Agent",}counts = Counter(preferences)pref_ratios = {k: