from langchain.agents import AgentType, initialize_agent, load_tools
from langchain.llms import OpenAI

from benchllm import SemanticEvaluator, Test, Tester

tools = load_tools(["serpapi", "llm-math"], llm=OpenAI(temperature=0))
agent = initialize_agent(tools, OpenAI(temperature=0), agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, verbose=True)

tests = [Test(input="How many people live in canada as of 2023?", expected=["approximately 38,625,801"])]

tester = Tester(lambda input: agent(input)["output"])
tester.add_tests(tests)
predictions = tester.run()

evaluator = SemanticEvaluator()
evaluator.load(predictions)
report = evaluator.run()

print(report)

langchain.llms.OpenAI
[((569, 588), 'benchllm.SemanticEvaluator', 'SemanticEvaluator', ([], {}), '()\n', (586, 588), False, 'from benchllm import SemanticEvaluator, Test, Tester\n'), ((261, 282), 'langchain.llms.OpenAI', 'OpenAI', ([], {'temperature': '(0)'}), '(temperature=0)\n', (267, 282), False, 'from langchain.llms import OpenAI\n'), ((353, 453), 'benchllm.Test', 'Test', ([], {'input': '"""How many people live in canada as of 2023?"""', 'expected': "['approximately 38,625,801']"}), "(input='How many people live in canada as of 2023?', expected=[\n    'approximately 38,625,801'])\n", (357, 453), False, 'from benchllm import SemanticEvaluator, Test, Tester\n'), ((206, 227), 'langchain.llms.OpenAI', 'OpenAI', ([], {'temperature': '(0)'}), '(temperature=0)\n', (212, 227), False, 'from langchain.llms import OpenAI\n')]