%env ANTHROPIC_API_KEY=<API_KEY>from langchain.chat_models import ChatAnthropicllm = ChatAnthropic(temperature=0)evaluator = load_evaluator("criteria", llm=llm, criteria="conciseness")eval_result = evaluator.evaluate_strings(    prediction="What's 2+2? That's an elementary question. The answer you're looking for is that two and two is four.",    input="What's 2+2?",)print(eval_result)    {'reasoning': 'Step 1) Analyze the conciseness criterion: Is the submission concise and to the point?\nStep 2) The submission provides extraneous information beyond just answering the question directly. It characterizes the question as "elementary" and provides reasoning for why the answer is 4. This additional commentary makes the submission not fully concise.\nStep 3) Therefore, based on the analysis of the conciseness criterion, the submission does not meet the criteria.\n\nN', 'value': 'N', 'score': 0}Configuring the PromptIf you want to completely customize the prompt, you can initialize the evaluator with a custom prompt template as follows.from langchain.prompts import PromptTemplatefstring = """Respond Y or N based on how well the following response follows the specified rubric. Grade only based on the rubric and expected response:Grading Rubric: {criteria}Expected Response: {reference}DATA:---------Question: {input}Response: {output}---------Write out your explanation for each criterion, then respond with Y or N on a new line."""prompt = PromptTemplate.from_template(fstring)evaluator = load_evaluator(    "labeled_criteria", criteria="correctness", prompt=prompt)eval_result = evaluator.evaluate_strings(    prediction="What's 2+2? That's an elementary question. The answer you're looking for is that two and two is four.",    input="What's 2+2?",    reference="It's 17 now.",)print(eval_result)    {'reasoning': 'Correctness: No, the response is not correct. The expected response was "It\'s 17 now." but the response given was "What\'s 2+2? That\'s an elementary question. The
Open In Colab
Open In Colab ->: %env ANTHROPIC_API_KEY=<API_KEY>from langchain.chat_models import ChatAnthropicllm = ChatAnthropic(temperature=0)evaluator = load_evaluator("criteria", llm=llm, criteria="conciseness")eval_result = evaluator.evaluate_strings(    prediction="What's 2+2? That's an elementary question. The answer you're looking for is that two and two is four.",    input="What's 2+2?",)print(eval_result)    {'reasoning': 'Step 1) Analyze the conciseness criterion: Is the submission concise and to the point?\nStep 2) The submission provides extraneous information beyond just answering the question directly. It characterizes the question as "elementary" and provides reasoning for why the answer is 4. This additional commentary makes the submission not fully concise.\nStep 3) Therefore, based on the analysis of the conciseness criterion, the submission does not meet the criteria.\n\nN', 'value': 'N', 'score': 0}Configuring the PromptIf you want to completely customize the prompt, you can initialize the evaluator with a custom prompt template as follows.from langchain.prompts import PromptTemplatefstring = """Respond Y or N based on how well the following response follows the specified rubric. Grade only based on the rubric and expected response:Grading Rubric: {criteria}Expected Response: {reference}DATA:---------Question: {input}Response: {output}---------Write out your explanation for each criterion, then respond with Y or N on a new line."""prompt = PromptTemplate.from_template(fstring)evaluator = load_evaluator(    "labeled_criteria", criteria="correctness", prompt=prompt)eval_result = evaluator.evaluate_strings(    prediction="What's 2+2? That's an elementary question. The answer you're looking for is that two and two is four.",    input="What's 2+2?",    reference="It's 17 now.",)print(eval_result)    {'reasoning': 'Correctness: No, the response is not correct. The expected response was "It\'s 17 now." but the response given was "What\'s 2+2? That\'s an elementary question. The