Skip to main contentü¶úÔ∏èüîó LangChainDocsUse casesIntegrationsAPICommunityChat our docsLangSmithJS/TS DocsSearchCTRLKGet startedIntroductionInstallationQuickstartLangChain Expression LanguageInterfaceHow toCookbookLangChain Expression Language (LCEL)Why use LCEL?ModulesModel I/‚ÄãORetrievalChainsMemoryAgentsCallbacksModulesSecurityGuidesAdaptersDebuggingDeploymentEvaluationString EvaluatorsComparison EvaluatorsTrajectory EvaluatorsCustom Trajectory EvaluatorAgent TrajectoryExamplesFallbacksLangSmithRun LLMs locallyModel comparisonPrivacyPydantic compatibilitySafetyMoreGuidesEvaluationTrajectory EvaluatorsCustom Trajectory EvaluatorCustom Trajectory EvaluatorYou can make your own custom trajectory evaluators by inheriting from the AgentTrajectoryEvaluator class and overwriting the _evaluate_agent_trajectory (and _aevaluate_agent_action) method.In this example, you will make a simple trajectory evaluator that uses an LLM to determine if any actions were unnecessary.from typing import Any, Optional, Sequence, Tuplefrom langchain.chat_models import ChatOpenAIfrom langchain.chains import LLMChainfrom langchain.schema import AgentActionfrom langchain.evaluation import AgentTrajectoryEvaluatorclass StepNecessityEvaluator(AgentTrajectoryEvaluator):    """Evaluate the perplexity of a predicted string."""    def __init__(self) -> None:        llm = ChatOpenAI(model="gpt-4", temperature=0.0)        template = """Are any of the following steps unnecessary in answering {input}? Provide the verdict on a new line as a single "Y" for yes or "N" for no.        DATA        ------        Steps: {trajectory}        ------        Verdict:"""        self.chain = LLMChain.from_string(llm, template)    def _evaluate_agent_trajectory(        self,        *,        prediction: str,        input: str,        agent_trajectory: Sequence[Tuple[AgentAction, str]],        reference: Optional[str] = None,        **kwargs: Any,    ) -> dict:        vals = [            f"{i}:
Open In Colab
Open In Colab ->: Skip to main contentü¶úÔ∏èüîó LangChainDocsUse casesIntegrationsAPICommunityChat our docsLangSmithJS/TS DocsSearchCTRLKGet startedIntroductionInstallationQuickstartLangChain Expression LanguageInterfaceHow toCookbookLangChain Expression Language (LCEL)Why use LCEL?ModulesModel I/‚ÄãORetrievalChainsMemoryAgentsCallbacksModulesSecurityGuidesAdaptersDebuggingDeploymentEvaluationString EvaluatorsComparison EvaluatorsTrajectory EvaluatorsCustom Trajectory EvaluatorAgent TrajectoryExamplesFallbacksLangSmithRun LLMs locallyModel comparisonPrivacyPydantic compatibilitySafetyMoreGuidesEvaluationTrajectory EvaluatorsCustom Trajectory EvaluatorCustom Trajectory EvaluatorYou can make your own custom trajectory evaluators by inheriting from the AgentTrajectoryEvaluator class and overwriting the _evaluate_agent_trajectory (and _aevaluate_agent_action) method.In this example, you will make a simple trajectory evaluator that uses an LLM to determine if any actions were unnecessary.from typing import Any, Optional, Sequence, Tuplefrom langchain.chat_models import ChatOpenAIfrom langchain.chains import LLMChainfrom langchain.schema import AgentActionfrom langchain.evaluation import AgentTrajectoryEvaluatorclass StepNecessityEvaluator(AgentTrajectoryEvaluator):    """Evaluate the perplexity of a predicted string."""    def __init__(self) -> None:        llm = ChatOpenAI(model="gpt-4", temperature=0.0)        template = """Are any of the following steps unnecessary in answering {input}? Provide the verdict on a new line as a single "Y" for yes or "N" for no.        DATA        ------        Steps: {trajectory}        ------        Verdict:"""        self.chain = LLMChain.from_string(llm, template)    def _evaluate_agent_trajectory(        self,        *,        prediction: str,        input: str,        agent_trajectory: Sequence[Tuple[AgentAction, str]],        reference: Optional[str] = None,        **kwargs: Any,    ) -> dict:        vals = [            f"{i}: