run_name=RUN_NAME, sagemaker_session=session) as run:    # Create SageMaker Callback    sagemaker_callback = SageMakerCallbackHandler(run)    # Create prompt templates for the chain    prompt_template1 = PromptTemplate.from_template(template=PROMPT_TEMPLATE_1)    prompt_template2 = PromptTemplate.from_template(template=PROMPT_TEMPLATE_2)    # Define LLM model with callback    llm = OpenAI(callbacks=[sagemaker_callback], **HPARAMS)    # Create chain1    chain1 = LLMChain(llm=llm, prompt=prompt_template1, callbacks=[sagemaker_callback])    # Create chain2    chain2 = LLMChain(llm=llm, prompt=prompt_template2, callbacks=[sagemaker_callback])    # Create Sequential chain    overall_chain = SimpleSequentialChain(chains=[chain1, chain2], callbacks=[sagemaker_callback])    # Run overall sequential chain    overall_chain.run(**INPUT_VARIABLES)    # Reset the callback    sagemaker_callback.flush_tracker()Scenario 3 - Agent with Tools‚ÄãRUN_NAME = "run-scenario-3"PROMPT_TEMPLATE = "Who is the oldest person alive? And what is their current age raised to the power of 1.51?"with Run(experiment_name=EXPERIMENT_NAME, run_name=RUN_NAME, sagemaker_session=session) as run:    # Create SageMaker Callback    sagemaker_callback = SageMakerCallbackHandler(run)    # Define LLM model with callback    llm = OpenAI(callbacks=[sagemaker_callback], **HPARAMS)    # Define tools    tools = load_tools(["serpapi", "llm-math"], llm=llm, callbacks=[sagemaker_callback])    # Initialize agent with all the tools    agent = initialize_agent(tools, llm, agent="zero-shot-react-description", callbacks=[sagemaker_callback])    # Run agent    agent.run(input=PROMPT_TEMPLATE)    # Reset the callback    sagemaker_callback.flush_tracker()Load Log Data‚ÄãOnce the prompts are logged, we can easily load and convert them to Pandas DataFrame as follows.#Loadlogs = ExperimentAnalytics(experiment_name=EXPERIMENT_NAME)#Convert as pandas dataframedf = logs.dataframe(force_refresh=True)print(df.shape)df.head()As can be
This notebook shows how LangChain Callback can be used to log and track prompts and other LLM hyperparameters into SageMaker Experiments. Here, we use different scenarios to showcase the capability:
This notebook shows how LangChain Callback can be used to log and track prompts and other LLM hyperparameters into SageMaker Experiments. Here, we use different scenarios to showcase the capability: ->: run_name=RUN_NAME, sagemaker_session=session) as run:    # Create SageMaker Callback    sagemaker_callback = SageMakerCallbackHandler(run)    # Create prompt templates for the chain    prompt_template1 = PromptTemplate.from_template(template=PROMPT_TEMPLATE_1)    prompt_template2 = PromptTemplate.from_template(template=PROMPT_TEMPLATE_2)    # Define LLM model with callback    llm = OpenAI(callbacks=[sagemaker_callback], **HPARAMS)    # Create chain1    chain1 = LLMChain(llm=llm, prompt=prompt_template1, callbacks=[sagemaker_callback])    # Create chain2    chain2 = LLMChain(llm=llm, prompt=prompt_template2, callbacks=[sagemaker_callback])    # Create Sequential chain    overall_chain = SimpleSequentialChain(chains=[chain1, chain2], callbacks=[sagemaker_callback])    # Run overall sequential chain    overall_chain.run(**INPUT_VARIABLES)    # Reset the callback    sagemaker_callback.flush_tracker()Scenario 3 - Agent with Tools‚ÄãRUN_NAME = "run-scenario-3"PROMPT_TEMPLATE = "Who is the oldest person alive? And what is their current age raised to the power of 1.51?"with Run(experiment_name=EXPERIMENT_NAME, run_name=RUN_NAME, sagemaker_session=session) as run:    # Create SageMaker Callback    sagemaker_callback = SageMakerCallbackHandler(run)    # Define LLM model with callback    llm = OpenAI(callbacks=[sagemaker_callback], **HPARAMS)    # Define tools    tools = load_tools(["serpapi", "llm-math"], llm=llm, callbacks=[sagemaker_callback])    # Initialize agent with all the tools    agent = initialize_agent(tools, llm, agent="zero-shot-react-description", callbacks=[sagemaker_callback])    # Run agent    agent.run(input=PROMPT_TEMPLATE)    # Reset the callback    sagemaker_callback.flush_tracker()Load Log Data‚ÄãOnce the prompts are logged, we can easily load and convert them to Pandas DataFrame as follows.#Loadlogs = ExperimentAnalytics(experiment_name=EXPERIMENT_NAME)#Convert as pandas dataframedf = logs.dataframe(force_refresh=True)print(df.shape)df.head()As can be