cluster# gpu = rh.cluster(ips=['<ip of the cluster>'],#                  ssh_creds={'ssh_user': '...', 'ssh_private_key':'<path_to_key>'},#                  name='rh-a10x')template = """Question: {question}Answer: Let's think step by step."""prompt = PromptTemplate(template=template, input_variables=["question"])llm = SelfHostedHuggingFaceLLM(    model_id="gpt2", hardware=gpu, model_reqs=["pip:./", "transformers", "torch"])llm_chain = LLMChain(prompt=prompt, llm=llm)question = "What NFL team won the Super Bowl in the year Justin Beiber was born?"llm_chain.run(question)    INFO | 2023-02-17 05:42:23,537 | Running _generate_text via gRPC    INFO | 2023-02-17 05:42:24,016 | Time to send message: 0.48 seconds    "\n\nLet's say we're talking sports teams who won the Super Bowl in the year Justin Beiber"You can also load more custom models through the SelfHostedHuggingFaceLLM interface:llm = SelfHostedHuggingFaceLLM(    model_id="google/flan-t5-small",    task="text2text-generation",    hardware=gpu,)llm("What is the capital of Germany?")    INFO | 2023-02-17 05:54:21,681 | Running _generate_text via gRPC    INFO | 2023-02-17 05:54:21,937 | Time to send message: 0.25 seconds    'berlin'Using a custom load function, we can load a custom pipeline directly on the remote hardware:def load_pipeline():    from transformers import (        AutoModelForCausalLM,        AutoTokenizer,        pipeline,    )  # Need to be inside the fn in notebooks    model_id = "gpt2"    tokenizer = AutoTokenizer.from_pretrained(model_id)    model = AutoModelForCausalLM.from_pretrained(model_id)    pipe = pipeline(        "text-generation", model=model, tokenizer=tokenizer, max_new_tokens=10    )    return pipedef inference_fn(pipeline, prompt, stop=None):    return pipeline(prompt)[0]["generated_text"][len(prompt) :]llm = SelfHostedHuggingFaceLLM(    model_load_fn=load_pipeline, hardware=gpu, inference_fn=inference_fn)llm("Who is the current US president?")    INFO | 2023-02-17 05:42:59,219 |
The Runhouse allows remote compute and data across environments and users. See the Runhouse docs.
The Runhouse allows remote compute and data across environments and users. See the Runhouse docs. ->: cluster# gpu = rh.cluster(ips=['<ip of the cluster>'],#                  ssh_creds={'ssh_user': '...', 'ssh_private_key':'<path_to_key>'},#                  name='rh-a10x')template = """Question: {question}Answer: Let's think step by step."""prompt = PromptTemplate(template=template, input_variables=["question"])llm = SelfHostedHuggingFaceLLM(    model_id="gpt2", hardware=gpu, model_reqs=["pip:./", "transformers", "torch"])llm_chain = LLMChain(prompt=prompt, llm=llm)question = "What NFL team won the Super Bowl in the year Justin Beiber was born?"llm_chain.run(question)    INFO | 2023-02-17 05:42:23,537 | Running _generate_text via gRPC    INFO | 2023-02-17 05:42:24,016 | Time to send message: 0.48 seconds    "\n\nLet's say we're talking sports teams who won the Super Bowl in the year Justin Beiber"You can also load more custom models through the SelfHostedHuggingFaceLLM interface:llm = SelfHostedHuggingFaceLLM(    model_id="google/flan-t5-small",    task="text2text-generation",    hardware=gpu,)llm("What is the capital of Germany?")    INFO | 2023-02-17 05:54:21,681 | Running _generate_text via gRPC    INFO | 2023-02-17 05:54:21,937 | Time to send message: 0.25 seconds    'berlin'Using a custom load function, we can load a custom pipeline directly on the remote hardware:def load_pipeline():    from transformers import (        AutoModelForCausalLM,        AutoTokenizer,        pipeline,    )  # Need to be inside the fn in notebooks    model_id = "gpt2"    tokenizer = AutoTokenizer.from_pretrained(model_id)    model = AutoModelForCausalLM.from_pretrained(model_id)    pipe = pipeline(        "text-generation", model=model, tokenizer=tokenizer, max_new_tokens=10    )    return pipedef inference_fn(pipeline, prompt, stop=None):    return pipeline(prompt)[0]["generated_text"][len(prompt) :]llm = SelfHostedHuggingFaceLLM(    model_load_fn=load_pipeline, hardware=gpu, inference_fn=inference_fn)llm("Who is the current US president?")    INFO | 2023-02-17 05:42:59,219 |