Update app.py

app.py

@@ -7,8 +7,13 @@ import llama_cpp
 from llama_cpp import Llama
 from huggingface_hub import hf_hub_download  
 
+import transformers
+import torch
+
+
 app = FastAPI()
 
+"""
 model_path = hf_hub_download(
         repo_id="TheBloke/Mistral-7B-v0.1-GGUF",
         filename="mistral-7b-v0.1.Q4_K_M.gguf")
@@ -18,7 +23,7 @@ llm = Llama(
     n_ctx=2048,
     n_threads=2
 ) 
-
+"""
 
 @app.get("/")
 async def generate_text():
@@ -29,7 +34,6 @@ async def generate_text():
           max_tokens=32, 
           stop=["Q:", "\n"], 
           echo=True)
-        """
         output = llm.create_chat_completion(
         messages=[
         {
@@ -43,6 +47,10 @@ async def generate_text():
     },
     temperature=0.7,
 )
-        return output
+    """
+    model_id = "meta-llama/Meta-Llama-3-8B"
+    pipeline = transformers.pipeline("text-generation", model=model_id, model_kwargs={"torch_dtype": torch.bfloat16}, device_map="auto")
+    pipeline("Hey how are you doing today?")
+        return  pipeline("Hey how are you doing today?")
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))