Update app.py

app.py

@@ -1,14 +1,20 @@
-import gradio as gr
+from fastapi import FastAPI
+from pydantic import BaseModel
 from transformers import AutoModelForCausalLM, AutoTokenizer
 
+app = FastAPI()
+
 model_name = "Xennus/niko-mistral-cyberbot"
 tokenizer = AutoTokenizer.from_pretrained(model_name)
 model = AutoModelForCausalLM.from_pretrained(model_name)
+model.eval()
 
-def generate_response(prompt):
-    inputs = tokenizer(prompt, return_tensors="pt")
-    outputs = model.generate(**inputs, max_length=200)
-    return tokenizer.decode(outputs[0], skip_special_tokens=True)
+class Request(BaseModel):
+    prompt: str
 
-iface = gr.Interface(fn=generate_response, inputs="text", outputs="text")
-iface.launch()
+@app.post("/generate")
+async def generate(request: Request):
+    inputs = tokenizer(request.prompt, return_tensors="pt")
+    outputs = model.generate(**inputs, max_length=200)
+    text = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    return {"response": text}