|
import gradio as gr |
|
from huggingface_hub import InferenceClient |
|
from optimum.intel import OVModelForCausalLM |
|
from transformers import AutoTokenizer, pipeline |
|
|
|
|
|
model_id = "HelloSun/Qwen2.5-0.5B-Instruct-openvino" |
|
model = OVModelForCausalLM.from_pretrained(model_id) |
|
tokenizer = AutoTokenizer.from_pretrained(model_id) |
|
|
|
|
|
pipe = pipeline("text-generation", model=model, tokenizer=tokenizer) |
|
|
|
def respond(message, history): |
|
|
|
input_text = message if not history else history[-1]["content"] + " " + message |
|
|
|
|
|
response = pipe(input_text, max_length=100, truncation=True, num_return_sequences=1) |
|
reply = response[0]['generated_text'] |
|
|
|
|
|
return [{"role": "user", "content": message}, {"role": "assistant", "content": reply}], history + [{"role": "user", "content": message}, {"role": "assistant", "content": reply}] |
|
|
|
|
|
demo = gr.ChatInterface(fn=respond, title="Chat with Qwen 2.5", description="與 HelloSun/Qwen2.5-0.5B-Instruct-openvino 聊天!", type='messages') |
|
|
|
if __name__ == "__main__": |
|
demo.launch() |
|
|