Upload folder using huggingface_hub

Test_RAG.py

@@ -303,25 +303,39 @@ if llm_model_id == "red-pajama-3b-chat" and "GPU" in core.available_devices and
 #     pipeline_kwargs={"max_new_tokens": 2},
 # )
 from optimum.intel.openvino import OVModelForCausalLM
+from transformers import pipeline
 
-llm = OVModelForCausalLM.from_pretrained(
-    model_id = "meta-llama/Meta-Llama-3-8B",
+
+model_id = "meta-llama/Meta-Llama-3-8B"
+ov_config = {"PERFORMANCE_HINT": "LATENCY"}  # 这是一个例子，检查你的实际 ov_config
+
+# 使用 OpenVINO 导出模型
+model = OVModelForCausalLM.from_pretrained(
+    model_id,
     export=True,  # 将模型转换为 OpenVINO 格式
     use_cache=False,
     ov_config=ov_config,
     trust_remote_code=True  # 支持远程代码的信任问题
 )
 
+# 保存 OpenVINO 模型
+model.save_pretrained("./openvino_llama_model")
+
+# Step 2: 加载保存的 OpenVINO 模型并设置推理任务
+llm_device = "CPU"  # 确保你根据环境设置正确的设备
+llm = pipeline(
+    task="text-generation",
+    model=OVModelForCausalLM.from_pretrained("./openvino_llama_model"),
+    device=llm_device,
+    max_new_tokens=2  # 生成的最大新token数量
+)
+
+# Step 3: 执行推理
+output = llm("2 + 2 =")
+print(output)
 
-# # 设置 pad_token_id 为 eos_token_id
-# from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
-# tokenizer = AutoTokenizer.from_pretrained('meta-llama/Llama-3.2-3B-Instruct')
-# if tokenizer.pad_token_id is None:
-#     tokenizer.pad_token_id = tokenizer.eos_token_id
-# # # 同时确保 HuggingFacePipeline 使用的 tokenizer 也设置了 pad_token_id
-# llm.pipeline.tokenizer.pad_token_id = tokenizer.pad_token_id
-print("test:2+2:")
-print(llm.invoke("2 + 2 ="))
+# print("test:2+2:")
+# print(llm.invoke("2 + 2 ="))
 import re
 from typing import List
 from langchain.text_splitter import (