Download deploy/aligner_inference_demo.py from alignmentforever/huawei_aligner_v2: direct link, hf CLI and curl.
- Browser
- Download file 5.96 kB
-
https://huggingface.co/alignmentforever/huawei_aligner_v2/resolve/main/deploy/aligner_inference_demo.py
- Command line
-
hf download hf://alignmentforever/huawei_aligner_v2/deploy/aligner_inference_demo.py
-
curl -L -o aligner_inference_demo.py https://huggingface.co/alignmentforever/huawei_aligner_v2/resolve/main/deploy/aligner_inference_demo.py
5.96 kB
| # Copyright 2024 PKU-Alignment Team. All Rights Reserved. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| # ============================================================================== | |
| """访问文本模型的命令行界面""" | |
| import argparse | |
| import os | |
| from openai import OpenAI | |
| import gradio as gr | |
| import random | |
| random.seed(42) | |
| CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) | |
| # 系统提示词,可以根据需要修改 | |
| SYSTEM_PROMPT = "你是一个有帮助的AI助手,能够回答用户的问题并提供帮助。" | |
| # 连接设置 | |
| openai_api_key = "jiayi" # 不重要,仅用于初始化客户端 | |
| aligner_port = 8013 | |
| base_port = 8011 | |
| aligner_api_base = f"http://0.0.0.0:{aligner_port}/v1" | |
| base_api_base = f"http://0.0.0.0:{base_port}/v1" | |
| # openai_api_base = "http://0.0.0.0:8009/v1" # 请修改为实际的模型API端口 | |
| # NOTE please modify the model path | |
| aligner_model = "" | |
| base_model = "" | |
| aligner_client = OpenAI( | |
| api_key = openai_api_key, | |
| base_url = aligner_api_base, | |
| ) | |
| base_client = OpenAI( | |
| api_key = openai_api_key, | |
| base_url = base_api_base, | |
| ) | |
| # 示例问题 | |
| # TEXT_EXAMPLES = [ | |
| # {"text": "介绍一下北京大学的历史"}, | |
| # {"text": "解释一下什么是深度学习"}, | |
| # {"text": "写一首关于春天的诗"}, | |
| # ] | |
| TEXT_EXAMPLES = [ | |
| "介绍一下北京大学的历史", | |
| "解释一下什么是深度学习", | |
| "写一首关于春天的诗", | |
| ] | |
| # # 初始化OpenAI客户端 | |
| # client = OpenAI( | |
| # api_key=openai_api_key, | |
| # base_url=openai_api_base, | |
| # ) | |
| def text_conversation(text: str, role: str = 'user'): | |
| """创建单条文本消息""" | |
| return [{'role': role, 'content': text}] | |
| def question_answering(message: str, history: list): | |
| """处理文本问答(流式输出)""" | |
| conversation = text_conversation(SYSTEM_PROMPT, 'system') | |
| # 处理历史对话记录 | |
| for past_user_msg, past_bot_msg in history: | |
| if past_user_msg: | |
| conversation.extend(text_conversation(past_user_msg, 'user')) | |
| if past_bot_msg: | |
| conversation.extend(text_conversation(past_bot_msg, 'assistant')) | |
| # 添加当前问题 | |
| current_question = message | |
| conversation.extend(text_conversation(current_question)) | |
| # 调用模型API(启用流式输出) | |
| stream = base_client.chat.completions.create( | |
| model=base_model, | |
| stream=True, | |
| messages=conversation, | |
| ) | |
| # 流式输出处理 | |
| total_answer = "" | |
| base_section = "🌟 **原始回答:**\n" | |
| total_answer += base_section | |
| # NOTE 额外用一个base_answer 作为aligner的输入,其他的可以用total_answer 做总的输出 | |
| base_answer = "" | |
| yield total_answer | |
| for chunk in stream: | |
| if chunk.choices[0].delta.content is not None: | |
| base_answer += chunk.choices[0].delta.content | |
| total_answer += chunk.choices[0].delta.content | |
| yield f"```bash\n{base_section}{base_answer}\n```" | |
| # 结束原始回答部分,开始aligner部分 | |
| aligner_section = "\n**Aligner 修正中...**\n\n🌟 **修正后回答:**\n" | |
| # 创建新的total_answer,不再包含在bash格式中 | |
| total_answer = f"```bash\n{base_section}{base_answer}\n```{aligner_section}" | |
| yield total_answer | |
| aligner_conversation = text_conversation(SYSTEM_PROMPT,'system') | |
| aligner_current_question = f'##Question: {current_question}\n##Answer: {base_answer}\n##Correction: ' | |
| aligner_conversation.extend(text_conversation(aligner_current_question)) | |
| aligner_stream = aligner_client.chat.completions.create( | |
| model=aligner_model, | |
| stream=True, | |
| messages=aligner_conversation, | |
| ) | |
| aligner_answer = "" | |
| for chunk in aligner_stream: | |
| if chunk.choices[0].delta.content is not None: | |
| aligner_answer += chunk.choices[0].delta.content | |
| aligner_answer = aligner_answer.replace('##CORRECTION:', '') | |
| yield f"```bash\n{base_section}{base_answer}\n```{aligner_section}{aligner_answer}" | |
| # print('answer:', answer) | |
| # print('current question:', current_question) | |
| # # 可选:格式化回答(在流式输出完成后处理) | |
| # if "**Final Answer**" in answer: | |
| # reasoning_content, final_answer = answer.split("**Final Answer**", 1) | |
| # if len(reasoning_content) > 5: | |
| # answer = f"""🤔 思考过程:\n```bash{reasoning_content}\n```\n✨ 最终答案:\n{final_answer}""" | |
| # yield answer | |
| if __name__ == '__main__': | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--port", type=int, default=7860, help="Gradio服务端口") | |
| parser.add_argument("--share", default='True',action="store_true", help="是否创建公共链接") | |
| parser.add_argument("--api-only", default='False',action="store_true", help="只输出Python API调用示例") | |
| args = parser.parse_args() | |
| # if args.api_only: | |
| # print("Python API调用示例输出:") | |
| # print(python_api_example()) | |
| # else: | |
| # 创建Gradio界面(启用流式输出) | |
| iface = gr.ChatInterface( | |
| fn=question_answering, | |
| title='Aligner', | |
| description='网络安全 Aligner', | |
| examples=TEXT_EXAMPLES, | |
| theme=gr.themes.Soft( | |
| text_size='lg', | |
| spacing_size='lg', | |
| radius_size='lg', | |
| ), | |
| ) | |
| iface.launch(server_port=args.port, share=args.share) |