added original code from: https://github.com/mgjinnn/TurtleSoupBaseline

TurtleSoupBaseline/openai_api_server.py

@@ -0,0 +1,549 @@
+import os
+import time
+from asyncio.log import logger
+
+import uvicorn
+import gc
+import json
+import torch
+
+from vllm import SamplingParams, AsyncEngineArgs, AsyncLLMEngine
+from fastapi import FastAPI, HTTPException, Response
+from fastapi.middleware.cors import CORSMiddleware
+from contextlib import asynccontextmanager
+from typing import List, Literal, Optional, Union
+from pydantic import BaseModel, Field
+from transformers import AutoTokenizer, LogitsProcessor
+from sse_starlette.sse import EventSourceResponse
+
+EventSourceResponse.DEFAULT_PING_INTERVAL = 1000
+MODEL_PATH = 'THUDM/glm-4-9b-chat'
+MAX_MODEL_LENGTH = 8192
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    yield
+    if torch.cuda.is_available():
+        torch.cuda.empty_cache()
+        torch.cuda.ipc_collect()
+
+
+app = FastAPI(lifespan=lifespan)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+class ModelCard(BaseModel):
+    id: str
+    object: str = "model"
+    created: int = Field(default_factory=lambda: int(time.time()))
+    owned_by: str = "owner"
+    root: Optional[str] = None
+    parent: Optional[str] = None
+    permission: Optional[list] = None
+
+
+class ModelList(BaseModel):
+    object: str = "list"
+    data: List[ModelCard] = []
+
+
+class FunctionCallResponse(BaseModel):
+    name: Optional[str] = None
+    arguments: Optional[str] = None
+
+
+class ChatMessage(BaseModel):
+    role: Literal["user", "assistant", "system", "tool"]
+    content: str = None
+    name: Optional[str] = None
+    function_call: Optional[FunctionCallResponse] = None
+
+
+class DeltaMessage(BaseModel):
+    role: Optional[Literal["user", "assistant", "system"]] = None
+    content: Optional[str] = None
+    function_call: Optional[FunctionCallResponse] = None
+
+
+class EmbeddingRequest(BaseModel):
+    input: Union[List[str], str]
+    model: str
+
+
+class CompletionUsage(BaseModel):
+    prompt_tokens: int
+    completion_tokens: int
+    total_tokens: int
+
+
+class EmbeddingResponse(BaseModel):
+    data: list
+    model: str
+    object: str
+    usage: CompletionUsage
+
+
+class UsageInfo(BaseModel):
+    prompt_tokens: int = 0
+    total_tokens: int = 0
+    completion_tokens: Optional[int] = 0
+
+
+class ChatCompletionRequest(BaseModel):
+    model: str
+    messages: List[ChatMessage]
+    temperature: Optional[float] = 0.8
+    top_p: Optional[float] = 0.8
+    max_tokens: Optional[int] = None
+    stream: Optional[bool] = False
+    tools: Optional[Union[dict, List[dict]]] = None
+    tool_choice: Optional[Union[str, dict]] = "None"
+    repetition_penalty: Optional[float] = 1.1
+
+
+class ChatCompletionResponseChoice(BaseModel):
+    index: int
+    message: ChatMessage
+    finish_reason: Literal["stop", "length", "function_call"]
+
+
+class ChatCompletionResponseStreamChoice(BaseModel):
+    delta: DeltaMessage
+    finish_reason: Optional[Literal["stop", "length", "function_call"]]
+    index: int
+
+
+class ChatCompletionResponse(BaseModel):
+    model: str
+    id: str
+    object: Literal["chat.completion", "chat.completion.chunk"]
+    choices: List[Union[ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice]]
+    created: Optional[int] = Field(default_factory=lambda: int(time.time()))
+    usage: Optional[UsageInfo] = None
+
+
+class InvalidScoreLogitsProcessor(LogitsProcessor):
+    def __call__(
+            self, input_ids: torch.LongTensor, scores: torch.FloatTensor
+    ) -> torch.FloatTensor:
+        if torch.isnan(scores).any() or torch.isinf(scores).any():
+            scores.zero_()
+            scores[..., 5] = 5e4
+        return scores
+
+
+def process_response(output: str, use_tool: bool = False) -> Union[str, dict]:
+    content = ""
+    for response in output.split("<|assistant|>"):
+        if "\n" in response:
+            metadata, content = response.split("\n", maxsplit=1)
+        else:
+            metadata, content = "", response
+        if not metadata.strip():
+            content = content.strip()
+        else:
+            if use_tool:
+                parameters = eval(content.strip())
+                content = {
+                    "name": metadata.strip(),
+                    "arguments": json.dumps(parameters, ensure_ascii=False)
+                }
+            else:
+                content = {
+                    "name": metadata.strip(),
+                    "content": content
+                }
+    return content
+
+
+@torch.inference_mode()
+async def generate_stream_glm4(params):
+    messages = params["messages"]
+    tools = params["tools"]
+    tool_choice = params["tool_choice"]
+    temperature = float(params.get("temperature", 1.0))
+    repetition_penalty = float(params.get("repetition_penalty", 1.0))
+    top_p = float(params.get("top_p", 1.0))
+    max_new_tokens = int(params.get("max_tokens", 8192))
+    messages = process_messages(messages, tools=tools, tool_choice=tool_choice)
+    inputs = tokenizer.apply_chat_template(messages, add_generation_prompt=True, tokenize=False)
+    params_dict = {
+        "n": 1,
+        "best_of": 1,
+        "presence_penalty": 1.0,
+        "frequency_penalty": 0.0,
+        "temperature": temperature,
+        "top_p": top_p,
+        "top_k": -1,
+        "repetition_penalty": repetition_penalty,
+        "use_beam_search": False,
+        "length_penalty": 1,
+        "early_stopping": False,
+        "stop_token_ids": [151329, 151336, 151338],
+        "ignore_eos": False,
+        "max_tokens": max_new_tokens,
+        "logprobs": None,
+        "prompt_logprobs": None,
+        "skip_special_tokens": True,
+    }
+    sampling_params = SamplingParams(**params_dict)
+    async for output in engine.generate(inputs=inputs, sampling_params=sampling_params, request_id="glm-4-9b"):
+        output_len = len(output.outputs[0].token_ids)
+        input_len = len(output.prompt_token_ids)
+        ret = {
+            "text": output.outputs[0].text,
+            "usage": {
+                "prompt_tokens": input_len,
+                "completion_tokens": output_len,
+                "total_tokens": output_len + input_len
+            },
+            "finish_reason": output.outputs[0].finish_reason,
+        }
+        yield ret
+    gc.collect()
+    torch.cuda.empty_cache()
+
+
+def process_messages(messages, tools=None, tool_choice="none"):
+    _messages = messages
+    messages = []
+    msg_has_sys = False
+
+    def filter_tools(tool_choice, tools):
+        function_name = tool_choice.get('function', {}).get('name', None)
+        if not function_name:
+            return []
+        filtered_tools = [
+            tool for tool in tools
+            if tool.get('function', {}).get('name') == function_name
+        ]
+        return filtered_tools
+
+    if tool_choice != "none":
+        if isinstance(tool_choice, dict):
+            tools = filter_tools(tool_choice, tools)
+        if tools:
+            messages.append(
+                {
+                    "role": "system",
+                    "content": None,
+                    "tools": tools
+                }
+            )
+            msg_has_sys = True
+
+    # add to metadata
+    if isinstance(tool_choice, dict) and tools:
+        messages.append(
+            {
+                "role": "assistant",
+                "metadata": tool_choice["function"]["name"],
+                "content": ""
+            }
+        )
+
+    for m in _messages:
+        role, content, func_call = m.role, m.content, m.function_call
+        if role == "function":
+            messages.append(
+                {
+                    "role": "observation",
+                    "content": content
+                }
+            )
+        elif role == "assistant" and func_call is not None:
+            for response in content.split("<|assistant|>"):
+                if "\n" in response:
+                    metadata, sub_content = response.split("\n", maxsplit=1)
+                else:
+                    metadata, sub_content = "", response
+                messages.append(
+                    {
+                        "role": role,
+                        "metadata": metadata,
+                        "content": sub_content.strip()
+                    }
+                )
+        else:
+            if role == "system" and msg_has_sys:
+                msg_has_sys = False
+                continue
+            messages.append({"role": role, "content": content})
+
+    return messages
+
+
+@app.get("/health")
+async def health() -> Response:
+    """Health check."""
+    return Response(status_code=200)
+
+
+@app.get("/v1/models", response_model=ModelList)
+async def list_models():
+    model_card = ModelCard(id="glm-4")
+    return ModelList(data=[model_card])
+
+
+@app.post("/v1/chat/completions", response_model=ChatCompletionResponse)
+async def create_chat_completion(request: ChatCompletionRequest):
+    if len(request.messages) < 1 or request.messages[-1].role == "assistant":
+        raise HTTPException(status_code=400, detail="Invalid request")
+
+    gen_params = dict(
+        messages=request.messages,
+        temperature=request.temperature,
+        top_p=request.top_p,
+        max_tokens=request.max_tokens or 1024,
+        echo=False,
+        stream=request.stream,
+        repetition_penalty=request.repetition_penalty,
+        tools=request.tools,
+        tool_choice=request.tool_choice,
+    )
+    logger.debug(f"==== request ====\n{gen_params}")
+
+    if request.stream:
+        predict_stream_generator = predict_stream(request.model, gen_params)
+        output = await anext(predict_stream_generator)
+        if output:
+            return EventSourceResponse(predict_stream_generator, media_type="text/event-stream")
+        logger.debug(f"First result output：\n{output}")
+
+        function_call = None
+        if output and request.tools:
+            try:
+                function_call = process_response(output, use_tool=True)
+            except:
+                logger.warning("Failed to parse tool call")
+
+        # CallFunction
+        if isinstance(function_call, dict):
+            function_call = FunctionCallResponse(**function_call)
+            tool_response = ""
+            if not gen_params.get("messages"):
+                gen_params["messages"] = []
+            gen_params["messages"].append(ChatMessage(role="assistant", content=output))
+            gen_params["messages"].append(ChatMessage(role="tool", name=function_call.name, content=tool_response))
+            generate = predict(request.model, gen_params)
+            return EventSourceResponse(generate, media_type="text/event-stream")
+        else:
+            generate = parse_output_text(request.model, output)
+            return EventSourceResponse(generate, media_type="text/event-stream")
+
+    response = ""
+    async for response in generate_stream_glm4(gen_params):
+        pass
+
+    if response["text"].startswith("\n"):
+        response["text"] = response["text"][1:]
+    response["text"] = response["text"].strip()
+
+    usage = UsageInfo()
+    function_call, finish_reason = None, "stop"
+    if request.tools:
+        try:
+            function_call = process_response(response["text"], use_tool=True)
+        except:
+            logger.warning(
+                "Failed to parse tool call, maybe the response is not a function call(such as cogview drawing) or have been answered.")
+
+    if isinstance(function_call, dict):
+        finish_reason = "function_call"
+        function_call = FunctionCallResponse(**function_call)
+
+    message = ChatMessage(
+        role="assistant",
+        content=response["text"],
+        function_call=function_call if isinstance(function_call, FunctionCallResponse) else None,
+    )
+
+    logger.debug(f"==== message ====\n{message}")
+
+    choice_data = ChatCompletionResponseChoice(
+        index=0,
+        message=message,
+        finish_reason=finish_reason,
+    )
+    task_usage = UsageInfo.model_validate(response["usage"])
+    for usage_key, usage_value in task_usage.model_dump().items():
+        setattr(usage, usage_key, getattr(usage, usage_key) + usage_value)
+
+    return ChatCompletionResponse(
+        model=request.model,
+        id="",  # for open_source model, id is empty
+        choices=[choice_data],
+        object="chat.completion",
+        usage=usage
+    )
+
+
+async def predict(model_id: str, params: dict):
+    choice_data = ChatCompletionResponseStreamChoice(
+        index=0,
+        delta=DeltaMessage(role="assistant"),
+        finish_reason=None
+    )
+    chunk = ChatCompletionResponse(model=model_id, id="", choices=[choice_data], object="chat.completion.chunk")
+    yield "{}".format(chunk.model_dump_json(exclude_unset=True))
+
+    previous_text = ""
+    async for new_response in generate_stream_glm4(params):
+        decoded_unicode = new_response["text"]
+        delta_text = decoded_unicode[len(previous_text):]
+        previous_text = decoded_unicode
+
+        finish_reason = new_response["finish_reason"]
+        if len(delta_text) == 0 and finish_reason != "function_call":
+            continue
+
+        function_call = None
+        if finish_reason == "function_call":
+            try:
+                function_call = process_response(decoded_unicode, use_tool=True)
+            except:
+                logger.warning(
+                    "Failed to parse tool call, maybe the response is not a tool call or have been answered.")
+
+        if isinstance(function_call, dict):
+            function_call = FunctionCallResponse(**function_call)
+
+        delta = DeltaMessage(
+            content=delta_text,
+            role="assistant",
+            function_call=function_call if isinstance(function_call, FunctionCallResponse) else None,
+        )
+
+        choice_data = ChatCompletionResponseStreamChoice(
+            index=0,
+            delta=delta,
+            finish_reason=finish_reason
+        )
+        chunk = ChatCompletionResponse(
+            model=model_id,
+            id="",
+            choices=[choice_data],
+            object="chat.completion.chunk"
+        )
+        yield "{}".format(chunk.model_dump_json(exclude_unset=True))
+
+    choice_data = ChatCompletionResponseStreamChoice(
+        index=0,
+        delta=DeltaMessage(),
+        finish_reason="stop"
+    )
+    chunk = ChatCompletionResponse(
+        model=model_id,
+        id="",
+        choices=[choice_data],
+        object="chat.completion.chunk"
+    )
+    yield "{}".format(chunk.model_dump_json(exclude_unset=True))
+    yield '[DONE]'
+
+
+async def predict_stream(model_id, gen_params):
+    output = ""
+    is_function_call = False
+    has_send_first_chunk = False
+    async  for new_response in generate_stream_glm4(gen_params):
+        decoded_unicode = new_response["text"]
+        delta_text = decoded_unicode[len(output):]
+        output = decoded_unicode
+
+        if not is_function_call and len(output) > 7:
+            is_function_call = output and 'get_' in output
+            if is_function_call:
+                continue
+
+            finish_reason = new_response["finish_reason"]
+            if not has_send_first_chunk:
+                message = DeltaMessage(
+                    content="",
+                    role="assistant",
+                    function_call=None,
+                )
+                choice_data = ChatCompletionResponseStreamChoice(
+                    index=0,
+                    delta=message,
+                    finish_reason=finish_reason
+                )
+                chunk = ChatCompletionResponse(
+                    model=model_id,
+                    id="",
+                    choices=[choice_data],
+                    created=int(time.time()),
+                    object="chat.completion.chunk"
+                )
+                yield "{}".format(chunk.model_dump_json(exclude_unset=True))
+
+            send_msg = delta_text if has_send_first_chunk else output
+            has_send_first_chunk = True
+            message = DeltaMessage(
+                content=send_msg,
+                role="assistant",
+                function_call=None,
+            )
+            choice_data = ChatCompletionResponseStreamChoice(
+                index=0,
+                delta=message,
+                finish_reason=finish_reason
+            )
+            chunk = ChatCompletionResponse(
+                model=model_id,
+                id="",
+                choices=[choice_data],
+                created=int(time.time()),
+                object="chat.completion.chunk"
+            )
+            yield "{}".format(chunk.model_dump_json(exclude_unset=True))
+
+    if is_function_call:
+        yield output
+    else:
+        yield '[DONE]'
+
+
+async def parse_output_text(model_id: str, value: str):
+    choice_data = ChatCompletionResponseStreamChoice(
+        index=0,
+        delta=DeltaMessage(role="assistant", content=value),
+        finish_reason=None
+    )
+    chunk = ChatCompletionResponse(model=model_id, id="", choices=[choice_data], object="chat.completion.chunk")
+    yield "{}".format(chunk.model_dump_json(exclude_unset=True))
+    choice_data = ChatCompletionResponseStreamChoice(
+        index=0,
+        delta=DeltaMessage(),
+        finish_reason="stop"
+    )
+    chunk = ChatCompletionResponse(model=model_id, id="", choices=[choice_data], object="chat.completion.chunk")
+    yield "{}".format(chunk.model_dump_json(exclude_unset=True))
+    yield '[DONE]'
+
+
+if __name__ == "__main__":
+    tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH, trust_remote_code=True)
+    engine_args = AsyncEngineArgs(
+        model=MODEL_PATH,
+        tokenizer=MODEL_PATH,
+        tensor_parallel_size=1,
+        dtype="bfloat16",
+        trust_remote_code=True,
+        gpu_memory_utilization=0.9,
+        enforce_eager=True,
+        worker_use_ray=True,
+        engine_use_ray=False,
+        disable_log_requests=True,
+        max_model_len=MAX_MODEL_LENGTH,
+    )
+    engine = AsyncLLMEngine.from_engine_args(engine_args)
+    uvicorn.run(app, host='0.0.0.0', port=8000, workers=1)

TurtleSoupBaseline/process_transform.py

@@ -0,0 +1,23 @@
+import pandas as pd
+
+data = pd.read_csv('predict_result.csv',encoding='utf8')
+
+# 做一定程度上的转换，转换不同说法但表达意思相同的答案。需写清说明。
+def trans(ans):
+
+    res = ans
+    if len(ans)<25:
+        if "是的。" in ans:
+            res = "是"
+        if "问法错误。" in ans:
+            res = "问法错误"
+        if "回答正确" in ans:
+            res = "回答正确"
+        if "不重要。" in ans:
+            res = "不重要"
+        if "不是。" in ans:
+            res = "不是"
+    return res
+data['answer'] = data['answer'].apply(lambda x: trans(x))
+
+print(f"label acc is :{len(data[data['label']==data['answer']])/len(data)}")

TurtleSoupBaseline/readme.md

@@ -0,0 +1,19 @@
+# baseline说明
+本baseline参考glm4官网:https://github.com/THUDM/GLM-4.git
+
+1、安装环境:pip install -r requirements.txt 具体环境要求请查看https://github.com/THUDM/GLM-4/blob/main/basic_demo/README.md
+
+2、启动服务端：
+```shell
+python openai_api_server.py
+```
+3、运行硬件环境：单卡24g 
+
+4、启动预测
+```shell
+python test_re.py
+```
+
+5、推理完成后，允许参考process_transform.py 文件，做一定程度上的转换，转换内容仅限于：由于模型的不稳定输出的不同说法但表达意思相同的答案，如“是的。”允许转换为“是。”
+
+6、baseline在测试集A的准确率约为64.7%

TurtleSoupBaseline/requirements.txt

@@ -0,0 +1,24 @@
+# use vllm
+# vllm>=0.4.3
+
+torch>=2.3.0
+torchvision>=0.18.0
+transformers==4.40.0
+huggingface-hub>=0.23.1
+sentencepiece>=0.2.0
+pydantic>=2.7.1
+timm>=0.9.16
+tiktoken>=0.7.0
+accelerate>=0.30.1
+sentence_transformers>=2.7.0
+
+# web demo
+gradio>=4.33.0
+
+# openai demo
+openai>=1.31.1
+einops>=0.7.0
+sse-starlette>=2.1.0
+
+# INT4
+bitsandbytes>=0.43.1

TurtleSoupBaseline/test_re.py

@@ -0,0 +1,57 @@
+from openai import OpenAI
+base_url = "http://localhost:8000/v1/"
+client = OpenAI(api_key="EMPTY", base_url=base_url)
+import time
+import pandas as pd
+
+
+test_a = pd.read_csv('test_a.csv',encoding='gbk')
+
+
+def simple_chat(sys_content,usr_content,use_stream=False):
+    messages = [
+        {
+            "role": "system",
+            "content": sys_content,
+        },
+        {
+            "role": "user",
+            "content": usr_content
+        }
+    ]
+    response = client.chat.completions.create(
+        model="glm-4",
+        messages=messages,
+        stream=use_stream,
+        max_tokens=1024,
+        temperature=0.1,
+        presence_penalty=1.1,
+        top_p=0.8)
+    if response:
+        if use_stream:
+            stream_list=[]
+            for chunk in response:
+                stream_list.append(chunk.choices[0].delta.content)
+            return stream_list
+        else:
+            content = response.choices[0].message.content
+            return content
+    else:
+        return (f"Error:, {response.status_code}")
+
+def prompt1(x,y,z):
+    sys_prom=f'''你是海龟汤出题人，我们来玩一个叫做海龟汤的游戏。海龟汤是一种情景猜谜的推理游戏。其玩法是:出题者提出一个简单又难以理解的事件，
+    玩家可以提出任何封闭式问题以试图缩小范围并找出事件背后真正的原因，封闭式问题指的是问题答案只能为："是。"或者"不是。"。如果玩家的问题不是一个封闭式问题，请回答："问法错误。"。
+    海龟汤由汤面和汤底组成，汤面指的是海龟汤的题目，汤底指的是题目背后的真相。如果用户的问题和汤面和汤底不相关，请回答："不重要。",如果用户的答案命中了汤底的核心真相，且大部分内容都得到了还原，请回答："回答正确。"。游戏过程中，你需要根据汤底、汤面、玩家的问题，以及上述规则，判断并选择以下五个选项中的一个来回答玩家提出的问题，不能给出更多的提示。你的回答选项： [是。|不是。|不重要。|问法错误。|回答正确。]。最后玩家通过这些问题和回答来逐渐找到事件的真相，以下是一份海龟汤的汤面和汤底，
+    汤面：[{x}]。汤底：[{y}]。请你扮演出题者的角色，我来扮演玩家的角色。由我先提问：'''
+    usr_prom = z
+    res = simple_chat(sys_content=sys_prom,usr_content=usr_prom)
+
+    return res
+
+t1 = time.time()
+print(f"now: {t1}")
+test_a['answer'] = test_a.apply(lambda x:prompt1(x.puzzle,x.truth,x.text),axis=1)
+print(f"cost:{time.time()-t1}")
+test_a_baseline_pre = test_a
+test_a_baseline_pre.to_csv('your_predict_result.csv',index=False)