diff --git a/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py
index faa6f653b835c..18816cd665b3e 100644
--- a/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py
+++ b/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py
@@ -12,8 +12,6 @@
                                               FunctionCall, ToolCall)
 from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
     ToolParser, ToolParserManager)
-from vllm.entrypoints.openai.tool_parsers.utils import (
-    extract_intermediate_diff)
 from vllm.logger import init_logger
 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
 from vllm.utils import random_uuid
@@ -190,8 +188,11 @@ def extract_tool_calls_streaming(
                 diff = self.prev_tool_call_arr[self.current_tool_id].get(
                     "arguments")
                 if diff:
-                    diff = json.dumps(diff).replace(
-                        self.streamed_args_for_tool[self.current_tool_id], "")
+                    diff = diff.encode('utf-8').decode(
+                        'unicode_escape') if diff is str else diff
+                    diff = json.dumps(
+                        diff, ensure_ascii=False
+                    )[len(self.streamed_args_for_tool[self.current_tool_id]):]
                     logger.debug(
                         "Finishing tool and found diff that had not "
                         "been streamed yet: %s", diff)
@@ -307,22 +308,20 @@ def extract_tool_calls_streaming(

             # last case -- we have an update to existing arguments.
             elif cur_arguments and prev_arguments:
+                if isinstance(delta_text, str) and len(delta_text.rstrip(
+                )) >= 1 and delta_text.rstrip()[-1] == '}':
+                    delta_text = delta_text.rstrip()[:-1]
+
+                logger.debug("got diff %s", delta_text)

-                cur_args_json = json.dumps(cur_arguments)
-                prev_args_json = json.dumps(prev_arguments)
-                logger.debug("Searching for diff between\n%s", cur_args_json)
-                logger.debug("and\n%s", prev_args_json)
-                argument_diff = extract_intermediate_diff(
-                    cur_args_json, prev_args_json)
-                logger.debug("got argument diff %s", argument_diff)
                 delta = DeltaMessage(tool_calls=[
                     DeltaToolCall(index=self.current_tool_id,
                                   function=DeltaFunctionCall(
-                                      arguments=argument_diff).model_dump(
+                                      arguments=delta_text).model_dump(
                                           exclude_none=True))
                 ])
                 self.streamed_args_for_tool[self.current_tool_id] \
-                    += argument_diff
+                    += delta_text

             # handle saving the state for the current tool into
             # the "prev" list for use in diffing for the next iteration

[Bug]: Hermes tool parser output error stream arguments in some cases.
### Your current environment

The output of `python collect_env.py`

```text
WARNING 11-16 23:18:45 _custom_ops.py:20] Failed to import from vllm._C with ModuleNotFoundError("No module named 'vllm._C'")
/home/oem/repo/vllm/vllm/connections.py:8: RuntimeWarning: Failed to read commit hash:
No module named 'vllm._version'
  from vllm.version import __version__ as VLLM_VERSION
Collecting environment information...
/home/oem/anaconda3/envs/vllm-test/lib/python3.10/site-packages/torch/cuda/__init__.py:129: UserWarning: CUDA initialization: The NVIDIA driver on your system is too old (found version 11040). Please update your GPU driver by downloading and installing a new version from the URL: http://www.nvidia.com/Download/index.aspx Alternatively, go to: https://pytorch.org to install a PyTorch version that has been compiled with your version of the CUDA driver. (Triggered internally at ../c10/cuda/CUDAFunctions.cpp:108.)
  return torch._C._cuda_getDeviceCount() > 0
PyTorch version: 2.5.1+cu124
Is debug build: False
CUDA used to build PyTorch: 12.4
ROCM used to build PyTorch: N/A

OS: Ubuntu 20.04.6 LTS (x86_64)
GCC version: (Ubuntu 9.4.0-1ubuntu1~20.04.2) 9.4.0
Clang version: Could not collect
CMake version: Could not collect
Libc version: glibc-2.31

Python version: 3.10.15 (main, Oct  3 2024, 07:27:34) [GCC 11.2.0] (64-bit runtime)
Python platform: Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.31
Is CUDA available: False
CUDA runtime version: No CUDA
CUDA_MODULE_LOADING set to: N/A
GPU models and configuration: No CUDA
Nvidia driver version: No CUDA
cuDNN version: No CUDA
HIP runtime version: N/A
MIOpen runtime version: N/A
Is XNNPACK available: True

CPU:
Architecture:                    x86_64
CPU op-mode(s):                  32-bit, 64-bit
Byte Order:                      Little Endian
Address sizes:                   46 bits physical, 48 bits virtual
CPU(s):                          20
On-line CPU(s) list:             0-19
Thread(s) per core:              2
Core(s) per socket:              10
Socket(s):                       1
Vendor ID:                       GenuineIntel
CPU family:                      6
Model:                           154
Model name:                      12th Gen Intel(R) Core(TM) i9-12900H
Stepping:                        3
CPU MHz:                         2918.408
BogoMIPS:                        5836.81
Virtualization:                  VT-x
Hypervisor vendor:               Microsoft
Virtualization type:             full
L1d cache:                       480 KiB
L1i cache:                       320 KiB
L2 cache:                        12.5 MiB
L3 cache:                        24 MiB
Vulnerability Itlb multihit:     Not affected
Vulnerability L1tf:              Not affected
Vulnerability Mds:               Not affected
Vulnerability Meltdown:          Not affected
Vulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp
Vulnerability Spectre v1:        Mitigation; usercopy/swapgs barriers and __user pointer sanitization
Vulnerability Spectre v2:        Mitigation; Enhanced IBRS, IBPB conditional, RSB filling
Vulnerability Srbds:             Not affected
Vulnerability Tsx async abort:   Not affected
Flags:                           fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology tsc_reliable nonstop_tsc cpuid pni pclmulqdq vmx ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves umip waitpkg gfni vaes vpclmulqdq rdpid movdiri movdir64b fsrm md_clear serialize flush_l1d arch_capabilities

Versions of relevant libraries:
[pip3] mypy==1.11.1
[pip3] mypy-extensions==1.0.0
[pip3] numpy==1.26.4
[pip3] nvidia-cublas-cu12==12.4.5.8
[pip3] nvidia-cuda-cupti-cu12==12.4.127
[pip3] nvidia-cuda-nvrtc-cu12==12.4.127
[pip3] nvidia-cuda-runtime-cu12==12.4.127
[pip3] nvidia-cudnn-cu12==9.1.0.70
[pip3] nvidia-cufft-cu12==11.2.1.3
[pip3] nvidia-curand-cu12==10.3.5.147
[pip3] nvidia-cusolver-cu12==11.6.1.9
[pip3] nvidia-cusparse-cu12==12.3.1.170
[pip3] nvidia-nccl-cu12==2.21.5
[pip3] nvidia-nvjitlink-cu12==12.4.127
[pip3] nvidia-nvtx-cu12==12.4.127
[pip3] sentence-transformers==3.2.1
[pip3] torch==2.5.1
[pip3] torchvision==0.20.1
[pip3] transformers==4.45.2
[pip3] transformers-stream-generator==0.0.5
[pip3] triton==3.1.0
[conda] numpy                     1.26.4                   pypi_0    pypi
[conda] nvidia-cublas-cu12        12.4.5.8                 pypi_0    pypi
[conda] nvidia-cuda-cupti-cu12    12.4.127                 pypi_0    pypi
[conda] nvidia-cuda-nvrtc-cu12    12.4.127                 pypi_0    pypi
[conda] nvidia-cuda-runtime-cu12  12.4.127                 pypi_0    pypi
[conda] nvidia-cudnn-cu12         9.1.0.70                 pypi_0    pypi
[conda] nvidia-cufft-cu12         11.2.1.3                 pypi_0    pypi
[conda] nvidia-curand-cu12        10.3.5.147               pypi_0    pypi
[conda] nvidia-cusolver-cu12      11.6.1.9                 pypi_0    pypi
[conda] nvidia-cusparse-cu12      12.3.1.170               pypi_0    pypi
[conda] nvidia-nccl-cu12          2.21.5                   pypi_0    pypi
[conda] nvidia-nvjitlink-cu12     12.4.127                 pypi_0    pypi
[conda] nvidia-nvtx-cu12          12.4.127                 pypi_0    pypi
[conda] sentence-transformers     3.2.1                    pypi_0    pypi
[conda] torch                     2.5.1                    pypi_0    pypi
[conda] torchvision               0.20.1                   pypi_0    pypi
[conda] transformers              4.45.2                   pypi_0    pypi
[conda] transformers-stream-generator 0.0.5                    pypi_0    pypi
[conda] triton                    3.1.0                    pypi_0    pypi
ROCM Version: Could not collect
Neuron SDK Version: N/A
vLLM Version: N/A (dev)
vLLM Build Flags:
CUDA Archs: Not Set; ROCm: Disabled; Neuron: Disabled
GPU Topology:
Could not collect

LD_LIBRARY_PATH=/home/oem/anaconda3/envs/vllm-test/lib/python3.10/site-packages/cv2/../../lib64:

```

### 🐛 Describe the bug

The `extract_tool_calls_streaming` component of the Hermes tool parser generates error arguments during the parsing of streaming tool function outputs when the LLM produces a specific output content.

### LLM
`Qwen2.5-72B-Instruct-AWQ`

The docker-compose file to run the LLM:

```
version: '3.8'

services:
  Qwen72BAWQ:
    image: vllm/vllm-openai:v0.6.3.post1
    entrypoint: ["python3","-u", "-m","vllm.entrypoints.openai.api_server", "--served-model-name","Qwen2.5-72B-Instruct-AWQ","--model","/data/models/Qwen2.5-72B-Instruct-AWQ", "--enable-auto-tool-choice", "--tool-call-parser", "hermes", "--enforce-eager",  "--tensor-parallel-size", "1",   "--gpu_memory_utilization", "0.97" ]

    environment:
      - "CUDA_VISIBLE_DEVICES=1"
    restart: always
    ports:
      - "8000:8000"
    volumes:
      - /data/models:/data/models
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
```

vLLM server log

```

  | INFO 11-16 08:15:10 logger.py:37] Received request chat-a183cc0045ba4201833c24cd2be5c43e: prompt: '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{"type": "function", "function": {"name": "get_current_weather", "description": "Get the current weather in a given location", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city to find the weather for, e.g. \'San Francisco\'"}, "state": {"type": "string", "description": "the two-letter abbreviation for the state that the city is in, e.g. \'CA\' which would mean \'California\'"}, "unit": {"type": "string", "description": "The unit to fetch the temperature in", "enum": ["celsius", "fahrenheit"]}}, "required": ["city", "state", "unit"]}}}\n{"type": "function", "function": {"name": "predict_x", "description": "predict x using wd, sd, injection and module_id.", "parameters": {"type": "object", "properties": {"wd": {"type": "number", "description": "shi du"}, "sd": {"type": "number", "description": "wen du"}, "injection": {"type": "string", "description": "injection type"}, "module_id": {"type": "string", "description": "module id"}}, "required": ["wd", "sd", "injection", "module_id"]}}}\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{"name": <function-name>, "arguments": <args-json-object>}\n</tool_call><|im_end|>\n<|im_start|>user\nHi! How are you doing today?<|im_end|>\n<|im_start|>assistant\nI\'m doing well! How can I help you?<|im_end|>\n<|im_start|>user\nwhen wd=23, sd=0.8, injection=ZSJB39, module_id=M62121G9142, what\'s x ? <|im_end|>\n<|im_start|>assistant\n', params: SamplingParams(n=1, presence_penalty=0.0, frequency_penalty=0.0, repetition_penalty=1.0, temperature=0.7, top_p=1.0, top_k=-1, min_p=0.0, seed=None, stop=[], stop_token_ids=[], include_stop_str_in_output=False, ignore_eos=False, max_tokens=32306, min_tokens=0, logprobs=None, prompt_logprobs=None, skip_special_tokens=True, spaces_between_special_tokens=True, truncate_prompt_tokens=None), guided_decoding=GuidedDecodingParams(json=None, regex=None, choice=None, grammar=None, json_object=None, backend=None, whitespace_pattern=None), prompt_token_ids: [......], lora_request: None, prompt_adapter_request: None.
  | DEBUG 11-16 08:15:10 async_llm_engine.py:523] Building guided decoding logits processor. Params: GuidedDecodingParams(json=None, regex=None, choice=None, grammar=None, json_object=None, backend=None, whitespace_pattern=None)
  | DEBUG 11-16 08:15:11 client.py:154] Heartbeat successful.
  | INFO 11-16 08:15:11 engine.py:290] Added request chat-a183cc0045ba4201833c24cd2be5c43e.
  | INFO 11-16 08:15:11 metrics.py:349] Avg prompt throughput: 65.4 tokens/s, Avg generation throughput: 0.1 tokens/s, Running: 1 reqs, Swapped: 0 reqs, Pending: 0 reqs, GPU KV cache usage: 0.5%, CPU KV cache usage: 0.0%.
  | DEBUG 11-16 08:15:12 client.py:154] Heartbeat successful.
  | DEBUG 11-16 08:15:13 llm_engine.py:1460] Stopping remote worker execution loop.
  | INFO:     10.182.93.184:54706 - "POST /v1/chat/completions HTTP/1.1" 200 OK
  | INFO 11-16 08:15:14 logger.py:37] Received request chat-5fca4945ed9844eabbe21db8e2de215d: prompt: '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n{"type": "function", "function": {"name": "get_current_weather", "description": "Get the current weather in a given location", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city to find the weather for, e.g. \'San Francisco\'"}, "state": {"type": "string", "description": "the two-letter abbreviation for the state that the city is in, e.g. \'CA\' which would mean \'California\'"}, "unit": {"type": "string", "description": "The unit to fetch the temperature in", "enum": ["celsius", "fahrenheit"]}}, "required": ["city", "state", "unit"]}}}\n{"type": "function", "function": {"name": "predict_x", "description": "predict x using wd, sd, injection and module_id.", "parameters": {"type": "object", "properties": {"wd": {"type": "number", "description": "shi du"}, "sd": {"type": "number", "description": "wen du"}, "injection": {"type": "string", "description": "injection type"}, "module_id": {"type": "string", "description": "module id"}}, "required": ["wd", "sd", "injection", "module_id"]}}}\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{"name": <function-name>, "arguments": <args-json-object>}\n</tool_call><|im_end|>\n<|im_start|>user\nHi! How are you doing today?<|im_end|>\n<|im_start|>assistant\nI\'m doing well! How can I help you?<|im_end|>\n<|im_start|>user\nwhen wd=23, sd=0.8, injection=ZSJB39, module_id=M62121G9142, what\'s x ? <|im_end|>\n<|im_start|>assistant\n', params: SamplingParams(n=1, presence_penalty=0.0, frequency_penalty=0.0, repetition_penalty=1.0, temperature=0.7, top_p=1.0, top_k=-1, min_p=0.0, seed=None, stop=[], stop_token_ids=[], include_stop_str_in_output=False, ignore_eos=False, max_tokens=32306, min_tokens=0, logprobs=None, prompt_logprobs=None, skip_special_tokens=True, spaces_between_special_tokens=True, truncate_prompt_tokens=None), guided_decoding=GuidedDecodingParams(json=None, regex=None, choice=None, grammar=None, json_object=None, backend=None, whitespace_pattern=None), prompt_token_ids: [......], lora_request: None, prompt_adapter_request: None.
  | INFO:     10.182.93.184:54706 - "POST /v1/chat/completions HTTP/1.1" 200 OK
async_llm_engine.py:523] Building guided decoding logits processor. Params: GuidedDecodingParams(json=None, regex=None, choice=None, grammar=None, json_object=None, backend=None, whitespace_pattern=None)
  | INFO 11-16 08:15:14 engine.py:290] Added request chat-5fca4945ed9844eabbe21db8e2de215d.
hermes_tool_parser.py:124] delta_text: <tool_call>
hermes_tool_parser.py:125] delta_token_ids: [151657]
hermes_tool_parser.py:175] Starting on a new tool 0
hermes_tool_parser.py:218] Parsed tool call None
  | ERROR 11-16 08:15:14 hermes_tool_parser.py:337] Error trying to handle streaming tool call.
  | ERROR 11-16 08:15:14 hermes_tool_parser.py:337] Traceback (most recent call last):
  | ERROR 11-16 08:15:14 hermes_tool_parser.py:337]   File "/usr/local/lib/python3.12/dist-packages/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py", line 226, in extract_tool_calls_streaming
  | ERROR 11-16 08:15:14 hermes_tool_parser.py:337]     function_name: Union[str, None] = current_tool_call.get("name")
  | ERROR 11-16 08:15:14 hermes_tool_parser.py:337]                                       ^^^^^^^^^^^^^^^^^^^^^
  | ERROR 11-16 08:15:14 hermes_tool_parser.py:337] AttributeError: 'NoneType' object has no attribute 'get'
hermes_tool_parser.py:124] delta_text:
hermes_tool_parser.py:124]
hermes_tool_parser.py:125] delta_token_ids: [198]
hermes_tool_parser.py:220] not enough tokens to parse into JSON yet
hermes_tool_parser.py:124] delta_text: {"
hermes_tool_parser.py:125] delta_token_ids: [4913]
hermes_tool_parser.py:218] Parsed tool call {}
hermes_tool_parser.py:124] delta_text: name
hermes_tool_parser.py:125] delta_token_ids: [606]
hermes_tool_parser.py:218] Parsed tool call {}
hermes_tool_parser.py:124] delta_text: ":
hermes_tool_parser.py:125] delta_token_ids: [788]
hermes_tool_parser.py:218] Parsed tool call {}
hermes_tool_parser.py:124] delta_text:  "
hermes_tool_parser.py:125] delta_token_ids: [330]
hermes_tool_parser.py:218] Parsed tool call {}
hermes_tool_parser.py:124] delta_text: predict
hermes_tool_parser.py:125] delta_token_ids: [34698]
hermes_tool_parser.py:218] Parsed tool call {}
hermes_tool_parser.py:124] delta_text: _x
hermes_tool_parser.py:125] delta_token_ids: [3212]
hermes_tool_parser.py:218] Parsed tool call {}
hermes_tool_parser.py:124] delta_text: ",
hermes_tool_parser.py:125] delta_token_ids: [497]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x'}
hermes_tool_parser.py:124] delta_text:  "
hermes_tool_parser.py:125] delta_token_ids: [330]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x'}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: None
hermes_tool_parser.py:267] against new ones: None
hermes_tool_parser.py:271] Skipping text  " - no arguments
hermes_tool_parser.py:124] delta_text: arguments
hermes_tool_parser.py:125] delta_token_ids: [16370]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x'}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: None
hermes_tool_parser.py:267] against new ones: None
hermes_tool_parser.py:271] Skipping text arguments - no arguments
client.py:154] Heartbeat successful.
hermes_tool_parser.py:124] delta_text: ":
hermes_tool_parser.py:125] delta_token_ids: [788]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x'}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: None
hermes_tool_parser.py:267] against new ones: None
hermes_tool_parser.py:271] Skipping text ": - no arguments
hermes_tool_parser.py:124] delta_text:  {"
hermes_tool_parser.py:125] delta_token_ids: [5212]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: None
hermes_tool_parser.py:267] against new ones: {}
hermes_tool_parser.py:271] Skipping text  {" - no arguments
hermes_tool_parser.py:124] delta_text: wd
hermes_tool_parser.py:125] delta_token_ids: [6377]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {}
hermes_tool_parser.py:267] against new ones: {}
hermes_tool_parser.py:271] Skipping text wd - no arguments
hermes_tool_parser.py:124] delta_text: ":
hermes_tool_parser.py:125] delta_token_ids: [788]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {}
hermes_tool_parser.py:267] against new ones: {}
hermes_tool_parser.py:271] Skipping text ": - no arguments
hermes_tool_parser.py:124] delta_text:
hermes_tool_parser.py:125] delta_token_ids: [220]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {}
hermes_tool_parser.py:267] against new ones: {}
hermes_tool_parser.py:271] Skipping text   - no arguments
hermes_tool_parser.py:124] delta_text: 2
hermes_tool_parser.py:125] delta_token_ids: [17]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 2}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {}
hermes_tool_parser.py:267] against new ones: {'wd': 2}
hermes_tool_parser.py:286] finding 2 in {"wd": 2}
hermes_tool_parser.py:295] First tokens in arguments received: {"wd": 2
hermes_tool_parser.py:124] delta_text: 3
hermes_tool_parser.py:125] delta_token_ids: [18]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 2}
hermes_tool_parser.py:267] against new ones: {'wd': 23}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 2}
hermes_tool_parser.py:316] got argument diff 3
hermes_tool_parser.py:124] delta_text: ,
hermes_tool_parser.py:125] delta_token_ids: [11]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23}
hermes_tool_parser.py:267] against new ones: {'wd': 23}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23}
hermes_tool_parser.py:316] got argument diff
hermes_tool_parser.py:124] delta_text:  "
hermes_tool_parser.py:125] delta_token_ids: [330]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23}
hermes_tool_parser.py:267] against new ones: {'wd': 23}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23}
hermes_tool_parser.py:316] got argument diff
hermes_tool_parser.py:124] delta_text: sd
hermes_tool_parser.py:125] delta_token_ids: [13446]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23}
hermes_tool_parser.py:267] against new ones: {'wd': 23}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23}
hermes_tool_parser.py:316] got argument diff
hermes_tool_parser.py:124] delta_text: ":
hermes_tool_parser.py:125] delta_token_ids: [788]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23}
hermes_tool_parser.py:267] against new ones: {'wd': 23}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23}
hermes_tool_parser.py:316] got argument diff
hermes_tool_parser.py:124] delta_text:
hermes_tool_parser.py:125] delta_token_ids: [220]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23}
hermes_tool_parser.py:267] against new ones: {'wd': 23}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23}
hermes_tool_parser.py:316] got argument diff
hermes_tool_parser.py:124] delta_text: 0
hermes_tool_parser.py:125] delta_token_ids: [15]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23}
hermes_tool_parser.py:316] got argument diff , "sd": 0
hermes_tool_parser.py:124] delta_text: .
hermes_tool_parser.py:125] delta_token_ids: [13]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0}
hermes_tool_parser.py:316] got argument diff
hermes_tool_parser.py:124] delta_text: 8
hermes_tool_parser.py:125] delta_token_ids: [23]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0}
hermes_tool_parser.py:316] got argument diff .8
hermes_tool_parser.py:124] delta_text: ,
hermes_tool_parser.py:125] delta_token_ids: [11]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8}
hermes_tool_parser.py:316] got argument diff
hermes_tool_parser.py:124] delta_text:  "
hermes_tool_parser.py:125] delta_token_ids: [330]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8}
hermes_tool_parser.py:316] got argument diff
hermes_tool_parser.py:124] delta_text: in
hermes_tool_parser.py:125] delta_token_ids: [258]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8}
hermes_tool_parser.py:316] got argument diff
hermes_tool_parser.py:124] delta_text: jection
hermes_tool_parser.py:125] delta_token_ids: [7606]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8}
hermes_tool_parser.py:316] got argument diff
hermes_tool_parser.py:124] delta_text: ":
hermes_tool_parser.py:125] delta_token_ids: [788]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8}
hermes_tool_parser.py:316] got argument diff

hermes_tool_parser.py:124] delta_text:  "
                                       🐛

hermes_tool_parser.py:125] delta_token_ids: [330]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8, 'injection': ''}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8, 'injection': ''}
hermes_tool_parser.py:312] Searching for diff between

hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8, "injection": ""}

hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8}

hermes_tool_parser.py:316] got argument diff , "injection": ""
                                                             🐛

hermes_tool_parser.py:124] delta_text: Z
hermes_tool_parser.py:125] delta_token_ids: [57]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8, 'injection': 'Z'}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8, 'injection': ''}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8, 'injection': 'Z'}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8, "injection": "Z"}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8, "injection": ""}
hermes_tool_parser.py:316] got argument diff Z
hermes_tool_parser.py:124] delta_text: S
hermes_tool_parser.py:125] delta_token_ids: [50]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8, 'injection': 'ZS'}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8, 'injection': 'Z'}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8, 'injection': 'ZS'}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8, "injection": "ZS"}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8, "injection": "Z"}
hermes_tool_parser.py:316] got argument diff S
hermes_tool_parser.py:124] delta_text: JB
hermes_tool_parser.py:125] delta_token_ids: [46107]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB'}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8, 'injection': 'ZS'}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB'}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8, "injection": "ZSJB"}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8, "injection": "ZS"}
hermes_tool_parser.py:316] got argument diff JB
hermes_tool_parser.py:124] delta_text: 3
hermes_tool_parser.py:125] delta_token_ids: [18]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB3'}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB'}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB3'}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8, "injection": "ZSJB3"}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8, "injection": "ZSJB"}
hermes_tool_parser.py:316] got argument diff 3
hermes_tool_parser.py:124] delta_text: 9
hermes_tool_parser.py:125] delta_token_ids: [24]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39'}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB3'}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39'}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8, "injection": "ZSJB39"}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8, "injection": "ZSJB3"}
hermes_tool_parser.py:316] got argument diff 9
hermes_tool_parser.py:124] delta_text: ",
hermes_tool_parser.py:125] delta_token_ids: [497]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39'}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39'}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39'}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8, "injection": "ZSJB39"}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8, "injection": "ZSJB39"}
hermes_tool_parser.py:316] got argument diff
hermes_tool_parser.py:124] delta_text:  "
hermes_tool_parser.py:125] delta_token_ids: [330]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39'}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39'}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39'}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8, "injection": "ZSJB39"}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8, "injection": "ZSJB39"}
hermes_tool_parser.py:316] got argument diff
hermes_tool_parser.py:124] delta_text: module
hermes_tool_parser.py:125] delta_token_ids: [4352]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39'}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39'}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39'}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8, "injection": "ZSJB39"}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8, "injection": "ZSJB39"}
hermes_tool_parser.py:316] got argument diff
hermes_tool_parser.py:124] delta_text: _id
hermes_tool_parser.py:125] delta_token_ids: [842]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39'}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39'}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39'}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8, "injection": "ZSJB39"}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8, "injection": "ZSJB39"}
hermes_tool_parser.py:316] got argument diff
hermes_tool_parser.py:124] delta_text: ":
hermes_tool_parser.py:125] delta_token_ids: [788]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39'}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39'}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39'}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8, "injection": "ZSJB39"}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8, "injection": "ZSJB39"}
hermes_tool_parser.py:316] got argument diff
hermes_tool_parser.py:124] delta_text:  "
hermes_tool_parser.py:125] delta_token_ids: [330]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': ''}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39'}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': ''}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": ""}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8, "injection": "ZSJB39"}
hermes_tool_parser.py:316] got argument diff ", "module_id": "
hermes_tool_parser.py:124] delta_text: M
hermes_tool_parser.py:125] delta_token_ids: [44]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M'}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': ''}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M'}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": "M"}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": ""}
hermes_tool_parser.py:316] got argument diff M
hermes_tool_parser.py:124] delta_text: 6
hermes_tool_parser.py:125] delta_token_ids: [21]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M6'}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M'}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M6'}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": "M6"}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": "M"}
hermes_tool_parser.py:316] got argument diff 6
hermes_tool_parser.py:124] delta_text: 2
hermes_tool_parser.py:125] delta_token_ids: [17]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M62'}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M6'}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M62'}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": "M62"}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": "M6"}
hermes_tool_parser.py:316] got argument diff 2
hermes_tool_parser.py:124] delta_text: 1
hermes_tool_parser.py:125] delta_token_ids: [16]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M621'}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M62'}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M621'}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": "M621"}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": "M62"}
hermes_tool_parser.py:316] got argument diff 1
hermes_tool_parser.py:124] delta_text: 2
hermes_tool_parser.py:125] delta_token_ids: [17]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M6212'}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M621'}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M6212'}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": "M6212"}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": "M621"}
hermes_tool_parser.py:316] got argument diff 2
hermes_tool_parser.py:124] delta_text: 1
hermes_tool_parser.py:125] delta_token_ids: [16]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M62121'}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M6212'}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M62121'}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": "M62121"}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": "M6212"}
hermes_tool_parser.py:316] got argument diff 1
hermes_tool_parser.py:124] delta_text: G
hermes_tool_parser.py:125] delta_token_ids: [38]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M62121G'}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M62121'}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M62121G'}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": "M62121G"}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": "M62121"}
hermes_tool_parser.py:316] got argument diff G
hermes_tool_parser.py:124] delta_text: 9
hermes_tool_parser.py:125] delta_token_ids: [24]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M62121G9'}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M62121G'}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M62121G9'}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": "M62121G9"}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": "M62121G"}
hermes_tool_parser.py:316] got argument diff 9
hermes_tool_parser.py:124] delta_text: 1
hermes_tool_parser.py:125] delta_token_ids: [16]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M62121G91'}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M62121G9'}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M62121G91'}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": "M62121G91"}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": "M62121G9"}
hermes_tool_parser.py:316] got argument diff 1
hermes_tool_parser.py:124] delta_text: 4
hermes_tool_parser.py:125] delta_token_ids: [19]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M62121G914'}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M62121G91'}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M62121G914'}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": "M62121G914"}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": "M62121G91"}
hermes_tool_parser.py:316] got argument diff 4
hermes_tool_parser.py:124] delta_text: 2
hermes_tool_parser.py:125] delta_token_ids: [17]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M62121G9142'}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M62121G914'}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M62121G9142'}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": "M62121G9142"}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": "M62121G914"}
hermes_tool_parser.py:316] got argument diff 2
hermes_tool_parser.py:124] delta_text: "}}
hermes_tool_parser.py:124]
hermes_tool_parser.py:125] delta_token_ids: [95642]
hermes_tool_parser.py:218] Parsed tool call {'name': 'predict_x', 'arguments': {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M62121G9142'}}
hermes_tool_parser.py:252] Trying to parse current tool call with ID 0
hermes_tool_parser.py:266] diffing old arguments: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M62121G9142'}
hermes_tool_parser.py:267] against new ones: {'wd': 23, 'sd': 0.8, 'injection': 'ZSJB39', 'module_id': 'M62121G9142'}
hermes_tool_parser.py:312] Searching for diff between
hermes_tool_parser.py:312] {"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": "M62121G9142"}
hermes_tool_parser.py:313] and
hermes_tool_parser.py:313] {"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": "M62121G9142"}
hermes_tool_parser.py:316] got argument diff
hermes_tool_parser.py:124] delta_text: </tool_call>
hermes_tool_parser.py:125] delta_token_ids: [151658]

hermes_tool_parser.py:194] Finishing tool and found diff that had not been streamed yet: {"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": "M62121G9142"}

llm_engine.py:1460] Stopping remote worker execution loop.
hermes_tool_parser.py:124] delta_text:
hermes_tool_parser.py:125] delta_token_ids: [151645]
hermes_tool_parser.py:147] Generating text content! skipping tool parsing.
  | DEBUG 11-16 08:15:26 client.py:170] Waiting for output from MQLLMEngine.
  | INFO 11-16 08:15:26 metrics.py:349] Avg prompt throughput: 32.0 tokens/s, Avg generation throughput: 7.8 tokens/s, Running: 0 reqs, Swapped: 0 reqs, Pending: 0 reqs, GPU KV cache usage: 0.0%, CPU KV cache usage: 0.0%.
  | DEBUG 11-16 08:15:26 engine.py:213] Waiting for new requests in engine loop.
  | DEBUG 11-16 08:15:26 client.py:154] Heartbeat successful.

```

</details>

<details>
<summary>Client code</summary>

```
import json

from openai import OpenAI

# Modify OpenAI's API key and API base to use vLLM's API server.

openai_api_key = "EMPTY"
openai_api_base = "http://localhost:8000/v1"

client = OpenAI(
    # defaults to os.environ.get("OPENAI_API_KEY")
    api_key=openai_api_key,
    base_url=openai_api_base,
)

models = client.models.list()
model = models.data[0].id

print(f"{model=}")

tools = [{
    "type": "function",
    "function": {
        "name": "get_current_weather",
        "description": "Get the current weather in a given location",
        "parameters": {
            "type": "object",
            "properties": {
                "city": {
                    "type":
                    "string",
                    "description":
                    "The city to find the weather for, e.g. 'San Francisco'"
                },
                "state": {
                    "type":
                    "string",
                    "description":
                    "the two-letter abbreviation for the state that the city is"
                    " in, e.g. 'CA' which would mean 'California'"
                },
                "unit": {
                    "type": "string",
                    "description": "The unit to fetch the temperature in",
                    "enum": ["celsius", "fahrenheit"]
                }
            },
            "required": ["city", "state", "unit"]
        }
    }
},{
    "type": "function",
    "function": {
        "name": "predict_x",
        "description": "predict x using wd, sd, injection and module_id.",
        "parameters": {
            "type": "object",
            "properties": {
                "wd": {
                    "type":
                    "number",
                    "description":
                    "shi du"
                },
                "sd": {
                    "type":
                    "number",
                    "description":
                    "wen du"
                },
                "injection": {
                    "type":
                    "string",
                    "description":
                    "injection type"
                },
                "module_id": {
                    "type":
                    "string",
                    "description":
                    "module id"
                }
            },
            "required": ["wd", "sd", "injection", "module_id"]
        }
    }
}]

messages = [{
    "role": "user",
    "content": "Hi! How are you doing today?"
}, {
    "role": "assistant",
    "content": "I'm doing well! How can I help you?"
}, {
    "role":
    "user",
    "content":
    "when wd=23, sd=0.8, injection=ZSJB39, module_id=M62121G9142, what's x ? "
}]

chat_completion = client.chat.completions.create(messages=messages,
                                                 model=model,
                                                 tools=tools)

print(chat_completion)
print("\n\n")

tool_calls_stream = client.chat.completions.create(messages=messages,
                                                   model=model,
                                                   tools=tools,
                                                   stream=True)

chunks = []
for chunk in tool_calls_stream:
    chunks.append(chunk)
    if chunk.choices[0].delta.tool_calls:
        arg=chunk.choices[0].delta.tool_calls[0].function.arguments
        print(f"===={arg}")
    else:
        # print(chunk.choices[0].delta)
        print(f"====❌")

arguments = []
tool_call_idx = -1
for chunk in chunks:
    if chunk.choices[0].delta.tool_calls:
        tool_call = chunk.choices[0].delta.tool_calls[0]

        if tool_call.index != tool_call_idx:
            if tool_call_idx >= 0:
                print(
                    f"streamed tool call arguments: {arguments[tool_call_idx]}"
                )
            tool_call_idx = chunk.choices[0].delta.tool_calls[0].index
            arguments.append("")
        if tool_call.id:
            print(f"streamed tool call id: {tool_call.id} ")

        if tool_call.function:
            if tool_call.function.name:
                print(f"streamed tool call name: {tool_call.function.name}")

            if tool_call.function.arguments:
                arguments[tool_call_idx] += tool_call.function.arguments

if len(arguments)<0:
    print(f"❌no tool used")
else:
    try:
        json.loads(arguments[-1])
        print(f"✅streamed tool call arguments is OK: {arguments[-1]}")
    except:
        print(f"❌streamed tool call arguments is ERROR: {arguments[-1]}")

```

</details>

<details>
<summary>Client log</summary>

```
python openai_chat_completion_client_with_tools_err.py
model='Qwen2.5-72B-Instruct-AWQ'

ChoiceDelta(content='', function_call=None, refusal=None, role='assistant', tool_calls=None)
stream arg: [None]
stream arg: [{"wd": 2]
stream arg: [3]
stream arg: []
stream arg: []
stream arg: []
stream arg: []
stream arg: []
stream arg: [, "sd": 0]
stream arg: []
stream arg: [.8]
stream arg: []
stream arg: []
stream arg: []
stream arg: []
stream arg: []
stream arg: [, "injection": ""]   <-------------------- 🐛 ERROR here
stream arg: [Z]
stream arg: [S]
stream arg: [JB]
stream arg: [3]
stream arg: [9]
stream arg: []
stream arg: []
stream arg: []
stream arg: []
stream arg: []
stream arg: [", "module_id": "]
stream arg: [M]
stream arg: [6]
stream arg: [2]
stream arg: [1]
stream arg: [2]
stream arg: [1]
stream arg: [G]
stream arg: [9]
stream arg: [1]
stream arg: [4]
stream arg: [2]
stream arg: []
stream arg: [{"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": "M62121G9142"}]
ChoiceDelta(content='', function_call=None, refusal=None, role=None, tool_calls=None)
streamed tool call id: chatcmpl-tool-4a6872656dae4b278f8754222e16ae64
streamed tool call name: predict_x
❌streamed tool call arguments is ERROR🐛: {"wd": 23, "sd": 0.8, "injection": ""ZSJB39", "module_id": "M62121G9142{"wd": 23, "sd": 0.8, "injection": "ZSJB39", "module_id": "M62121G9142"}

```

### Before submitting a new issue...

- [X] Make sure you already searched for relevant issues, and asked the chatbot living at the bottom right corner of the [documentation page](https://docs.vllm.ai/en/latest/), which can answer lots of frequently asked questions.


diff --git a/vllm/model_executor/models/falcon.py b/vllm/model_executor/models/falcon.py
index dcfcb6694feb5..b3dbf063ac298 100644
--- a/vllm/model_executor/models/falcon.py
+++ b/vllm/model_executor/models/falcon.py
@@ -250,6 +250,9 @@ def __init__(
         self.mlp = FalconMLP(config, quant_config)
         self.config = config

+        if (not hasattr(config, "num_ln_in_parallel_attn")):
+            config.num_ln_in_parallel_attn = None
+
         if (config.num_ln_in_parallel_attn is None
                 and config.new_decoder_architecture):
             config.num_ln_in_parallel_attn = 2

[Bug]: Falcon fails if `trust_remote_code=True`
### Your current environment

v0.4.3

### 🐛 Describe the bug

```python
from vllm import LLM
model = LLM("tiiuae/falcon-7b", trust_remote_code=True)
```

```bash
--- Logging error ---
Traceback (most recent call last):
  File "/home/rshaw/.pyenv/versions/3.10.14/lib/python3.10/logging/__init__.py", line 1100, in emit
    msg = self.format(record)
  File "/home/rshaw/.pyenv/versions/3.10.14/lib/python3.10/logging/__init__.py", line 943, in format
    return fmt.format(record)
  File "/home/rshaw/.pyenv/versions/vllm-upstream-pip/lib/python3.10/site-packages/vllm/logging/formatter.py", line 11, in format
    msg = logging.Formatter.format(self, record)
  File "/home/rshaw/.pyenv/versions/3.10.14/lib/python3.10/logging/__init__.py", line 678, in format
    record.message = record.getMessage()
  File "/home/rshaw/.pyenv/versions/3.10.14/lib/python3.10/logging/__init__.py", line 368, in getMessage
    msg = msg % self.args
TypeError: %d format: a real number is required, not list
Call stack:
  File "<stdin>", line 1, in <module>
  File "/home/rshaw/.pyenv/versions/vllm-upstream-pip/lib/python3.10/site-packages/vllm/entrypoints/llm.py", line 144, in __init__
    self.llm_engine = LLMEngine.from_engine_args(
  File "/home/rshaw/.pyenv/versions/vllm-upstream-pip/lib/python3.10/site-packages/vllm/engine/llm_engine.py", line 335, in from_engine_args
    engine_config = engine_args.create_engine_config()
  File "/home/rshaw/.pyenv/versions/vllm-upstream-pip/lib/python3.10/site-packages/vllm/engine/arg_utils.py", line 559, in create_engine_config
    model_config = ModelConfig(
  File "/home/rshaw/.pyenv/versions/vllm-upstream-pip/lib/python3.10/site-packages/vllm/config.py", line 133, in __init__
    self.max_model_len = _get_and_verify_max_len(
  File "/home/rshaw/.pyenv/versions/vllm-upstream-pip/lib/python3.10/site-packages/vllm/config.py", line 1208, in _get_and_verify_max_len
    logger.warning(
Message: "The model's config.json does not contain any of the following keys to determine the original maximum length of the model: %d. Assuming the model's maximum length is %d."
Arguments: (['max_position_embeddings', 'n_positions', 'max_seq_len', 'seq_length', 'model_max_length', 'max_sequence_length', 'max_seq_length', 'seq_len'], 2048)
INFO 06-09 12:53:58 llm_engine.py:161] Initializing an LLM engine (v0.4.3) with config: model='tiiuae/falcon-7b', speculative_config=None, tokenizer='tiiuae/falcon-7b', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, rope_scaling=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=LoadFormat.AUTO, tensor_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto, quantization_param_path=None, device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='outlines'), seed=0, served_model_name=tiiuae/falcon-7b)
[rank0]: Traceback (most recent call last):
[rank0]:   File "<stdin>", line 1, in <module>
[rank0]:   File "/home/rshaw/.pyenv/versions/vllm-upstream-pip/lib/python3.10/site-packages/vllm/entrypoints/llm.py", line 144, in __init__
[rank0]:     self.llm_engine = LLMEngine.from_engine_args(
[rank0]:   File "/home/rshaw/.pyenv/versions/vllm-upstream-pip/lib/python3.10/site-packages/vllm/engine/llm_engine.py", line 359, in from_engine_args
[rank0]:     engine = cls(
[rank0]:   File "/home/rshaw/.pyenv/versions/vllm-upstream-pip/lib/python3.10/site-packages/vllm/engine/llm_engine.py", line 222, in __init__
[rank0]:     self.model_executor = executor_class(
[rank0]:   File "/home/rshaw/.pyenv/versions/vllm-upstream-pip/lib/python3.10/site-packages/vllm/executor/executor_base.py", line 41, in __init__
[rank0]:     self._init_executor()
[rank0]:   File "/home/rshaw/.pyenv/versions/vllm-upstream-pip/lib/python3.10/site-packages/vllm/executor/gpu_executor.py", line 24, in _init_executor
[rank0]:     self.driver_worker.load_model()
[rank0]:   File "/home/rshaw/.pyenv/versions/vllm-upstream-pip/lib/python3.10/site-packages/vllm/worker/worker.py", line 121, in load_model
[rank0]:     self.model_runner.load_model()
[rank0]:   File "/home/rshaw/.pyenv/versions/vllm-upstream-pip/lib/python3.10/site-packages/vllm/worker/model_runner.py", line 134, in load_model
[rank0]:     self.model = get_model(
[rank0]:   File "/home/rshaw/.pyenv/versions/vllm-upstream-pip/lib/python3.10/site-packages/vllm/model_executor/model_loader/__init__.py", line 21, in get_model
[rank0]:     return loader.load_model(model_config=model_config,
[rank0]:   File "/home/rshaw/.pyenv/versions/vllm-upstream-pip/lib/python3.10/site-packages/vllm/model_executor/model_loader/loader.py", line 240, in load_model
[rank0]:     model = _initialize_model(model_config, self.load_config,
[rank0]:   File "/home/rshaw/.pyenv/versions/vllm-upstream-pip/lib/python3.10/site-packages/vllm/model_executor/model_loader/loader.py", line 91, in _initialize_model
[rank0]:     return model_class(config=model_config.hf_config,
[rank0]:   File "/home/rshaw/.pyenv/versions/vllm-upstream-pip/lib/python3.10/site-packages/vllm/model_executor/models/falcon.py", line 389, in __init__
[rank0]:     self.transformer = FalconModel(config, cache_config, quant_config)
[rank0]:   File "/home/rshaw/.pyenv/versions/vllm-upstream-pip/lib/python3.10/site-packages/vllm/model_executor/models/falcon.py", line 350, in __init__
[rank0]:     self.h = nn.ModuleList([
[rank0]:   File "/home/rshaw/.pyenv/versions/vllm-upstream-pip/lib/python3.10/site-packages/vllm/model_executor/models/falcon.py", line 351, in <listcomp>
[rank0]:     FalconDecoderLayer(config, cache_config, quant_config)
[rank0]:   File "/home/rshaw/.pyenv/versions/vllm-upstream-pip/lib/python3.10/site-packages/vllm/model_executor/models/falcon.py", line 249, in __init__
[rank0]:     if (config.num_ln_in_parallel_attn is None
[rank0]:   File "/home/rshaw/.pyenv/versions/vllm-upstream-pip/lib/python3.10/site-packages/transformers/configuration_utils.py", line 264, in __getattribute__
[rank0]:     return super().__getattribute__(key)
[rank0]: AttributeError: 'FalconConfig' object has no attribute 'num_ln_in_parallel_attn'
```

Confirmed the same issue on my end. Actually it doesn't matter whether trust_remote_code is True or False (I have tried both), what matters is that this parameter is set. Once I remove this then all worked well to me.
This issue has been automatically marked as stale because it has not had any activity within 90 days. It will be automatically closed if no further activity occurs within 30 days. Leave a comment if you feel this issue should remain open. Thank you!


diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py
index e7aeac8f8c018..e31dc2ced61fb 100644
--- a/vllm/entrypoints/openai/serving_engine.py
+++ b/vllm/entrypoints/openai/serving_engine.py
@@ -443,29 +443,28 @@ async def _preprocess_chat(
             tokenizer,
         )

+        _chat_template_kwargs: Dict[str, Any] = dict(
+            chat_template=chat_template,
+            add_generation_prompt=add_generation_prompt,
+            continue_final_message=continue_final_message,
+            tools=tool_dicts,
+            documents=documents,
+        )
+        _chat_template_kwargs.update(chat_template_kwargs or {})
+
         request_prompt: Union[str, List[int]]
         is_mistral_tokenizer = isinstance(tokenizer, MistralTokenizer)
         if is_mistral_tokenizer:
             request_prompt = apply_mistral_chat_template(
                 tokenizer,
                 messages=messages,
-                chat_template=chat_template,
-                add_generation_prompt=add_generation_prompt,
-                continue_final_message=continue_final_message,
-                tools=tool_dicts,
-                documents=documents,
-                **(chat_template_kwargs or {}),
+                **_chat_template_kwargs,
             )
         else:
             request_prompt = apply_hf_chat_template(
                 tokenizer,
                 conversation=conversation,
-                chat_template=chat_template,
-                add_generation_prompt=add_generation_prompt,
-                continue_final_message=continue_final_message,
-                tools=tool_dicts,
-                documents=documents,
-                **(chat_template_kwargs or {}),
+                **_chat_template_kwargs,
             )

         mm_data = await mm_data_future

[Bug]: When apply continue_final_message, encounter: got multiple values for keyword argument
### Your current environment

vLLM Version: 0.6.3.post2.dev256+g4be3a451
<details>
<summary>The output of `python collect_env.py`</summary>

```text
Collecting environment information...                                                                                INFO 11-06 09:39:21 importing.py:15] Triton not installed or not compatible; certain GPU-related functions will not be available.                                                                           PyTorch version: 2.4.0+cpu
Is debug build: False
CUDA used to build PyTorch: None
ROCM used to build PyTorch: N/A

OS: Ubuntu 22.04.4 LTS (x86_64)
GCC version: (Ubuntu 12.3.0-1ubuntu1~22.04) 12.3.0
Clang version: Could not collect
CMake version: version 3.30.5
Libc version: glibc-2.35

Python version: 3.10.12 (main, Jul 29 2024, 16:56:48) [GCC 11.4.0] (64-bit runtime)
Python platform: Linux-5.10.25-nvidia-gpu-x86_64-with-glibc2.35
Is CUDA available: False
CUDA runtime version: No CUDA
CUDA_MODULE_LOADING set to: N/A
GPU models and configuration: No CUDA
Nvidia driver version: No CUDA
cuDNN version: No CUDA
HIP runtime version: N/A
MIOpen runtime version: N/A

CPU:
Architecture:                    x86_64
CPU op-mode(s):                  32-bit, 64-bit
Address sizes:                   46 bits physical, 57 bits virtual
Byte Order:                      Little Endian
CPU(s):                          17
On-line CPU(s) list:             0-16
Vendor ID:                       GenuineIntel
Model name:                      Intel(R) Xeon(R) Gold 6330 CPU @ 2.00GHz
CPU family:                      6
Model:                           106
Thread(s) per core:              1
Core(s) per socket:              1
Socket(s):                       17
Stepping:                        6
BogoMIPS:                        4000.00
Flags:                           fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch cpuid_fault invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves wbnoinvd arat avx512vbmi umip pku avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid md_clear arch_capabilities
Hypervisor vendor:               KVM
Virtualization type:             full
L1d cache:                       544 KiB (17 instances)
L1i cache:                       544 KiB (17 instances)
L2 cache:                        68 MiB (17 instances)
L3 cache:                        272 MiB (17 instances)
Vulnerability Itlb multihit:     Not affected
Vulnerability L1tf:              Not affected
Vulnerability Mds:               Not affected
Vulnerability Meltdown:          Not affected
Vulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp
Vulnerability Spectre v1:        Mitigation; usercopy/swapgs barriers and __user pointer sanitization
Vulnerability Spectre v2:        Mitigation; Enhanced IBRS, IBPB conditional, RSB filling
Vulnerability Srbds:             Not affected
Vulnerability Tsx async abort:   Not affected

Versions of relevant libraries:
[pip3] intel_extension_for_pytorch==2.4.0
[pip3] numpy==1.26.4
[pip3] pyzmq==26.2.0
[pip3] torch==2.4.0+cpu
[pip3] torchvision==0.19.0+cpu
[pip3] transformers==4.46.2
[conda] Could not collect
ROCM Version: Could not collect
Neuron SDK Version: N/A
vLLM Version: 0.6.3.post2.dev256+g4be3a451
vLLM Build Flags:
CUDA Archs: Not Set; ROCm: Disabled; Neuron: Disabled
GPU Topology:
Could not collect
```

</details>

### Model Input Dumps

_No response_

### 🐛 Describe the bug

Reproduce using curl
```bash
curl -X POST "http://39.105.21.95:12481/v1/chat/completions" \
     -H "Content-Type: application/json" \
     -d '{
           "model": "meta-llama/Meta-Llama-3-8B-Instruct",
           "messages": [
             {
               "role": "user",
               "content": "tell me a common saying"
             },
             {
               "role": "assistant",
               "content": "Here is a common saying about apple. An apple a day, keeps"
             }
           ],
           "add_generation_prompt": false, "chat_template_kwargs":{"continue_final_message": true}
         }'

Internal Server Error%
```

Error message on server
```bash
  File "/usr/local/lib/python3.10/dist-packages/vllm/entrypoints/openai/api_server.py", line 338, in create_chat_completion
    generator = await handler.create_chat_completion(request, raw_request)
  File "/usr/local/lib/python3.10/dist-packages/vllm/entrypoints/openai/serving_chat.py", line 140, in create_chat_completion
    ) = await self._preprocess_chat(
  File "/usr/local/lib/python3.10/dist-packages/vllm/entrypoints/openai/serving_engine.py", line 460, in _preprocess_chat
    request_prompt = apply_hf_chat_template(
TypeError: vllm.entrypoints.chat_utils.apply_hf_chat_template() got multiple values for keyword argument 'continue_final_message'
```

### Before submitting a new issue...

- [X] Make sure you already searched for relevant issues, and asked the chatbot living at the bottom right corner of the [documentation page](https://docs.vllm.ai/en/latest/), which can answer lots of frequently asked questions.

You should pass `continue_final_message` directly (like how you did it with `add_generation_prompt`) instead of using separate `chat_template_kwargs`.
Previously, it could only be called in `chat_template_kwargs`, there is already a lot of code to support the “continuation generating” in this way, can vLLM support both options?
I see, feel free to open a PR to support both methods.


diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py
index bef36ffdbfcd3..917b347ff1161 100644
--- a/vllm/entrypoints/openai/api_server.py
+++ b/vllm/entrypoints/openai/api_server.py
@@ -569,7 +569,8 @@ async def run_server(args, **uvicorn_kwargs) -> None:
     # This avoids race conditions with ray.
     # see https://github.com/vllm-project/vllm/issues/8204
     sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    sock.bind(("", args.port))
+    sock.bind((args.host or "", args.port))
+    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

     def signal_handler(*_) -> None:
         # Interrupt server on sigterm while initializing
@@ -593,13 +594,14 @@ def signal_handler(*_) -> None:
             ssl_certfile=args.ssl_certfile,
             ssl_ca_certs=args.ssl_ca_certs,
             ssl_cert_reqs=args.ssl_cert_reqs,
-            fd=sock.fileno(),
             **uvicorn_kwargs,
         )

     # NB: Await server shutdown only after the backend context is exited
     await shutdown_task

+    sock.close()
+

 if __name__ == "__main__":
     # NOTE(simon):

[Bug]: "Address already in use" for 1 minute after crash (since 0.6.2)
### 🐛 Describe the bug

Since version 0.6.2 (happens also in 0.6.3.post1), after the server dies (due to an exception/crash or hitting ctrl-c), for about a minute, it fails to start again with:
```
Traceback (most recent call last):
  File "/usr/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/usr/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/home/user/code/debug/.venv/lib/python3.10/site-packages/vllm/entrypoints/openai/api_server.py", line 585, in <module>
    uvloop.run(run_server(args))
  File "/home/user/code/debug/.venv/lib/python3.10/site-packages/uvloop/__init__.py", line 82, in run
    return loop.run_until_complete(wrapper())
  File "uvloop/loop.pyx", line 1517, in uvloop.loop.Loop.run_until_complete
  File "/home/user/code/debug/.venv/lib/python3.10/site-packages/uvloop/__init__.py", line 61, in wrapper
    return await main
  File "/home/user/code/debug/.venv/lib/python3.10/site-packages/vllm/entrypoints/openai/api_server.py", line 544, in run_server
    sock.bind(("", args.port))
OSError: [Errno 98] Address already in use
```
This prolongs recovery from crashes. In example upon crash Kubernetes immediately restarts the container - previously it would immediately start loading the model again, but now it will do several crash/restart loops until the port is freed.

Verified it happens also with `--disable-frontend-multiprocessing`.

To reproduce it, start vllm with default args, in example:
```
python -m vllm.entrypoints.openai.api_server --model TinyLlama/TinyLlama-1.1B-Chat-v1.0
```
and then send at least one chat or completion request to it (without this it won't reproduce).
then hit Ctrl-C to kill the server.
starting vllm again should throw the "Address already in use" error.
This doesn't happen with vllm <= 0.6.1.

I tried to see why the port is busy, and interestingly the vllm process is dead during this ~1 minute and no other process listens on it. However I noticed that there is a socket open *from* the 8000 port. Can see it via:
```
netstat | grep  ':8000'
```
which would show something like:
```
tcp        0      0 localhost:8000          localhost:40452         TIME_WAIT   -
tcp        0      0 localhost:8000          localhost:56324         TIME_WAIT   -
tcp        0      0 localhost:8000          localhost:40466         TIME_WAIT   -
```
After a minute these entries will disappear and then also vllm will manage to start.
I couldn't attribute it to a PID, nor with various `nestat` or `lsof` flags. Maybe it remains open in the kernel due to unclean process exit?

I encountered the same issue, but it seems to only occur in the Docker container. I haven’t experienced this issue on the bare server.
@yansh97 👍
In the description above I reproduce it also by running locally the pip package in WSL (and it also happens in Kubernetes, not in WSL)
Ya ever since some version, maybe 0.6.1 or so, always see this, constant problem, eventually in docker with restart always it comes up, but annoying.
The original report says

> (due to an exception/crash or hitting ctrl-c)

Can someone confirm they see this with ctrl-c (not in docker) ? In my testing with exiting that way, the process cleans up properly before exiting.

I'm guessing this is happening when `vllm` is not given a chance to exit cleanly. Can you clarify exactly how you're running vllm and exiting?
Issue is not exit cleanly, first time docker comes up, fails randomly with this and repeats about 5-15 times before finally coming up.

I think it's a race inside vLLM itself.
> The original report says 原始报告说
>
> > (due to an exception/crash or hitting ctrl-c)（由于异常/崩溃或按 Ctrl-C）
>
> Can someone confirm they see this with ctrl-c (not in docker) ? In my testing with exiting that way, the process cleans up properly before exiting.有人可以确认他们使用 ctrl-c（不在 docker 中）看到这一点吗？在我以这种方式退出的测试中，该过程在退出之前正确清理。
>
> I'm guessing this is happening when `vllm` is not given a chance to exit cleanly. Can you clarify exactly how you're running vllm and exiting?我猜这是在没有机会干净地退出时发生的。您能澄清一下您是如何运行 vllm 和退出的吗？

I’ve only encountered this issue inside a Docker container. The ways to exit include pressing Ctrl+C, sending SIGINT or SIGTERM to the process from outside, or sending SIGINT or SIGTERM to itself within the Python code.
can I use `sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)` reused address to solve the problem ?
```
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    sock.bind(("", args.port))
```
@russellb to double-check, did you send at least one chat completion request to the server before killing it with ctrl-c? it is required to reproduce 🙏
> @russellb to double-check, did you send at least one chat completion request to the server before killing it with ctrl-c? it is required to reproduce 🙏

I did, yes, using `curl`
@russellb I also couldn't reproduce it with `curl` now! so seems it is related to the client also. I managed to reproduce again with both `openai` client and also with `requests`, with the interesting bit that the client process should keep running after the request. As `requests` has less magic going-on, here is how I managed to reproduce it using it:

start the vllm server with:
```bash
 python -m vllm.entrypoints.openai.api_server --model TinyLlama/TinyLlama-1.1B-Chat-v1.0
```

then run this python script:
```python
import requests
import time

url = "http://localhost:8000/v1/chat/completions"
data = {
    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
    "messages": [{"role": "user", "content": "hello"}],
}
res = requests.post(url, json=data)
print(res.json()['choices'][0]['message'])

print("\nsleeping.. try killing vllm now and starting it again")
time.sleep(60)
```

then while the client is running (few seconds after the chat request), kill the vllm server with ctrl-c, and try starting vllm again (which should now fail with `OSError: [Errno 98] Address already in use`).
And during the first minute can still see the outgoing connection from port 8000 in netstat:
`netstat | grep  ':8000'`:
```
tcp        0      0 localhost:8000          localhost:42454         FIN_WAIT2
```
(I also noticed that the client process doesn't have to stay up for the whole minute, it's enough to wait few seconds until the connection in netstat shows `FIN_WAIT2` - then even if it's killed, the vlllm server still would fail to use the port during the first minute)

Thanks for looking into it! 🙏

> @russellb I also couldn't reproduce it with `curl` now! so seems it is related to the client also. I managed to reproduce again with both `openai` client and also with `requests`, with the interesting bit that the client process should keep running after the request. As `requests` has less magic going-on, here is how I managed to reproduce it using it:

In this case I think `requests` is not closing the connection.  Try this change to your script:

```python
#with requests.post(url, json=data) as res:
#    print(res.json()['choices'][0]['message'])

with requests.session() as s:
    s.headers['Connection'] = 'close'
    res = s.post(url, json=data)
    print(res.json()['choices'][0]['message'])
```

With that change, I don't see a connection hanging open.
It is reproducing for me despite this change above 🤔 (as long as the client process stays up for few seconds)

I think it's very common scenario that the clients are still up when the server is crashing/exiting. This is also the default behavior of the openai client - it's actually recommended in the openai docs to create a client object once and then reuse it for multiple chat requests (but don't know if internally it keeps sockets up or not).
e.g. this also reproduces it:
```python
import openai
import time

client = openai.OpenAI(
    base_url="http://localhost:8000/v1",
    api_key="foo"
)

response = client.chat.completions.create(
    model="TinyLlama/TinyLlama-1.1B-Chat-v1.0",
    messages=[{"role": "user", "content": "hello"}],
)
print(response.choices[0].message)
time.sleep(5)
```

The client is not listening on the 8000 port, so I don't see why would the server fail to listen on it again after restart. And this didn't happen before 0.6.2.
Thanks for all the info. I dug into this more closely today. The change in behavior was introduced by #8537. I'm still working on a fix.


diff --git a/vllm/platforms/openvino.py b/vllm/platforms/openvino.py
index 35dbe22abf7ff..31fe3f1fcbfe4 100644
--- a/vllm/platforms/openvino.py
+++ b/vllm/platforms/openvino.py
@@ -1,10 +1,12 @@
 import torch

 import vllm.envs as envs
-from vllm.utils import print_warning_once
+from vllm.logger import init_logger

 from .interface import Platform, PlatformEnum

+logger = init_logger(__name__)
+

 class OpenVinoPlatform(Platform):
     _enum = PlatformEnum.OPENVINO
@@ -27,5 +29,5 @@ def is_openvino_gpu(self) -> bool:

     @classmethod
     def is_pin_memory_available(self) -> bool:
-        print_warning_once("Pin memory is not supported on OpenViNO.")
+        logger.warning("Pin memory is not supported on OpenViNO.")
         return False

[Bug]: from vllm.platforms import current_platform infinite loop error with OpenVino Build.
### Your current environment

Unrelated

### Model Input Dumps

Unrelated

### 🐛 Describe the bug

In OpenVino Build, from vllm.platforms import current_platform for OpenVino will...
 * reference openvino.py in vllm.platforms.openvino.py as OpenVino is identified as current platform
 * openvino.py line 4 will reference from vllm.utils import print_warning_once
 * vllm.utils line 40 will reference from vllm.platforms import current_platform
 * Due to circular self-reference, will throw error

This impacts any vllm functionality on the openvino build due to vllm requiring detection of the current_platform.

Resolution: **removing print_warning_once dependency from openvino.py will fix, either by defining print function locally or removing logging altogether from openvino.py**

### Before submitting a new issue...

- [X] Make sure you already searched for relevant issues, and asked the chatbot living at the bottom right corner of the [documentation page](https://docs.vllm.ai/en/latest/), which can answer lots of frequently asked questions.


diff --git a/sentence_transformers/cross_encoder/CrossEncoder.py b/sentence_transformers/cross_encoder/CrossEncoder.py
index c4345017a..3d7ee0abb 100644
--- a/sentence_transformers/cross_encoder/CrossEncoder.py
+++ b/sentence_transformers/cross_encoder/CrossEncoder.py
@@ -527,6 +527,11 @@ def rank(
                 'score': -5.082967,
                 'text': "The 'Harry Potter' series, which consists of seven fantasy novels written by British author J.K. Rowling, is among the most popular and critically acclaimed books of the modern era."}]
         """
+        if self.config.num_labels != 1:
+            raise ValueError(
+                "CrossEncoder.rank() only works for models with num_labels=1. "
+                "Consider using CrossEncoder.predict() with input pairs instead."
+            )
         query_doc_pairs = [[query, doc] for doc in documents]
         scores = self.predict(
             sentences=query_doc_pairs,

CrossEncoder .rank condition error in CrossEncoder.py
I get the following error when I use .rank method:

```
File /usr/local/lib/python3.12/dist-packages/sentence_transformers/cross_encoder/CrossEncoder.py:551, in CrossEncoder.rank(self, query, documents, top_k, return_documents, batch_size, show_progress_bar, num_workers, activation_fct, apply_softmax, convert_to_numpy, convert_to_tensor)
    548     if return_documents:
    549         results[-1].update({"text": documents[i]})
--> 551 results = sorted(results, key=lambda x: x["score"], reverse=True)
    552 return results[:top_k]

ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()
```
I use sentence_transformers v3.3.0.

A snippet:
```python

cross_encoder = sentence_transformers.CrossEncoder("amberoad/bert-multilingual-passage-reranking-msmarco", device='cpu', max_length=256)
cross_encoder.rank(query, docs)
```

@saeeddhqan can you also share the structure of your query and docs?
```python
cross_encoder.rank('docx', ['doc1', 'doc2', 'doc3'])
```
@saeeddhqan it works for me

```
from sentence_transformers.cross_encoder import CrossEncoder
cross_encoder = CrossEncoder("cross-encoder/stsb-distilroberta-base")

cross_encoder.rank('docx', ['doc1', 'doc2', 'doc3'])
```

Response
```
[{'corpus_id': 0, 'score': np.float32(0.5175216)},
 {'corpus_id': 2, 'score': np.float32(0.4488596)},
 {'corpus_id': 1, 'score': np.float32(0.43759635)}]
```
@JINO-ROHIT The issue seems to be model specific.

@saeeddhqan thanks for opening! The `CrossEncoder` class wraps around the `AutoModelForSequenceClassification` class from `transformers`, and those models can predict logits for $n$ classes per sequence (query-document pairs in this case). The `CrossEncoder.predict` method will call this underlying model and return all predictions. For [amberoad/bert-multilingual-passage-reranking-msmarco](https://huggingface.co/amberoad/bert-multilingual-passage-reranking-msmarco), that's 2:
```python
from sentence_transformers.cross_encoder import CrossEncoder

cross_encoder = CrossEncoder("amberoad/bert-multilingual-passage-reranking-msmarco", device='cpu', max_length=256)
print(cross_encoder.predict([('docx', 'doc1')]))
# [[-1.2904704  1.1504961]]
print(cross_encoder.config.num_labels)
# 2
```
whereas for a lot of CrossEncoder models (e.g. [cross-encoder/stsb-distilroberta-base](https://huggingface.co/cross-encoder/stsb-distilroberta-base)) it's just 1:
```python
from sentence_transformers.cross_encoder import CrossEncoder

cross_encoder = CrossEncoder("cross-encoder/stsb-distilroberta-base", device='cpu', max_length=256)
print(cross_encoder.predict([('docx', 'doc1')]))
# [0.51752156]
print(cross_encoder.config.num_labels)
# 1
```

Beyond that, the `CrossEncoder.rank` method internally calls `CrossEncoder.predict` and then expects that each query-document pair results in 1 value (i.e. that the model only has 1 label). What's missing is a `raise ValueError` in `CrossEncoder.rank` if `self.config.num_labels != 1`, because if there's multiple values per prediction, then it's unclear which one denotes the similarity. In short: CrossEncoder models with more than 1 label can't be used with `CrossEncoder.rank` at the moment, only with `CrossEncoder.predict`, and then you can do the ranking yourself if you know which value corresponds with similarity.

- Tom Aarsen
ahh okay makes sense, i can help with a PR for this if youre not working on this 😊
That would be much appreciated!

- Tom Aarsen


diff --git a/sentence_transformers/evaluation/NanoBEIREvaluator.py b/sentence_transformers/evaluation/NanoBEIREvaluator.py
index ec29ad094..82907e3ed 100644
--- a/sentence_transformers/evaluation/NanoBEIREvaluator.py
+++ b/sentence_transformers/evaluation/NanoBEIREvaluator.py
@@ -420,6 +420,8 @@ def _load_dataset(self, dataset_name: DatasetNameType, **ir_evaluator_kwargs) ->
         )

     def _validate_dataset_names(self):
+        if len(self.dataset_names) == 0:
+            raise ValueError("dataset_names cannot be empty. Use None to evaluate on all datasets.")
         if missing_datasets := [
             dataset_name for dataset_name in self.dataset_names if dataset_name.lower() not in dataset_name_to_id
         ]:

NanoBeirEvaluator takes in empty dataset
I noticed that NanoBeir takes in an empty dataset and returns no error during instantiation. But when the model gets passed, the error kinda seems confusing.
So -
1. is it fine to allow the evaluator to take in an empty dataset.
2. Should we change the error message saying "invalid datset or something"

Who can help?
@tomaarsen

Snippet reproduction
```
from sentence_transformers import SentenceTransformer
from sentence_transformers.evaluation import NanoBEIREvaluator

datasets = []

#model = SentenceTransformer("sentence-transformers-testing/stsb-bert-tiny-safetensors")

evaluator = NanoBEIREvaluator( # this bit here returns no error
    dataset_names=datasets,
)

results = evaluator(model) #this raised an error
```

#####################################################################

Error log
```
{
	"name": "KeyError",
	"message": "'cosine_ndcg@10'",
	"stack": "---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
Cell In[4], line 1
----> 1 evaluator(model)

sentence_transformers\\evaluation\\NanoBEIREvaluator.py:351, in NanoBEIREvaluator.__call__(self, model, output_path, epoch, steps, *args, **kwargs)
    348 if not self.primary_metric:
    349     if self.main_score_function is None:
    350         score_function = max(
--> 351             [(name, agg_results[f\"{name}_ndcg@{max(self.ndcg_at_k)}\"]) for name in self.score_function_names],
    352             key=lambda x: x[1],
    353         )[0]
    354         self.primary_metric = f\"{score_function}_ndcg@{max(self.ndcg_at_k)}\"
    355     else:

sentence-transformers\\sentence_transformers\\evaluation\\NanoBEIREvaluator.py:351, in <listcomp>(.0)
    348 if not self.primary_metric:
    349     if self.main_score_function is None:
    350         score_function = max(
--> 351             [(name, agg_results[f\"{name}_ndcg@{max(self.ndcg_at_k)}\"]) for name in self.score_function_names],
    352             key=lambda x: x[1],
    353         )[0]
    354         self.primary_metric = f\"{score_function}_ndcg@{max(self.ndcg_at_k)}\"
    355     else:

KeyError: 'cosine_ndcg@10'"
}
```

Hello!

Well spotted, I think we should indeed add an error stating that `dataset_names` cannot be an empty list. It can be `None`, which will default to a full list, but it shouldn't be empty.

- Tom Aarsen
cool, ill be away for a few days im going back to school for my graduation :)

ill be back and raise a PR for this.
Congratulations! 🤗


diff --git a/sentence_transformers/SentenceTransformer.py b/sentence_transformers/SentenceTransformer.py
index 830564a57..ba02f8bfa 100644
--- a/sentence_transformers/SentenceTransformer.py
+++ b/sentence_transformers/SentenceTransformer.py
@@ -1559,8 +1559,8 @@ def _load_module_class_from_ref(
                     revision=revision,
                     code_revision=code_revision,
                 )
-            except OSError:
-                # Ignore the error if the file does not exist, and fall back to the default import
+            except (OSError, ValueError):
+                # Ignore the error if 1) the file does not exist, or 2) the class_ref is not correctly formatted/found
                 pass

         return import_from_string(class_ref)

Breaking change for module loading in PyLate
Hello,

Since the version 3.1, the module are loaded with [get_class_from_dynamic_module](https://github.com/UKPLab/sentence-transformers/commit/6257cb00ef87ddc8f445f0e436377ea4c48879b2#diff-b85567d4fdaffe34a3ccd8fe6cd1fcb15a986ebd34af373c71f1f5cf5efff021R1466) when using ```trust_remote_code=True```
However, this function [breaks if the type of the modules contains more than one dot](https://github.com/huggingface/transformers/blob/31299670cda29f25fbc655f6f166e7b8cc21c89f/src/transformers/dynamic_module_utils.py#L536)
The Dense module from PyLate is saved with the type pylate.models.Dense.Dense, [due to this line](https://github.com/UKPLab/sentence-transformers/blob/ba8cb2ef1d7a5ad6b9169eddee469e900456532c/sentence_transformers/SentenceTransformer.py#L1189).

Before 3.1, the type of the module was pylate.models.Dense, but this still break when parsing on the dot. As the PyLate module still loads correctly with the ```import_from_string``` function (that was used to load the modules prior to 3.1), one of the solution is to broaden a bit the exception to prevent crash if the file is not there locally AND also if the parsing fail (```ValueError: too many values to unpack (expected 2)```)
The other possibility is to add ```class_ref.startswith("pylate.")``` to [this test](https://github.com/UKPLab/sentence-transformers/blob/ba8cb2ef1d7a5ad6b9169eddee469e900456532c/sentence_transformers/SentenceTransformer.py#L1550). It is a bit hardcoded but prevents not catching an exception earlier if there is some other ValueError

Let me know what solution you prefer and I can create a PR for this


diff --git a/sentence_transformers/evaluation/SentenceEvaluator.py b/sentence_transformers/evaluation/SentenceEvaluator.py
index 4708ea6e2..da5876208 100644
--- a/sentence_transformers/evaluation/SentenceEvaluator.py
+++ b/sentence_transformers/evaluation/SentenceEvaluator.py
@@ -57,7 +57,7 @@ def __call__(
     def prefix_name_to_metrics(self, metrics: dict[str, float], name: str) -> dict[str, float]:
         if not name:
             return metrics
-        metrics = {name + "_" + key: value for key, value in metrics.items()}
+        metrics = {name + "_" + key: float(value) for key, value in metrics.items()}
         if hasattr(self, "primary_metric") and not self.primary_metric.startswith(name + "_"):
             self.primary_metric = name + "_" + self.primary_metric
         return metrics

NanoBeirEvaluator returns a mix of floats and numpy.float64 in the results.
I was writing test cases and noticed that the results were returned as a mix of floats and np floats. Should we aim to convert these to floats as well?

```
{'NanoQuoraRetrieval_cosine_accuracy@1': 0.1,
 'NanoQuoraRetrieval_cosine_accuracy@3': 0.18,
 'NanoQuoraRetrieval_cosine_accuracy@5': 0.24,
 'NanoQuoraRetrieval_cosine_accuracy@10': 0.28,
 'NanoQuoraRetrieval_cosine_precision@1': np.float64(0.1),
 'NanoQuoraRetrieval_cosine_precision@3': np.float64(0.06666666666666667),
 'NanoQuoraRetrieval_cosine_precision@5': np.float64(0.052000000000000005),
 'NanoQuoraRetrieval_cosine_precision@10': np.float64(0.034),
 'NanoQuoraRetrieval_cosine_recall@1': np.float64(0.09),
 'NanoQuoraRetrieval_cosine_recall@3': np.float64(0.18),
 'NanoQuoraRetrieval_cosine_recall@5': np.float64(0.21666666666666665),
 'NanoQuoraRetrieval_cosine_recall@10': np.float64(0.27),
 'NanoQuoraRetrieval_cosine_ndcg@10': np.float64(0.1759841616710832),
 'NanoQuoraRetrieval_cosine_mrr@10': 0.14733333333333334,
 'NanoQuoraRetrieval_cosine_map@100': np.float64(0.15211020997551106),
 'NanoMSMARCO_cosine_accuracy@1': 0.0,
 'NanoMSMARCO_cosine_accuracy@3': 0.02,
 'NanoMSMARCO_cosine_accuracy@5': 0.04,
 'NanoMSMARCO_cosine_accuracy@10': 0.06,
 'NanoMSMARCO_cosine_precision@1': np.float64(0.0),
 'NanoMSMARCO_cosine_precision@3': np.float64(0.006666666666666666),
 'NanoMSMARCO_cosine_precision@5': np.float64(0.008),
 'NanoMSMARCO_cosine_precision@10': np.float64(0.006000000000000001),
 'NanoMSMARCO_cosine_recall@1': np.float64(0.0),
 'NanoMSMARCO_cosine_recall@3': np.float64(0.02),
 'NanoMSMARCO_cosine_recall@5': np.float64(0.04),
 'NanoMSMARCO_cosine_recall@10': np.float64(0.06),
 'NanoMSMARCO_cosine_ndcg@10': np.float64(0.02702231788278665),
 'NanoMSMARCO_cosine_mrr@10': 0.016857142857142855,
 'NanoMSMARCO_cosine_map@100': np.float64(0.019069983940167533),
 'NanoBEIR_mean_cosine_accuracy@1': np.float64(0.05),
 'NanoBEIR_mean_cosine_accuracy@3': np.float64(0.09999999999999999),
 'NanoBEIR_mean_cosine_accuracy@5': np.float64(0.13999999999999999),
 'NanoBEIR_mean_cosine_accuracy@10': np.float64(0.17),
 'NanoBEIR_mean_cosine_precision@1': np.float64(0.05),
 'NanoBEIR_mean_cosine_precision@3': np.float64(0.03666666666666667),
 'NanoBEIR_mean_cosine_precision@5': np.float64(0.030000000000000002),
 'NanoBEIR_mean_cosine_precision@10': np.float64(0.02),
 'NanoBEIR_mean_cosine_recall@1': np.float64(0.045),
 'NanoBEIR_mean_cosine_recall@3': np.float64(0.09999999999999999),
 'NanoBEIR_mean_cosine_recall@5': np.float64(0.12833333333333333),
 'NanoBEIR_mean_cosine_recall@10': np.float64(0.165),
 'NanoBEIR_mean_cosine_ndcg@10': np.float64(0.10150323977693491),
 'NanoBEIR_mean_cosine_mrr@10': np.float64(0.0820952380952381),
 'NanoBEIR_mean_cosine_map@100': np.float64(0.0855900969578393)}
```

Who can help?
@tomaarsen

Hello!

Yes, it'd be ideal to normalize this to floats. I think this might mean that the same happens in InformationRetrievalEvaluator.

- Tom Aarsen
cool, ill run across the other ones too and check if the same happens and raise a PR at one go