update

doc/chat-template/DeepSeek-R1-0528/chat_template.jinja

@@ -0,0 +1,14 @@
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='', is_first_sp=true, is_last_user=false) %}{%- for message in messages %}{%- if message['role'] == 'system' %}{%- if ns.is_first_sp %}{% set ns.system_prompt = ns.system_prompt + message['content'] %}{% set ns.is_first_sp = false %}{%- else %}{% set ns.system_prompt = ns.system_prompt + '
+
+' + message['content'] %}{%- endif %}{%- endif %}{%- endfor %}{{ bos_token }}{{ ns.system_prompt }}{%- for message in messages %}{% set content = message['content'] %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{%- set ns.is_first = false -%}{%- set ns.is_last_user = true -%}{{'<｜User｜>' + content + '<｜Assistant｜>'}}{%- endif %}{%- if message['role'] == 'assistant' %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{% endif %}{%- if message['role'] == 'assistant' and message['tool_calls'] is defined and message['tool_calls'] is not none %}{%- set ns.is_last_user = false -%}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{%- endif %}{%- set ns.is_first = false %}{%- set ns.is_tool = false -%}{%- set ns.is_output_first = true %}{%- for tool in message['tool_calls'] %}{%- if not ns.is_first %}{%- if content is none %}{{'<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '
+' + '```json' + '
+' + tool['function']['arguments'] + '
+' + '```' + '<｜tool▁call▁end｜>'}}{%- else %}{{content + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '
+' + '```json' + '
+' + tool['function']['arguments'] + '
+' + '```' + '<｜tool▁call▁end｜>'}}{%- endif %}{%- set ns.is_first = true -%}{%- else %}{{'
+' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '
+' + '```json' + '
+' + tool['function']['arguments'] + '
+' + '```' + '<｜tool▁call▁end｜>'}}{%- endif %}{%- endfor %}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- if message['role'] == 'assistant' and (message['tool_calls'] is not defined or message['tool_calls'] is none)%}{%- set ns.is_last_user = false -%}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + content + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{{content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_last_user = false -%}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + content + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'
+<｜tool▁output▁begin｜>' + content + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_last_user and not ns.is_tool %}{{'<｜Assistant｜>'}}{% endif %}

doc/chat-template/DeepSeek-R1-0528/deepseek.md

@@ -0,0 +1,151 @@
+
+
+## R1-template
+
+```sh
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='', is_first_sp=true) %}{%- for message in messages %}{%- if message['role'] == 'system' %}{%- if ns.is_first_sp %}{% set ns.system_prompt = ns.system_prompt + message['content'] %}{% set ns.is_first_sp = false %}{%- else %}{% set ns.system_prompt = ns.system_prompt + '\n\n' + message['content'] %}{%- endif %}{%- endif %}{%- endfor %}{{ bos_token }}{{ ns.system_prompt }}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and 'tool_calls' in message %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls'] %}{%- if not ns.is_first %}{%- if message['content'] is none %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}{%- else %}{{'<｜Assistant｜>' + message['content'] + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<tool▁call▁end｜>'}}{%- endif %}{%- set ns.is_first = true -%}{%- else %}{{'\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}{%- endif %}{%- endfor %}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- if message['role'] == 'assistant' and 'tool_calls' not in message %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜>'}}{% endif %}
+```
+
+FAQ：
+- 问: deepseek 的 system 为什么没有 eos_token？
+- 答: 因为 deepseek 没有单独的 system special token，而是仅仅讲system内容拼到了user前而已。 `{{ bos_token }}{{ ns.system_prompt }}  {{'<｜User｜>' + message['content']}}`
+
+- 问: 是否推荐用system
+- 答: https://github.com/deepseek-ai/DeepSeek-R1/blob/main/README.md 这里不推荐用system
+
+- 问: 如果不用system，system的内容放在哪呢？
+- 答: ss
+
+- 问: vllm 的推理，为什么给system之后添加了 eos_token?
+- 答: 不知道
+
+- 问: 为什么是 end▁of▁sentence 不是 end_of_sentence？
+- 答: 
+
+
+官方更新
+- 增加 `<｜Assistant｜><think>\n`，强制输出 `<think>`
+  - https://huggingface.co/deepseek-ai/DeepSeek-R1/commit/8a58a132790c9935686eb97f042afa8013451c9f
+-  `"add_bos_token": false`， 改成了 `true`, 
+  - https://huggingface.co/deepseek-ai/DeepSeek-R1/commit/cb48aa8cb28c160ec8d853707278e0402c9ad01a
+
+```sh
+{% if not add_generation_prompt is defined %}
+  {% set add_generation_prompt = false %}
+{% endif %}
+{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='', is_first_sp=true) %}
+{%- for message in messages %}
+  {%- if message['role'] == 'system' %}
+    {%- if ns.is_first_sp %}
+      {% set ns.system_prompt = ns.system_prompt + message['content'] %}
+      {% set ns.is_first_sp = false %}
+    {%- else %}
+      {% set ns.system_prompt = ns.system_prompt + '\n\n' + message['content'] %}
+    {%- endif %}
+  {%- endif %}
+{%- endfor %}
+{{ bos_token }}{{ ns.system_prompt }}
+{%- for message in messages %}
+  {%- if message['role'] == 'user' %}
+    {%- set ns.is_tool = false -%}
+    {{'<｜User｜>' + message['content']}}
+  {%- endif %}
+{%- if message['role'] == 'assistant' and 'tool_calls' in message %}
+  {%- set ns.is_tool = false -%}
+  {%- for tool in message['tool_calls'] %}
+    {%- if not ns.is_first %}
+      {%- if message['content'] is none %}
+        {{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}
+      {%- else %}
+        {{'<｜Assistant｜>' + message['content'] + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<tool▁call▁end｜>'}}
+      {%- endif %}
+      {%- set ns.is_first = true -%}
+    {%- else %}
+      {{'\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}
+    {%- endif %}
+  {%- endfor %}
+  {{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}
+{%- endif %}
+{%- if message['role'] == 'assistant' and 'tool_calls' not in message %}
+  {%- if ns.is_tool %}
+    {{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}
+    {%- set ns.is_tool = false -%}
+  {%- else %}
+    {% set content = message['content'] %}
+    {% if '</think>' in content %}
+      {% set content = content.split('</think>')[-1] %}{% endif %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}
+    {%- endif %}
+  {%- endif %}
+  {%- if message['role'] == 'tool' %}
+    {%- set ns.is_tool = true -%}
+    {%- if ns.is_output_first %}
+      {{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}
+      {%- set ns.is_output_first = false %}
+    {%- else %}
+      {{'<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}
+    {%- endif %}
+  {%- endif %}
+{%- endfor -%}
+{% if ns.is_tool %}
+  {{'<｜tool▁outputs▁end｜>'}}
+{% endif %}
+{% if add_generation_prompt and not ns.is_tool %}
+  {{'<｜Assistant｜>'}}
+{% endif %}
+```
+
+
+```sh
+## DeepSeek-R1-Distill-Qwen-1.5B
+system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif |~
+  %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['too|~
+  l_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>|~
+  ' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{|~
+  %- set ns.is_first = true -%}{%- else %}{{'\\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['nam|~
+  e'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜|~
+  end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none |~
+  %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- |~
+  else %}{% set content = message['content'] %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endi|~
+  f %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.|~
+  is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁o|~
+  utput▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\\n
+
+## R1
+system_prompt='', is_first_sp=true) %}{%- for message in messages %}{%- if message['role'] == 'system' %}{%- if ns.is_first_sp %}{% set ns.system_prompt = ns.system_prompt + message['content'] %}{% set ns.is_f|~
+  irst_sp = false %}{%- else %}{% set ns.system_prompt = ns.system_prompt + '\\n\\n' + message['content'] %}{%- endif %}{%- endif %}{%-|~
+   endfor %}{{ bos_token }}{{ ns.system_prompt }}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = |~
+  false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and 'tool_calls' in message %}{%- set|~
+   ns.is_tool = false -%}{%- for tool in message['tool_calls'] %}{%- if not ns.is_first %}{%- if message['content'] is none %}{{'<｜Ass|~
+  istant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json|~
+  ' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{%- else %}{{'<｜Assistant｜>' + message['content'|~
+  ] + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + |~
+  '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{%- endif %}{%- set ns.is_first = true -%}{%- else %}{|~
+  {'\\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['fun|~
+  ction']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{%- endif %}{%- endfor %}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}|~
+  }{%- endif %}{%- if message['role'] == 'assistant' and 'tool_calls' not in message %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' +|~
+   message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '|~
+  </think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentenc|~
+  e｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool|~
+  ▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- el|~
+  se %}{{'
+
+```
+
+
+## V3-template  (没有think)
+
+```sh
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='', is_first_sp=true) %}{%- for message in messages %}{%- if message['role'] == 'system' %}{%- if ns.is_first_sp %}{% set ns.system_prompt = ns.system_prompt + message['content'] %}{% set ns.is_first_sp = false %}{%- else %}{% set ns.system_prompt = ns.system_prompt + '
+
+' + message['content'] %}{%- endif %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '
+' + '```json' + '
+' + tool['function']['arguments'] + '
+' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'
+' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '
+' + '```json' + '
+' + tool['function']['arguments'] + '
+' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁
+of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{{'<｜Assistant｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜
+tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'
+<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜>'}}{% endif %}
+```

doc/chat-template/DeepSeek-R1-0528/demo.py

@@ -0,0 +1,75 @@
+"""
+<｜begin▁of▁sentence｜><｜User｜>Hello, how are you?<｜Assistant｜>I'm doing great. How can I help you today?<｜end▁of▁sentence｜><｜User｜>I'd like to show off how chat templating works!
+
+"""
+
+from transformers import AutoTokenizer
+
+from transformers import AutoTokenizer
+# tokenizer = AutoTokenizer.from_pretrained("deepseek-ai/DeepSeek-V3")
+
+# tokenizer = AutoTokenizer.from_pretrained("deepseek-ai/DeepSeek-R1")
+tokenizer = AutoTokenizer.from_pretrained("deepseek-ai/DeepSeek-R1-0528")
+"""
+
+"""
+
+chat = [
+  {"role": "system", "content": "you are a helpful assistant."},
+  {"role": "user", "content": "Hello, how are you?"},
+  {"role": "assistant", "content": "I'm doing great. How can I help you today?"},
+  {"role": "user", "content": "I'd like to show off how chat templating works!"},
+  {"role": "tool", "content": "789"},
+  {"role": "assistant", "content": "123"},
+  {"role": "user", "content": "456"},
+  {"role": "tool", "content": "awq"},
+  
+]
+
+
+# chat = [
+  
+#   {"role": "user", "content": "Hello, how are you?"},
+#   {"role": "assistant", "content": "<think>i am thinking</think>I'm doing great. How can I help you today?"},
+#   {"role": "user", "content": "I'd like to show off how chat templating works!"},
+# ]
+
+def get_weather(location: str, unit: str):
+    """
+    Get the current weather in a given location.
+
+    Args:
+        location: The city and state, e.g., 'San Francisco, CA'.
+        unit: The unit of temperature, either 'celsius' or 'fahrenheit'.
+
+    Returns:
+        str: The current weather in the given location.
+    """
+    return f"Getting the weather for {location} in {unit}..."
+  
+
+prompt = tokenizer.apply_chat_template(chat, tools=[get_weather], tokenize=False)
+
+
+print(prompt)
+
+aad
+
+prompt_ids = tokenizer.apply_chat_template(chat)
+print(prompt_ids)
+
+"""
+<｜begin▁of▁sentence｜>you are a helpful assistant.<｜User｜>Hello, how are you?<｜Assistant｜>I'm doing great. How can I help you today?<｜end▁of▁sentence｜><｜User｜>I'd like to show off how chat templating works!<｜Assistant｜>123<｜end▁of▁sentence｜><｜User｜>456
+
+
+[0, 12829, 477, 260, 11502, 22896, 16, 128803, 19923, 14, 1192, 477, 440, 33, 128804, 43, 4571, 4843, 2405, 16, 1730, 588, 342, 1694, 440, 4316, 33, 1, 128803, 43, 7485, 1277, 304, 1801, 1375, 1192, 20297, 12202, 1217, 2984, 3, 128804, 6895, 1, 128803, 18009]
+
+0: <｜begin▁of▁sentence｜>
+1: <｜end▁of▁sentence｜>
+128803: <｜User｜>
+128804: <｜Assistant｜>
+"""
+
+text = tokenizer.decode([0, 15061, 14928, 35895, 23379, 303, 2788, 35895, 14928, 4, 844, 60949, 4, 24415, 27318, 478, 7625, 34092, 7524, 14928])
+
+print(text)

doc/chat-template/Hermes-3-Llama-3.1-405B/README.md

@@ -0,0 +1,64 @@
+## tool_call
+
+支持多个tool，
+
+
+
+## llama3.1-405b
+
+```yml
+<|begin_of_text|><|start_header_id|>system<|end_header_id|>
+
+Cutting Knowledge Date: December 2023
+Today Date: 26 Jul 2024
+
+You are a bot that responds to weather queries.<|eot_id|><|start_header_id|>user<|end_header_id|>
+
+Hey, what's the temperature in Paris right now?<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+{"name": "get_current_temperature", "parameters": {"location": "Paris, France"}}<|eot_id|><|start_header_id|>ipython<|end_header_id|>
+
+"22.0"<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+```
+
+
+## hermes 示例
+
+```sh
+<|im_start|>system
+You are a helpful assistant. The date today is 12/26/24.<|im_end|>
+
+<|im_start|>user
+who won the last womens singles wimbledon<|im_end|>
+
+<|im_start|>assistant
+<tool_call>
+{"name": "tavily_search_results_json", "arguments": {"query": "last women's singles wimbledon winner"}}
+</tool_call><|im_end|>
+
+<|im_start|>tool
+<tool_response>
+[{"url": "https://en.wikipedia.org/wiki/List_of_Wimbledon_ladies'_singles_champions", "content": "***"}, {"url": "https://www.tennis-x.com/winners/womens/wimbledon.php", "content": "***"},...]
+</tool_response><|im_end|>
+
+<|im_start|>assistant
+```
+
+
+
+很好，
+1. 简化了 llama的special token ( ):
+    - llama: `<|start_header_id|>user<|end_header_id|>`
+    - hermes: `<|im_start|>user`
+2. print出来更好看
+    - llama: ss
+    - hermes: ss
+3. tools:
+    - llama: tool统一用的 `ipython` 标识符；没有 `tool_response`
+
+
+
+## TODO:
+
+- 去掉日期
+- 

doc/chat-template/Hermes-3-Llama-3.1-405B/chat_template.default.jinja

@@ -0,0 +1,6 @@
+{{bos_token}}{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system
+You are a helpful assistant.<|im_end|>
+' }}{% endif %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

doc/chat-template/Hermes-3-Llama-3.1-405B/chat_template.tool_use.jinja

@@ -0,0 +1,152 @@
+{%- macro json_to_python_type(json_spec) %}
+{%- set basic_type_map = {
+    "string": "str",
+    "number": "float",
+    "integer": "int",
+    "boolean": "bool"
+} %}
+
+{%- if basic_type_map[json_spec.type] is defined %}
+    {{- basic_type_map[json_spec.type] }}
+{%- elif json_spec.type == "array" %}
+    {{- "list[" +  json_to_python_type(json_spec|items) + "]"}}
+{%- elif json_spec.type == "object" %}
+    {%- if json_spec.additionalProperties is defined %}
+        {{- "dict[str, " + json_to_python_type(json_spec.additionalProperties) + ']'}}
+    {%- else %}
+        {{- "dict" }}
+    {%- endif %}
+{%- elif json_spec.type is iterable %}
+    {{- "Union[" }}
+    {%- for t in json_spec.type %}
+      {{- json_to_python_type({"type": t}) }}
+      {%- if not loop.last %}
+        {{- "," }} 
+    {%- endif %}
+    {%- endfor %}
+    {{- "]" }}
+{%- else %}
+    {{- "Any" }}
+{%- endif %}
+{%- endmacro %}
+
+
+{{- bos_token }}
+{{- '<|im_start|>system
+' }}
+{{- "You are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. Here are the available tools: <tools> " }}
+{%- for tool in tools %}
+    {%- if tool.function is defined %}
+        {%- set tool = tool.function %}
+    {%- endif %}
+    {{- '{"type": "function", "function": ' }}
+    {{- '{"name": "' + tool.name + '", ' }}
+    {{- '"description": "' + tool.name + '(' }}
+    {%- for param_name, param_fields in tool.parameters.properties|items %}
+        {{- param_name + ": " + json_to_python_type(param_fields) }}
+        {%- if not loop.last %}
+            {{- ", " }}
+        {%- endif %}
+    {%- endfor %}
+    {{- ")" }}
+    {%- if tool.return is defined %}
+        {{- " -> " + json_to_python_type(tool.return) }}
+    {%- endif %}
+    {{- " - " + tool.description + "
+
+" }}
+    {%- for param_name, param_fields in tool.parameters.properties|items %}
+        {%- if loop.first %}
+            {{- "    Args:
+" }}
+        {%- endif %}
+        {{- "        " + param_name + "(" + json_to_python_type(param_fields) + "): " + param_fields.description|trim }}
+    {%- endfor %}
+    {%- if tool.return is defined and tool.return.description is defined %}
+        {{- "
+    Returns:
+        " + tool.return.description }}
+    {%- endif %}
+    {{- '"' }}
+    {{- ', "parameters": ' }}
+    {%- if tool.parameters.properties | length == 0 %}
+        {{- "{}" }}
+    {%- else %}
+        {{- tool.parameters|tojson }}
+    {%- endif %}
+    {{- "}" }}
+    {%- if not loop.last %}
+        {{- "
+" }}
+    {%- endif %}
+{%- endfor %}
+{{- " </tools>" }}
+{{- 'Use the following pydantic model json schema for each tool call you will make: {"properties": {"name": {"title": "Name", "type": "string"}, "arguments": {"title": "Arguments", "type": "object"}}, "required": ["name", "arguments"], "title": "FunctionCall", "type": "object"}}
+' }}
+{{- "For each function call return a json object with function name and arguments within <tool_call></tool_call> XML tags as follows:
+" }}
+{{- "<tool_call>
+" }}
+{{- '{"name": <function-name>, "arguments": <args-dict>}
+' }}
+{{- '</tool_call><|im_end|>
+' }}
+{%- for message in messages %}
+    {%- if message.role == "user" or message.role == "system" or (message.role == "assistant" and message.tool_calls is not defined) %}
+        {{- '<|im_start|>' + message.role + '
+' + message.content + '<|im_end|>' + '
+' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+    {%- for tool_call in message.tool_calls %}
+       {{- '
+<tool_call>
+' }}           {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '{' }}
+            {{- '"name": "' }}
+            {{- tool_call.name }}
+            {{- '"' }}
+            {{- ', '}}
+            {%- if tool_call.arguments is defined %}
+                {{- '"arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments|tojson }}
+                {%- endif %}
+            {%- endif %}
+             {{- '}' }}
+            {{- '
+</tool_call>' }}
+    {%- endfor %}
+        {{- '<|im_end|>
+' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.previtem and loop.previtem.role != "tool" %}
+            {{- '<|im_start|>tool
+' }}
+        {%- endif %}
+        {{- '<tool_response>
+' }}
+        {{- message.content }}
+        {%- if not loop.last %}
+            {{- '
+</tool_response>
+' }}
+        {%- else %}
+            {{- '
+</tool_response>' }}
+        {%- endif %}
+        {%- if not loop.last and loop.nextitem.role != "tool" %}
+            {{- '<|im_end|>' }}
+        {%- elif loop.last %}
+            {{- '<|im_end|>' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant
+' }}
+{%- endif %}

doc/chat-template/Llama-3.1-405B-Instruct/README.md

@@ -1,23 +0,0 @@
-
-
-
-## tool 示例
-
-```yml
-<|begin_of_text|><|start_header_id|>system<|end_header_id|>
-
-Cutting Knowledge Date: December 2023
-Today Date: 26 Jul 2024
-
-You are a bot that responds to weather queries.<|eot_id|><|start_header_id|>user<|end_header_id|>
-
-Hey, what's the temperature in Paris right now?<|eot_id|><|start_header_id|>assistant<|end_header_id|>
-
-{"name": "get_current_temperature", "parameters": {"location": "Paris, France"}}<|eot_id|><|start_header_id|>ipython<|end_header_id|>
-
-"22.0"<|eot_id|><|start_header_id|>assistant<|end_header_id|>
-```
-
-
-
-缺陷：见 hermes/README.md

doc/chat-template/Llama-3.1-405B-Instruct/{chat_template.md → chat_template.jinja}

@@ -1,8 +1,3 @@
-
-
-## chat_template
-
-```php
 {{- bos_token }}
 {%- if custom_tools is defined %}
     {%- set tools = custom_tools %}
@@ -112,4 +107,3 @@
 {%- if add_generation_prompt %}
     {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
 {%- endif %}
-```

doc/chat-template/Llama-3.1-405B-Instruct/demo.py

@@ -1,20 +0,0 @@
-
-
-
-from transformers import AutoTokenizer
-
-tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3.1-405B", use_fast=False)
-
-
-messages = [
-    {"role": "user", "content": "你好"},
-    {"role": "assistant", "content": "good"},
-]
-
-
-tokenizer.apply_chat_template(messages, )
-
-
-
-# print(token_id, decoding)
-

doc/chat-template/Llama-3.1-405B-Instruct/generate.py

@@ -1,24 +0,0 @@
-
-
-import transformers
-import torch
-
-model_id = "/workspace/czy/model_weights/Meta-Llama-3.1-8B-Instruct/"
-
-pipeline = transformers.pipeline(
-    "text-generation",
-    model=model_id,
-    model_kwargs={"torch_dtype": torch.bfloat16},
-    device_map="auto",
-)
-
-messages = [
-    {"role": "system", "content": "You are a pirate chatbot who always responds in pirate speak!"},
-    {"role": "user", "content": "Who are you?"},
-]
-
-outputs = pipeline(
-    messages,
-    max_new_tokens=256,
-)
-print(outputs[0]["generated_text"][-1])

doc/chat-template/export_chat_template.py

@@ -0,0 +1,35 @@
+
+
+import os
+import json
+from transformers import AutoTokenizer
+from transformers.utils import get_json_schema
+
+
+# MODEL_PATH = "meta-llama/Llama-3.1-405B-Instruct"
+MODEL_PATH = "NousResearch/Hermes-3-Llama-3.1-405B"  # messages里不支持tool_calls 
+# MODEL_PATH = "../../test/Llama-4-Maverick-17B-128E-Instruct/"
+# MODEL_PATH = "meta-llama/Llama-4-Maverick-17B-128E-Instruct"
+# MODEL_PATH = "Qwen/Qwen3-235B-A22B-Instruct-2507"
+# MODEL_PATH = "mistralai/Mistral-7B-Instruct-v0.1"  # messages里不支持tool_calls，不支持 role=tool，不支持 tools
+# MODEL_PATH = "mistralai/Ministral-8B-Instruct-2410" # 支持 tools, 支持tool_calls(必须要有id), 格式非主流
+MODEL_PATH = "deepseek-ai/DeepSeek-R1"
+# MODEL_PATH = "deepseek-ai/DeepSeek-R1-0528"
+tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)
+chat_template = tokenizer.chat_template
+
+
+output_dir = MODEL_PATH.split("/")[-1]
+
+os.makedirs(output_dir, exist_ok=True)
+if isinstance(chat_template, dict):
+    for k, v in chat_template.items():
+        with open(f"{output_dir}/chat_template.{k}.jinja", "w") as f_out:
+            f_out.write(v)
+else:
+    # chat_template = chat_template.replace("\\n", "\n")
+    with open(f"{output_dir}/chat_template.jinja", "w") as f_out:
+        f_out.write(chat_template)
+    
+
+

doc/chat-template/tool_demo.py

@@ -16,12 +16,13 @@ from transformers import AutoTokenizer
 from transformers.utils import get_json_schema
 
 
-MODEL_PATH = "meta-llama/Llama-3.1-405B-Instruct"
-MODEL_PATH = "NousResearch/Hermes-3-Llama-3.1-405B"  # messages里不支持tool_calls 
-MODEL_PATH = "meta-llama/Llama-4-Maverick-17B-128E-Instruct"
+# MODEL_PATH = "meta-llama/Llama-3.1-405B-Instruct"
+# MODEL_PATH = "NousResearch/Hermes-3-Llama-3.1-405B"  # messages里不支持tool_calls 
+# MODEL_PATH = "../../test/Llama-4-Maverick-17B-128E-Instruct/"
+# MODEL_PATH = "meta-llama/Llama-4-Maverick-17B-128E-Instruct"
 MODEL_PATH = "Qwen/Qwen3-235B-A22B-Instruct-2507"
 # MODEL_PATH = "mistralai/Mistral-7B-Instruct-v0.1"  # messages里不支持tool_calls，不支持 role=tool，不支持 tools
-MODEL_PATH = "mistralai/Ministral-8B-Instruct-2410" # 支持 tools, 支持tool_calls(必须要有id), 格式非主流
+# MODEL_PATH = "mistralai/Ministral-8B-Instruct-2410" # 支持 tools, 支持tool_calls(必须要有id), 格式非主流
 tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)
 
 # First, define a tool
@@ -39,7 +40,9 @@ def get_current_temperature(location: str) -> float:
 # Next, create a chat and apply the chat template
 messages = [
   {"role": "system", "content": "You are a bot that responds to weather queries."},
-  {"role": "user", "content": "Hey, what's the temperature in Paris right now?"}
+  {"role": "user", "content": "Hey, what's the temperature in Paris right now?"},
+  {"role": "assitant", "content": "test1"},
+  {"role": "user", "content": "test2"},
 ]
 
 # step1:

doc/chat-template/tools_and_llm_response.md

@@ -24,8 +24,8 @@ Respond in the format [func_name1(params_name1=params_value1, params_name2=param
 
 ## llm入参：messages and tools
 
-```json
-[
+```py
+messages = [
   {
     "role": "system",
     "content": "You are a bot that responds to weather queries."
@@ -35,6 +35,7 @@ Respond in the format [func_name1(params_name1=params_value1, params_name2=param
     "content": "Hey, what's the temperature in Paris right now?"
   }
 ]
+# chat_completion = client.chat.completions.create(messages=messages, model=model, tools=tools)  # 可以这样调用 LLM
 ```
 
 `json_schema` of tools
@@ -110,9 +111,13 @@ Respond in the format {"name": function name, "parameters": dictionary of argume
 Hey, what's the temperature in Paris right now?<|eot_id|><|start_header_id|>assistant<|end_header_id|>
 ```
 
-- **入参**: 支持的工具列表（`tools`） 是基于 json-schema 的，因为chat_template中采用的是 [`tojson(indent=4)`](https://github.com/vllm-project/vllm/blob/v0.10.1/examples/tool_chat_template_llama3.1_json.jinja#L48)
-- **出参**: 返回的`respone` 要求是 json 格式，
-`respond with a JSON ... Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}`
+- **入参**: 
+  - **tools格式**: 支持的工具列表（`tools`） 是基于 json-schema 的，因为chat_template中采用的是 [`tojson(indent=4)`](https://github.com/vllm-project/vllm/blob/v0.10.1/examples/tool_chat_template_llama3.1_json.jinja#L48)
+  - **tools在prompt中的位置**: 拼在第一个user轮内容的前面。(system不变，其他轮message也不变)
+
+- **出参**: 
+  - **出参格式**: 返回的`respone` 要求是 json 格式，`respond with a JSON ... Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}`
+  - **出参的工具解析**: ?
 
 
 
@@ -140,8 +145,9 @@ Hey, what's the temperature in Paris right now?<|im_end|>
 ```
 
 
-- **入参**: 支持的工具列表（`tools`） 是基于自定义格式的，详见 [chat_template](https://github.com/vllm-project/vllm/blob/v0.10.1/examples/tool_chat_template_hermes.jinja#L41)
-
+- **入参**: 
+  - **tools格式**: 支持的工具列表（`tools`） 是基于自定义格式的，详见 [chat_template](https://github.com/vllm-project/vllm/blob/v0.10.1/examples/tool_chat_template_hermes.jinja#L41)
+  - **tools在prompt中的位置**: 额外增加了一个`system`轮，放在最前面。（用户设置的`system`保持不变）
 - **出参**: 返回的`respone` 要求是 `<tool_call>` 包裹的json
 `return a json object with function name and arguments within <tool_call></tool_call> XML tags as follows: <tool_call>{"name": <function-name>, "arguments": <args-dict>}</tool_call>`
 
@@ -149,6 +155,47 @@ Hey, what's the temperature in Paris right now?<|im_end|>
 ## llama3.2
 
 
+## llama4
+
+```sh
+<|begin_of_text|><|header_start|>system<|header_end|>
+
+Environment: ipython
+You are a bot that responds to weather queries.<|eot|><|header_start|>user<|header_end|>
+
+Given the following functions, please respond with a JSON for a function call with its proper arguments that best answers the given prompt.
+
+Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.Do not use variables.
+
+{
+    "type": "function",
+    "function": {
+        "name": "get_current_temperature",
+        "description": "Get the current temperature at a location.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "location": {
+                    "type": "string",
+                    "description": "The location to get the temperature for, in the format \"City, Country\""
+                }
+            },
+            "required": [
+                "location"
+            ]
+        },
+        "return": {
+            "type": "number",
+            "description": "The current temperature at the specified location in the specified units, as a float."
+        }
+    }
+}
+
+Hey, what's the temperature in Paris right now?<|eot|><|header_start|>assistant<|header_end|>
+```
+
+跟 llama3.1差不多，只是少了`date`，并且换了`special_token`。(同样拼在第一个user轮)
+
 
 ## mistralai/Ministral-8B-Instruct-2410
 
@@ -164,4 +211,33 @@ Hey, what's the temperature in Paris right now?[/INST]
 
 
 
-## ss
+## qwen3
+
+```sh
+<|im_start|>system
+You are a bot that responds to weather queries.
+
+# Tools
+
+You may call one or more functions to assist with the user query.
+
+You are provided with function signatures within <tools></tools> XML tags:
+<tools>
+{"type": "function", "function": {"name": "get_current_temperature", "description": "Get the current temperature at a location.", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The location to get the temperature for, in the format \"City, Country\""}}, "required": ["location"]}, "return": {"type": "number", "description": "The current temperature at the specified location in the specified units, as a float."}}}
+</tools>
+
+For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:
+<tool_call>
+{"name": <function-name>, "arguments": <args-json-object>}
+</tool_call><|im_end|>
+<|im_start|>user
+Hey, what's the temperature in Paris right now?<|im_end|>
+<|im_start|>assistant
+```
+
+
+- **入参**: 
+  - **tools格式**: 支持的工具列表（`tools`） 是基于 json-schema 的.
+  - **tools在prompt中的位置**: 拼接到原始`system`的结尾。
+- **出参**: 返回的`respone` 要求是 `<tool_call>` 包裹的json
+`return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{"name": <function-name>, "arguments": <args-json-object>}\n</tool_call>`