tokenizer
Browse files
scripts/train_tokenizer.py
CHANGED
|
@@ -44,6 +44,8 @@ special_tokens = [
|
|
| 44 |
'</tool_call>',
|
| 45 |
'<tool_response>',
|
| 46 |
'</tool_response>',
|
|
|
|
|
|
|
| 47 |
'<think>',
|
| 48 |
'</think>',
|
| 49 |
'<answer>',
|
|
|
|
| 44 |
'</tool_call>',
|
| 45 |
'<tool_response>',
|
| 46 |
'</tool_response>',
|
| 47 |
+
'<question>',
|
| 48 |
+
'</question>',
|
| 49 |
'<think>',
|
| 50 |
'</think>',
|
| 51 |
'<answer>',
|