zRzRzRzRzRzRzR
commited on
Commit
•
37f2196
1
Parent(s):
b098244
fix for apply_chat_template
Browse files- tokenization_chatglm.py +1 -1
tokenization_chatglm.py
CHANGED
@@ -32,7 +32,7 @@ class SPTokenizer:
|
|
32 |
self.special_tokens[token] = self.n_words
|
33 |
self.index_special_tokens[self.n_words] = token
|
34 |
self.n_words += 1
|
35 |
-
self.role_special_token_expression = "|".join([re.escape(token) for token in
|
36 |
|
37 |
def tokenize(self, s: str, encode_special_tokens=False):
|
38 |
if encode_special_tokens:
|
|
|
32 |
self.special_tokens[token] = self.n_words
|
33 |
self.index_special_tokens[self.n_words] = token
|
34 |
self.n_words += 1
|
35 |
+
self.role_special_token_expression = "|".join([re.escape(token) for token in special_tokens]) # for apply_chat_template
|
36 |
|
37 |
def tokenize(self, s: str, encode_special_tokens=False):
|
38 |
if encode_special_tokens:
|