zRzRzRzRzRzRzR
commited on
Commit
•
456aa87
1
Parent(s):
340a2eb
fix piece id out of range (#20)
Browse files- fix piece id out of range (7a8cbaadbd1a69143089c30a5cb8ec66d5443dc3)
- tokenization_chatglm.py +1 -1
tokenization_chatglm.py
CHANGED
@@ -83,7 +83,7 @@ class SPTokenizer:
|
|
83 |
"""Converts an index (integer) in a token (str) using the vocab."""
|
84 |
if index in self.index_special_tokens:
|
85 |
return self.index_special_tokens[index]
|
86 |
-
if index in [self.eos_id, self.bos_id, self.pad_id] or index < 0:
|
87 |
return ""
|
88 |
return self.sp_model.IdToPiece(index)
|
89 |
|
|
|
83 |
"""Converts an index (integer) in a token (str) using the vocab."""
|
84 |
if index in self.index_special_tokens:
|
85 |
return self.index_special_tokens[index]
|
86 |
+
if index in [self.eos_id, self.bos_id, self.pad_id] or index < 0 or index > self.sp_model.vocab_size():
|
87 |
return ""
|
88 |
return self.sp_model.IdToPiece(index)
|
89 |
|