sylwia-kuros
commited on
Commit
•
919ac8c
1
Parent(s):
66ecaf1
Update tokenization_chatglm.py
Browse filesFix tokenizer initialization issue: `AttributeError: can't set attribute`
- tokenization_chatglm.py +3 -0
tokenization_chatglm.py
CHANGED
@@ -70,6 +70,9 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
|
|
70 |
|
71 |
self.vocab_file = vocab_file
|
72 |
self.tokenizer = SPTokenizer(vocab_file)
|
|
|
|
|
|
|
73 |
self.special_tokens = {
|
74 |
"<bos>": self.tokenizer.bos_id,
|
75 |
"<eos>": self.tokenizer.eos_id,
|
|
|
70 |
|
71 |
self.vocab_file = vocab_file
|
72 |
self.tokenizer = SPTokenizer(vocab_file)
|
73 |
+
kwargs.pop("eos_token", None)
|
74 |
+
kwargs.pop("pad_token", None)
|
75 |
+
kwargs.pop("unk_token", None)
|
76 |
self.special_tokens = {
|
77 |
"<bos>": self.tokenizer.bos_id,
|
78 |
"<eos>": self.tokenizer.eos_id,
|