sylwia-kuros commited on
Commit
919ac8c
1 Parent(s): 66ecaf1

Update tokenization_chatglm.py

Browse files

Fix tokenizer initialization issue: `AttributeError: can't set attribute`

Files changed (1) hide show
  1. tokenization_chatglm.py +3 -0
tokenization_chatglm.py CHANGED
@@ -70,6 +70,9 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
70
 
71
  self.vocab_file = vocab_file
72
  self.tokenizer = SPTokenizer(vocab_file)
 
 
 
73
  self.special_tokens = {
74
  "<bos>": self.tokenizer.bos_id,
75
  "<eos>": self.tokenizer.eos_id,
 
70
 
71
  self.vocab_file = vocab_file
72
  self.tokenizer = SPTokenizer(vocab_file)
73
+ kwargs.pop("eos_token", None)
74
+ kwargs.pop("pad_token", None)
75
+ kwargs.pop("unk_token", None)
76
  self.special_tokens = {
77
  "<bos>": self.tokenizer.bos_id,
78
  "<eos>": self.tokenizer.eos_id,