Hugging Face's logo

Khuong-itdev
/
IT_Lecture_QnA_Assistant

Text Generation
Transformers
TensorBoard
Safetensors
mistral
Generated from Trainer
trl
sft
conversational
text-generation-inference
Model card Files Metrics Community
Khuong-itdev commited on
Commit
bc177ae
·
verified ·
1 Parent(s): a7cd896

Upload tokenizer

Browse files
Files changed (2) hide show
  1. special_tokens_map.json +1 -7
  2. tokenizer_config.json +1 -1
special_tokens_map.json CHANGED
@@ -18,13 +18,7 @@
18
  "rstrip": false,
19
  "single_word": false
20
  },
21
- "pad_token": {
22
- "content": "<unk>",
23
- "lstrip": false,
24
- "normalized": false,
25
- "rstrip": false,
26
- "single_word": false
27
- },
28
  "unk_token": {
29
  "content": "<unk>",
30
  "lstrip": false,
 
18
  "rstrip": false,
19
  "single_word": false
20
  },
21
+ "pad_token": "</s>",
 
 
 
 
 
 
22
  "unk_token": {
23
  "content": "<unk>",
24
  "lstrip": false,
tokenizer_config.json CHANGED
@@ -72,7 +72,7 @@
72
  "extra_special_tokens": {},
73
  "legacy": true,
74
  "model_max_length": 1000000000000000019884624838656,
75
- "pad_token": "<unk>",
76
  "sp_model_kwargs": {},
77
  "spaces_between_special_tokens": false,
78
  "tokenizer_class": "LlamaTokenizer",
 
72
  "extra_special_tokens": {},
73
  "legacy": true,
74
  "model_max_length": 1000000000000000019884624838656,
75
+ "pad_token": "</s>",
76
  "sp_model_kwargs": {},
77
  "spaces_between_special_tokens": false,
78
  "tokenizer_class": "LlamaTokenizer",