geninhu commited on
Commit
1fd946a
·
verified ·
1 Parent(s): c5556f8

Upload of AutoGPTQ quantized model

Browse files
README.md ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: Rakuten/RakutenAI-7B-instruct
3
+ inference: false
4
+ language:
5
+ - en
6
+ - ja
7
+ license: apache-2.0
8
+ model_creator: Rakuten
9
+ model_type: llama
10
+ quantized_by: auto-gptq
11
+ tags:
12
+ - gptq
13
+ - 4bit
14
+ - vllm
15
+ - quantized
16
+ ---
17
+
18
+ # RakutenAI-7B-instruct GPTQ
19
+
20
+ This is a 4-bit GPTQ quantized version of [Rakuten/RakutenAI-7B-instruct](https://huggingface.co/Rakuten/RakutenAI-7B-instruct).
21
+
22
+ ## Quantization Details
23
+ - Method: GPTQ
24
+ - Bits: 4
25
+ - Group size: 128
26
+ - Symmetric: True
27
+
28
+ ## Usage with vLLM
29
+ ```python
30
+ from vllm import LLM
31
+
32
+ llm = LLM(model="geninhu/RakutenAI-7B-instruct-GPTQ")
33
+ ```
34
+
35
+ ## Usage with Transformers
36
+ ```python
37
+ from auto_gptq import AutoGPTQForCausalLM
38
+ from transformers import AutoTokenizer
39
+
40
+ model = AutoGPTQForCausalLM.from_quantized("geninhu/RakutenAI-7B-instruct-GPTQ")
41
+ tokenizer = AutoTokenizer.from_pretrained("geninhu/RakutenAI-7B-instruct-GPTQ")
42
+ ```
config.json ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "MistralForCausalLM"
4
+ ],
5
+ "attention_dropout": 0.0,
6
+ "bos_token_id": 1,
7
+ "eos_token_id": 2,
8
+ "head_dim": null,
9
+ "hidden_act": "silu",
10
+ "hidden_size": 4096,
11
+ "initializer_range": 0.02,
12
+ "intermediate_size": 14336,
13
+ "max_position_embeddings": 32768,
14
+ "model_type": "mistral",
15
+ "num_attention_heads": 32,
16
+ "num_hidden_layers": 32,
17
+ "num_key_value_heads": 8,
18
+ "quantization_config": {
19
+ "bits": 4,
20
+ "damp_percent": 0.1,
21
+ "desc_act": false,
22
+ "group_size": 128,
23
+ "is_marlin_format": false,
24
+ "model_file_base_name": null,
25
+ "model_name_or_path": null,
26
+ "quant_method": "gptq",
27
+ "static_groups": false,
28
+ "sym": true,
29
+ "true_sequential": true
30
+ },
31
+ "rms_norm_eps": 1e-05,
32
+ "rope_theta": 10000.0,
33
+ "sliding_window": 4096,
34
+ "tie_word_embeddings": false,
35
+ "torch_dtype": "float16",
36
+ "transformers_version": "4.52.2",
37
+ "use_cache": true,
38
+ "use_transformers_inputs": true,
39
+ "vocab_size": 48000
40
+ }
gptq_model-4bit-128g.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:486ef7d005b89c17ba7d81aa3a5fcd93088da48c84047db8f89618af816f0bad
3
+ size 4420806240
quantize_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bits": 4,
3
+ "group_size": 128,
4
+ "damp_percent": 0.1,
5
+ "desc_act": false,
6
+ "static_groups": false,
7
+ "sym": true,
8
+ "true_sequential": true,
9
+ "model_name_or_path": null,
10
+ "model_file_base_name": null,
11
+ "is_marlin_format": false,
12
+ "quant_method": "gptq"
13
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<unk>",
4
+ "<s>",
5
+ "</s>"
6
+ ],
7
+ "bos_token": {
8
+ "content": "<s>",
9
+ "lstrip": false,
10
+ "normalized": true,
11
+ "rstrip": false,
12
+ "single_word": false
13
+ },
14
+ "eos_token": {
15
+ "content": "</s>",
16
+ "lstrip": false,
17
+ "normalized": true,
18
+ "rstrip": false,
19
+ "single_word": false
20
+ },
21
+ "pad_token": {
22
+ "content": "</s>",
23
+ "lstrip": false,
24
+ "normalized": true,
25
+ "rstrip": false,
26
+ "single_word": false
27
+ },
28
+ "unk_token": {
29
+ "content": "<unk>",
30
+ "lstrip": false,
31
+ "normalized": true,
32
+ "rstrip": false,
33
+ "single_word": false
34
+ }
35
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7978987401ef447724ded0544d048831954b7517a96555ce1593149e3678b6dc
3
+ size 755169
tokenizer_config.json ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "add_prefix_space": null,
5
+ "added_tokens_decoder": {
6
+ "0": {
7
+ "content": "<unk>",
8
+ "lstrip": false,
9
+ "normalized": true,
10
+ "rstrip": false,
11
+ "single_word": false,
12
+ "special": true
13
+ },
14
+ "1": {
15
+ "content": "<s>",
16
+ "lstrip": false,
17
+ "normalized": true,
18
+ "rstrip": false,
19
+ "single_word": false,
20
+ "special": true
21
+ },
22
+ "2": {
23
+ "content": "</s>",
24
+ "lstrip": false,
25
+ "normalized": true,
26
+ "rstrip": false,
27
+ "single_word": false,
28
+ "special": true
29
+ }
30
+ },
31
+ "additional_special_tokens": [
32
+ "<unk>",
33
+ "<s>",
34
+ "</s>"
35
+ ],
36
+ "bos_token": "<s>",
37
+ "clean_up_tokenization_spaces": false,
38
+ "eos_token": "</s>",
39
+ "extra_special_tokens": {},
40
+ "legacy": true,
41
+ "model_max_length": 1000000000000000019884624838656,
42
+ "pad_token": "</s>",
43
+ "sp_model_kwargs": {},
44
+ "spaces_between_special_tokens": false,
45
+ "tokenizer_class": "LlamaTokenizer",
46
+ "unk_token": "<unk>",
47
+ "use_default_system_prompt": true
48
+ }