Hugging Face's logo

neuralmagic
/
Meta-Llama-3.1-8B-Instruct-quantized.w4a16

Text Generation
Transformers
Safetensors
llama
int4
vllm
conversational
text-generation-inference
Inference Endpoints
4-bit precision
gptq
Model card Files Community
1
alexmarques commited on
Commit
46cdbb6
•
1 Parent(s): c8857c0

Upload folder using huggingface_hub

Browse files
Files changed (3) hide show
  1. config.json +3 -3
  2. model.safetensors +2 -2
  3. quantize_config.json +3 -3
config.json CHANGED
@@ -24,10 +24,10 @@
24
  "pretraining_tp": 1,
25
  "quantization_config": {
26
  "bits": 4,
27
- "checkpoint_format": "gptq",
28
- "damp_percent": 0.01,
29
  "desc_act": true,
30
  "group_size": 128,
 
31
  "model_file_base_name": "model",
32
  "model_name_or_path": null,
33
  "quant_method": "gptq",
@@ -46,7 +46,7 @@
46
  "rope_theta": 500000.0,
47
  "tie_word_embeddings": false,
48
  "torch_dtype": "float16",
49
- "transformers_version": "4.43.2",
50
  "use_cache": true,
51
  "vocab_size": 128256
52
  }
 
24
  "pretraining_tp": 1,
25
  "quantization_config": {
26
  "bits": 4,
27
+ "damp_percent": 0.1,
 
28
  "desc_act": true,
29
  "group_size": 128,
30
+ "is_marlin_format": false,
31
  "model_file_base_name": "model",
32
  "model_name_or_path": null,
33
  "quant_method": "gptq",
 
46
  "rope_theta": 500000.0,
47
  "tie_word_embeddings": false,
48
  "torch_dtype": "float16",
49
+ "transformers_version": "4.44.0",
50
  "use_cache": true,
51
  "vocab_size": 128256
52
  }
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:bf009a22aa36f14072da925d73bb584cc83138f95007b123358f6b41d86d80ed
3
- size 5732943784
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5b65a3d34b7b4d350f0cbee60f4e690d1eee1411e7f484db0a8cfae081458602
3
+ size 5735720552
quantize_config.json CHANGED
@@ -1,13 +1,13 @@
1
  {
2
  "bits": 4,
3
  "group_size": 128,
4
- "damp_percent": 0.01,
5
  "desc_act": true,
6
  "static_groups": false,
7
  "sym": true,
8
  "true_sequential": true,
9
  "model_name_or_path": null,
10
  "model_file_base_name": "model",
11
- "quant_method": "gptq",
12
- "checkpoint_format": "gptq"
13
  }
 
1
  {
2
  "bits": 4,
3
  "group_size": 128,
4
+ "damp_percent": 0.1,
5
  "desc_act": true,
6
  "static_groups": false,
7
  "sym": true,
8
  "true_sequential": true,
9
  "model_name_or_path": null,
10
  "model_file_base_name": "model",
11
+ "is_marlin_format": false,
12
+ "quant_method": "gptq"
13
  }