kaitchup
/

Qwen2.5-7B-Instruct-gptqmodel-8bit

Text Generation

text-generation-inference

8-bit precision

Model card Files Files and versions Community

bnjmnmarie commited on Feb 4

Commit

0f50fff

·

verified ·

1 Parent(s): e709073

Update config.json

Files changed (1) hide show

config.json +3 -3

config.json CHANGED Viewed

@@ -4,7 +4,7 @@
     "Qwen2ForCausalLM"
   ],
   "quantization_config": {
-    "bits": 2,
     "dynamic": null,
     "group_size": 128,
     "desc_act": true,
@@ -20,8 +20,8 @@
       "damp_percent": 0.01,
       "damp_auto_increment": 0.0025,
       "static_groups": false,
-      "true_sequential": true,
-    },
   },
   "attention_dropout": 0.0,
   "bos_token_id": 151643,

     "Qwen2ForCausalLM"
   ],
   "quantization_config": {
+    "bits": 8,
     "dynamic": null,
     "group_size": 128,
     "desc_act": true,
       "damp_percent": 0.01,
       "damp_auto_increment": 0.0025,
       "static_groups": false,
+      "true_sequential": true
+    }
   },
   "attention_dropout": 0.0,
   "bos_token_id": 151643,