Upload GPTQ 3-bit quantized Mistral-Large-Instruct-2407

config.json

@@ -0,0 +1,52 @@
+{
+  "_name_or_path": "SillyTilly/Mistral-Large-Instruct-2407",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 12288,
+  "initializer_range": 0.02,
+  "intermediate_size": 28672,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 96,
+  "num_hidden_layers": 88,
+  "num_key_value_heads": 8,
+  "quantization_config": {
+    "amp": true,
+    "autoround_version": "0.4.4",
+    "batch_size": 4,
+    "bits": 3,
+    "damp_percent": 0.01,
+    "data_type": "int",
+    "desc_act": false,
+    "enable_minmax_tuning": true,
+    "enable_norm_bias_tuning": false,
+    "enable_quanted_input": true,
+    "gradient_accumulate_steps": 1,
+    "group_size": 128,
+    "iters": 512,
+    "low_gpu_mem_usage": true,
+    "lr": 0.001953125,
+    "minmax_lr": 0.001953125,
+    "nsamples": 256,
+    "quant_method": "gptq",
+    "scale_dtype": "torch.float16",
+    "seqlen": 2048,
+    "sym": true,
+    "to_quant_block_names": null,
+    "true_sequential": false
+  },
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.48.0",
+  "use_cache": true,
+  "vocab_size": 32768
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.48.0"
+}

model-00001-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb27bcab34c52d1e95b34129e48abb64524e8fbce0d0d4d7dc2ed34e4bc0f039
+size 4889290168

model-00002-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26f7b8339eea87481dd67dccee7c6894b4f28adaf9a7ea5d4e77b84fb6bcf6c1
+size 4906702768

model-00003-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46f9f3248fd1e041ceea60f27dc198befdd31774df6135d3ff949954c1c9ec83
+size 4906702848

model-00004-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddda332d5350c917fa690cbdcb5a0b0b4703b93ea5f31ca2e50aa3f47683f7b1
+size 4906702848

model-00005-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:169e9e637f31803fbbc0b052757be82e27ca37d6c6cbe99025312db64ebb52a6
+size 4906702848

model-00006-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bac435430f0197c7fbcc7285261a35c2b1ac83893167a8bb583e4305ace51c9b
+size 4906702848

model-00007-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a22641eccaaf3a8953ea4af100fea5727352f5076effc256efb8f10b6deacae
+size 4906702848

model-00008-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5b43000eef765f2abc01bc089f18d670b1ab7a541b0e73d4eb70ca056bb729e
+size 4906702848

model-00009-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b66e228cf53358d81316954f03c2f24087125a277d1f3d3f58a2a3fe789a648d
+size 4906702848

model-00010-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1209081eae3d77fc2d1cf3c0ea1fb23adc8743a6ab591ab1fbd25bc757568751
+size 4639068008

model-00011-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d563239f9a330a90db1e0f2097a04de76b84cc17cf7ba4643d79a4d277139fc
+size 805306496

quantize_config.json

@@ -0,0 +1,25 @@
+{
+  "bits": 3,
+  "group_size": 128,
+  "sym": true,
+  "data_type": "int",
+  "enable_quanted_input": true,
+  "enable_minmax_tuning": true,
+  "seqlen": 2048,
+  "batch_size": 4,
+  "scale_dtype": "torch.float16",
+  "lr": 0.001953125,
+  "minmax_lr": 0.001953125,
+  "gradient_accumulate_steps": 1,
+  "iters": 512,
+  "amp": true,
+  "nsamples": 256,
+  "low_gpu_mem_usage": true,
+  "to_quant_block_names": null,
+  "enable_norm_bias_tuning": false,
+  "autoround_version": "0.4.4",
+  "quant_method": "gptq",
+  "desc_act": false,
+  "true_sequential": false,
+  "damp_percent": 0.01
+}

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59f95e28944c062244741268596badc900df86c7f5ded05088d2da22a7379e06
+size 587583