yzhuang
/

QwQ-32B-attn

Text Generation

text-generation-inference

Model card Files Files and versions Community

yzhuang commited on 24 days ago

Commit

e3ba397

·

verified ·

1 Parent(s): 69ee246

Update config.json

Files changed (1) hide show

config.json +2 -2

config.json CHANGED Viewed

@@ -9,7 +9,7 @@
   "hidden_size": 5120,
   "initializer_range": 0.02,
   "intermediate_size": 27648,
-  "max_position_embeddings": 524288,
   "max_window_layers": 64,
   "model_type": "qwen2",
   "num_attention_heads": 40,
@@ -25,7 +25,7 @@
   "use_sliding_window": false,
   "vocab_size": 152064,
   "rope_scaling": {
-    "factor": 32.0,
     "original_max_position_embeddings": 32768,
     "type": "yarn"
   }

   "hidden_size": 5120,
   "initializer_range": 0.02,
   "intermediate_size": 27648,
+  "max_position_embeddings": 40960,
   "max_window_layers": 64,
   "model_type": "qwen2",
   "num_attention_heads": 40,
   "use_sliding_window": false,
   "vocab_size": 152064,
   "rope_scaling": {
+    "factor": 4.0,
     "original_max_position_embeddings": 32768,
     "type": "yarn"
   }