amd
/

grok-1-FP8-KV

bowenbaoamd commited on 23 days ago

Commit

9e466ae

verified ·

1 Parent(s): 3fef029

Update config.json

Files changed (1) hide show

config.json CHANGED Viewed

@@ -18,7 +18,7 @@
   "max_position_embeddings": 8192,
   "model_type": "grok-1",
   "num_attention_heads": 48,
-  "num_local_experts": 8,
   "num_experts_per_tok": 2,
   "num_hidden_layers": 64,
   "num_key_value_heads": 8,

   "max_position_embeddings": 8192,
   "model_type": "grok-1",
   "num_attention_heads": 48,
+  "num_experts": 8,
   "num_experts_per_tok": 2,
   "num_hidden_layers": 64,
   "num_key_value_heads": 8,