Upload OpenLMForCausalLM

config.json

@@ -0,0 +1,29 @@
+{
+  "apply_qk_norm": true,
+  "architectures": [
+    "OpenLMForCausalLM"
+  ],
+  "attn_func": "xformers_attn",
+  "attn_name": "xformers_attn",
+  "dim": 640,
+  "ffn_type": "swiglu",
+  "model_type": "openlm",
+  "moe_capacity_factor": 1.25,
+  "moe_expert_model_parallelism": false,
+  "moe_freq": 0,
+  "moe_loss_weight": 0.1,
+  "moe_num_experts": null,
+  "moe_top_k": 2,
+  "moe_weight_parallelism": false,
+  "n_heads": 4,
+  "n_layers": 15,
+  "norm_eps": 1e-05,
+  "norm_type": "default_layer_norm",
+  "positional_embedding_type": "rotary",
+  "post_embed_norm": false,
+  "seq_len": 2048,
+  "torch_dtype": "float32",
+  "transformers_version": "4.49.0",
+  "vocab_size": 50432,
+  "weight_tying": false
+}

generation_config.json

@@ -0,0 +1,4 @@
+{
+  "_from_model_config": true,
+  "transformers_version": "4.49.0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6f29756eda5b3d82c382499675f72cbde4432339217e3b7cbdcd2049416dc11
+size 563292704