Upload LlamaForCausalLM

config.json

@@ -1,38 +1,24 @@
 {
-  "_name_or_path": "syzymon/long_llama_3b_instruct",
+  "_name_or_path": "CobraMamba/mamba-gpt-3b-v3",
   "architectures": [
-    "LongLlamaForCausalLM"
+    "LlamaForCausalLM"
   ],
-  "auto_map": {
-    "AutoConfig": "syzymon/long_llama_3b_instruct--configuration_longllama.LongLlamaConfig",
-    "AutoModel": "syzymon/long_llama_3b_instruct--modeling_longllama.LongLlamaModel",
-    "AutoModelForCausalLM": "syzymon/long_llama_3b_instruct--modeling_longllama.LongLlamaForCausalLM",
-    "AutoModelForSequenceClassification": "syzymon/long_llama_3b_instruct--modeling_longllama.LongLlamaForSequenceClassification"
-  },
   "bos_token_id": 1,
   "eos_token_id": 2,
-  "gradient_checkpoint_every_ith": 1,
   "hidden_act": "silu",
   "hidden_size": 3200,
   "initializer_range": 0.02,
   "intermediate_size": 8640,
-  "last_context_length": 1024,
   "max_position_embeddings": 2048,
-  "mem_attention_grouping": null,
-  "mem_dtype": "bfloat16",
-  "mem_layers": [
-    6,
-    12,
-    18
-  ],
-  "mem_positionals": true,
-  "model_type": "longllama",
+  "model_type": "llama",
   "num_attention_heads": 32,
   "num_hidden_layers": 26,
+  "num_key_value_heads": 32,
   "pad_token_id": 0,
+  "pretraining_tp": 1,
   "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
   "tie_word_embeddings": false,
-  "torch_attention": false,
   "torch_dtype": "float32",
   "transformers_version": "4.32.0.dev0",
   "use_cache": true,

pytorch_model-00001-of-00005.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:907751722ee5d3371a6f0d424e2c9a267d7428fcc9b9b7a7e029d138cfe1b17e
-size 2969747401
+oid sha256:40581f586930983060edc1c6d84b8aa10057e17783146137a6a152bc18d2585e
+size 2969744687

pytorch_model-00002-of-00005.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5909dd80234c548c3c4e2fb1d84d0d7d2c5d39d749f9596a8e5cfebc44aa22aa
-size 2973871623
+oid sha256:5e462f6c410dedabfb4bb3671fff7ceee8af4a47da7de15cc7aee4a89f489768
+size 2973868379

pytorch_model-00003-of-00005.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dbbb1f3f7d980e94f4ee9f721713c648ae8664dbd111620633bcd9aeaf06b1b7
-size 2973871687
+oid sha256:279cf78e4e9726b9f19673d78c5fa440ca8091ebec93d540d8e538d26c7a2629
+size 2973868443

pytorch_model-00004-of-00005.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:885cc675d2fe901453a5faf3d39544e8e85bb2654046903b8d9a2b0c11cc2b06
-size 2973871687
+oid sha256:377d8d7dfaccddd52262869ea76470c308236e34214717893ebfbaeeed034c42
+size 2973868443

pytorch_model-00005-of-00005.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f548dd3cd3fda7cad7214fbe114ff7821e6e7b2414abb1029d06892ba21d875a
-size 1814628523
+oid sha256:b8615ad8d5e7dfcf9cc859c2cb2d4db322545b98515edda3a72e04ec9ede75b1
+size 1814626933

pytorch_model.bin.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 13705899600
+    "total_size": 13705894400
   },
   "weight_map": {
     "lm_head.weight": "pytorch_model-00005-of-00005.bin",
@@ -13,7 +13,6 @@
     "model.layers.0.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.0.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.0.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin",
-    "model.layers.0.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin",
     "model.layers.0.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.1.mlp.down_proj.weight": "pytorch_model-00001-of-00005.bin",
@@ -23,7 +22,6 @@
     "model.layers.1.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.1.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.1.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin",
-    "model.layers.1.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin",
     "model.layers.1.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.10.input_layernorm.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.10.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin",
@@ -33,7 +31,6 @@
     "model.layers.10.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.10.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.10.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin",
-    "model.layers.10.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin",
     "model.layers.10.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.11.input_layernorm.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.11.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin",
@@ -43,7 +40,6 @@
     "model.layers.11.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.11.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.11.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin",
-    "model.layers.11.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin",
     "model.layers.11.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.12.input_layernorm.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.12.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin",
@@ -53,7 +49,6 @@
     "model.layers.12.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.12.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.12.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin",
-    "model.layers.12.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin",
     "model.layers.12.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.13.input_layernorm.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.13.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin",
@@ -63,7 +58,6 @@
     "model.layers.13.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.13.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.13.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin",
-    "model.layers.13.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin",
     "model.layers.13.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.14.input_layernorm.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.14.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin",
@@ -73,7 +67,6 @@
     "model.layers.14.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.14.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.14.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin",
-    "model.layers.14.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin",
     "model.layers.14.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.15.input_layernorm.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.15.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin",
@@ -83,7 +76,6 @@
     "model.layers.15.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.15.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.15.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin",
-    "model.layers.15.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin",
     "model.layers.15.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.16.input_layernorm.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.16.mlp.down_proj.weight": "pytorch_model-00003-of-00005.bin",
@@ -93,7 +85,6 @@
     "model.layers.16.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.16.self_attn.o_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.16.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin",
-    "model.layers.16.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00005.bin",
     "model.layers.16.self_attn.v_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.17.input_layernorm.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.17.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin",
@@ -103,7 +94,6 @@
     "model.layers.17.self_attn.k_proj.weight": "pytorch_model-00003-of-00005.bin",
     "model.layers.17.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.17.self_attn.q_proj.weight": "pytorch_model-00003-of-00005.bin",
-    "model.layers.17.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin",
     "model.layers.17.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.18.input_layernorm.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.18.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin",
@@ -113,7 +103,6 @@
     "model.layers.18.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.18.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.18.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin",
-    "model.layers.18.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin",
     "model.layers.18.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.19.input_layernorm.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.19.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin",
@@ -123,7 +112,6 @@
     "model.layers.19.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.19.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.19.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin",
-    "model.layers.19.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin",
     "model.layers.19.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.2.input_layernorm.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.2.mlp.down_proj.weight": "pytorch_model-00001-of-00005.bin",
@@ -133,7 +121,6 @@
     "model.layers.2.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.2.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.2.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin",
-    "model.layers.2.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin",
     "model.layers.2.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.20.input_layernorm.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.20.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin",
@@ -143,7 +130,6 @@
     "model.layers.20.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.20.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.20.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin",
-    "model.layers.20.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin",
     "model.layers.20.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.21.input_layernorm.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.21.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin",
@@ -153,7 +139,6 @@
     "model.layers.21.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.21.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.21.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin",
-    "model.layers.21.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin",
     "model.layers.21.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.22.input_layernorm.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.22.mlp.down_proj.weight": "pytorch_model-00004-of-00005.bin",
@@ -163,7 +148,6 @@
     "model.layers.22.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.22.self_attn.o_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.22.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin",
-    "model.layers.22.self_attn.rotary_emb.inv_freq": "pytorch_model-00004-of-00005.bin",
     "model.layers.22.self_attn.v_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.23.input_layernorm.weight": "pytorch_model-00005-of-00005.bin",
     "model.layers.23.mlp.down_proj.weight": "pytorch_model-00005-of-00005.bin",
@@ -173,7 +157,6 @@
     "model.layers.23.self_attn.k_proj.weight": "pytorch_model-00004-of-00005.bin",
     "model.layers.23.self_attn.o_proj.weight": "pytorch_model-00005-of-00005.bin",
     "model.layers.23.self_attn.q_proj.weight": "pytorch_model-00004-of-00005.bin",
-    "model.layers.23.self_attn.rotary_emb.inv_freq": "pytorch_model-00005-of-00005.bin",
     "model.layers.23.self_attn.v_proj.weight": "pytorch_model-00005-of-00005.bin",
     "model.layers.24.input_layernorm.weight": "pytorch_model-00005-of-00005.bin",
     "model.layers.24.mlp.down_proj.weight": "pytorch_model-00005-of-00005.bin",
@@ -183,7 +166,6 @@
     "model.layers.24.self_attn.k_proj.weight": "pytorch_model-00005-of-00005.bin",
     "model.layers.24.self_attn.o_proj.weight": "pytorch_model-00005-of-00005.bin",
     "model.layers.24.self_attn.q_proj.weight": "pytorch_model-00005-of-00005.bin",
-    "model.layers.24.self_attn.rotary_emb.inv_freq": "pytorch_model-00005-of-00005.bin",
     "model.layers.24.self_attn.v_proj.weight": "pytorch_model-00005-of-00005.bin",
     "model.layers.25.input_layernorm.weight": "pytorch_model-00005-of-00005.bin",
     "model.layers.25.mlp.down_proj.weight": "pytorch_model-00005-of-00005.bin",
@@ -193,7 +175,6 @@
     "model.layers.25.self_attn.k_proj.weight": "pytorch_model-00005-of-00005.bin",
     "model.layers.25.self_attn.o_proj.weight": "pytorch_model-00005-of-00005.bin",
     "model.layers.25.self_attn.q_proj.weight": "pytorch_model-00005-of-00005.bin",
-    "model.layers.25.self_attn.rotary_emb.inv_freq": "pytorch_model-00005-of-00005.bin",
     "model.layers.25.self_attn.v_proj.weight": "pytorch_model-00005-of-00005.bin",
     "model.layers.3.input_layernorm.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.3.mlp.down_proj.weight": "pytorch_model-00001-of-00005.bin",
@@ -203,7 +184,6 @@
     "model.layers.3.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.3.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.3.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin",
-    "model.layers.3.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin",
     "model.layers.3.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.4.input_layernorm.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.4.mlp.down_proj.weight": "pytorch_model-00001-of-00005.bin",
@@ -213,7 +193,6 @@
     "model.layers.4.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.4.self_attn.o_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.4.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin",
-    "model.layers.4.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00005.bin",
     "model.layers.4.self_attn.v_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.5.input_layernorm.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.5.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin",
@@ -223,7 +202,6 @@
     "model.layers.5.self_attn.k_proj.weight": "pytorch_model-00001-of-00005.bin",
     "model.layers.5.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.5.self_attn.q_proj.weight": "pytorch_model-00001-of-00005.bin",
-    "model.layers.5.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin",
     "model.layers.5.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.6.input_layernorm.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.6.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin",
@@ -233,7 +211,6 @@
     "model.layers.6.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.6.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.6.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin",
-    "model.layers.6.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin",
     "model.layers.6.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.7.input_layernorm.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.7.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin",
@@ -243,7 +220,6 @@
     "model.layers.7.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.7.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.7.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin",
-    "model.layers.7.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin",
     "model.layers.7.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.8.input_layernorm.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.8.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin",
@@ -253,7 +229,6 @@
     "model.layers.8.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.8.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.8.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin",
-    "model.layers.8.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin",
     "model.layers.8.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.9.input_layernorm.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.9.mlp.down_proj.weight": "pytorch_model-00002-of-00005.bin",
@@ -263,7 +238,6 @@
     "model.layers.9.self_attn.k_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.9.self_attn.o_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.layers.9.self_attn.q_proj.weight": "pytorch_model-00002-of-00005.bin",
-    "model.layers.9.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00005.bin",
     "model.layers.9.self_attn.v_proj.weight": "pytorch_model-00002-of-00005.bin",
     "model.norm.weight": "pytorch_model-00005-of-00005.bin"
   }