Training in progress, step 379

config.json

@@ -4,6 +4,8 @@
     "LlamaForCausalLM"
   ],
   "bos_token_id": 1,
+  "dropout_attn": 0,
+  "dropout_mlp": 0.001,
   "eos_token_id": 2,
   "hidden_act": "silu",
   "hidden_size": 4096,
@@ -21,6 +23,6 @@
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.34.0.dev0",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 32002
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:605f054ed06c9c7b1940126d05ca54a7e49f89c9223a9d6867332cfda5691845
+oid sha256:488b33b36d8130ee1b9cfe5085a4dd123342c59610acba8973ea19100dc6636d
 size 7000644361

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dde1378c4571722e33f31ab6c245427370bd26cfe4aa0789bb45a87408f33605
-size 5755
+oid sha256:0a4d59665e469a2426f2e12442b74bbb7c8468ad9012c8c80b7217d296502bd7
+size 5819