Training in progress, step 2000, checkpoint

checkpoint-2000/config.json

@@ -0,0 +1,39 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3",
+  "use_cache": true,
+  "vocab_size": 128256
+}

checkpoint-2000/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.51.3"
+}

checkpoint-2000/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4219ec42d9f2dc3c5ed882ceb27784e76171f6933f6a29441fb29610a9b82bf
+size 4976698672

checkpoint-2000/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:084cfeaa6c50d1322c80b8df7b30788b0470ccbc0931e48012605d1e97be544a
+size 4999802720

checkpoint-2000/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87708b777e69399bab19fc1e4adce24e17f952764259bbae54b9c8a08935f3d2
+size 4915916176

checkpoint-2000/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b66d58c1917b6d5a8c7324fe81fb6c68921e758de0c564e9845126e4225b7ef
+size 1168138808

checkpoint-2000/model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 16060522496
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

checkpoint-2000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:682eb35020909a89985925afe4b44ec749799899b39c44eee7774ee00e454331
+size 12838522

checkpoint-2000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77bc39c839bb653a331afc274139aaccb66be82255be034bc865539fb28c85fb
+size 15006

checkpoint-2000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c00700e11d1fe674306114ce38a91c2f04990f7b83f58ed06deef722afcdb87
+size 1064

checkpoint-2000/trainer_state.json

@@ -0,0 +1,1434 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.6666666666666666,
+  "eval_steps": 500,
+  "global_step": 2000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0033333333333333335,
+      "grad_norm": 2992.0,
+      "learning_rate": 6.000000000000001e-07,
+      "loss": 3.3062,
+      "step": 10
+    },
+    {
+      "epoch": 0.006666666666666667,
+      "grad_norm": 924.0,
+      "learning_rate": 1.2666666666666669e-06,
+      "loss": 2.7836,
+      "step": 20
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 5216.0,
+      "learning_rate": 1.9333333333333336e-06,
+      "loss": 2.3297,
+      "step": 30
+    },
+    {
+      "epoch": 0.013333333333333334,
+      "grad_norm": 71.5,
+      "learning_rate": 2.6e-06,
+      "loss": 1.7359,
+      "step": 40
+    },
+    {
+      "epoch": 0.016666666666666666,
+      "grad_norm": 115.0,
+      "learning_rate": 3.266666666666667e-06,
+      "loss": 1.4844,
+      "step": 50
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 52.75,
+      "learning_rate": 3.9333333333333335e-06,
+      "loss": 1.218,
+      "step": 60
+    },
+    {
+      "epoch": 0.023333333333333334,
+      "grad_norm": 52.25,
+      "learning_rate": 4.600000000000001e-06,
+      "loss": 1.1391,
+      "step": 70
+    },
+    {
+      "epoch": 0.02666666666666667,
+      "grad_norm": 70.5,
+      "learning_rate": 5.2666666666666665e-06,
+      "loss": 1.0164,
+      "step": 80
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 23.25,
+      "learning_rate": 5.933333333333335e-06,
+      "loss": 0.8957,
+      "step": 90
+    },
+    {
+      "epoch": 0.03333333333333333,
+      "grad_norm": 9.8125,
+      "learning_rate": 6.600000000000001e-06,
+      "loss": 0.8219,
+      "step": 100
+    },
+    {
+      "epoch": 0.03666666666666667,
+      "grad_norm": 46.5,
+      "learning_rate": 7.266666666666668e-06,
+      "loss": 0.7836,
+      "step": 110
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 11.25,
+      "learning_rate": 7.933333333333334e-06,
+      "loss": 0.7531,
+      "step": 120
+    },
+    {
+      "epoch": 0.043333333333333335,
+      "grad_norm": 6.8125,
+      "learning_rate": 8.6e-06,
+      "loss": 0.7695,
+      "step": 130
+    },
+    {
+      "epoch": 0.04666666666666667,
+      "grad_norm": 12.0625,
+      "learning_rate": 9.266666666666667e-06,
+      "loss": 0.7535,
+      "step": 140
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 11.8125,
+      "learning_rate": 9.933333333333334e-06,
+      "loss": 0.7273,
+      "step": 150
+    },
+    {
+      "epoch": 0.05333333333333334,
+      "grad_norm": 5.25,
+      "learning_rate": 1.0600000000000002e-05,
+      "loss": 0.7281,
+      "step": 160
+    },
+    {
+      "epoch": 0.056666666666666664,
+      "grad_norm": 5.625,
+      "learning_rate": 1.1266666666666668e-05,
+      "loss": 0.6813,
+      "step": 170
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 9.875,
+      "learning_rate": 1.1933333333333335e-05,
+      "loss": 0.7129,
+      "step": 180
+    },
+    {
+      "epoch": 0.06333333333333334,
+      "grad_norm": 34.75,
+      "learning_rate": 1.2600000000000001e-05,
+      "loss": 0.7184,
+      "step": 190
+    },
+    {
+      "epoch": 0.06666666666666667,
+      "grad_norm": 11.125,
+      "learning_rate": 1.3266666666666668e-05,
+      "loss": 0.7078,
+      "step": 200
+    },
+    {
+      "epoch": 0.07,
+      "grad_norm": 5.625,
+      "learning_rate": 1.3933333333333334e-05,
+      "loss": 0.6992,
+      "step": 210
+    },
+    {
+      "epoch": 0.07333333333333333,
+      "grad_norm": 4.1875,
+      "learning_rate": 1.46e-05,
+      "loss": 0.723,
+      "step": 220
+    },
+    {
+      "epoch": 0.07666666666666666,
+      "grad_norm": 5.6875,
+      "learning_rate": 1.5266666666666667e-05,
+      "loss": 0.6816,
+      "step": 230
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 3.921875,
+      "learning_rate": 1.5933333333333336e-05,
+      "loss": 0.6926,
+      "step": 240
+    },
+    {
+      "epoch": 0.08333333333333333,
+      "grad_norm": 3.796875,
+      "learning_rate": 1.66e-05,
+      "loss": 0.6766,
+      "step": 250
+    },
+    {
+      "epoch": 0.08666666666666667,
+      "grad_norm": 3.734375,
+      "learning_rate": 1.726666666666667e-05,
+      "loss": 0.7387,
+      "step": 260
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 3.515625,
+      "learning_rate": 1.7933333333333333e-05,
+      "loss": 0.7113,
+      "step": 270
+    },
+    {
+      "epoch": 0.09333333333333334,
+      "grad_norm": 3.625,
+      "learning_rate": 1.86e-05,
+      "loss": 0.7109,
+      "step": 280
+    },
+    {
+      "epoch": 0.09666666666666666,
+      "grad_norm": 4.78125,
+      "learning_rate": 1.926666666666667e-05,
+      "loss": 0.8172,
+      "step": 290
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 3.734375,
+      "learning_rate": 1.9933333333333334e-05,
+      "loss": 0.6676,
+      "step": 300
+    },
+    {
+      "epoch": 0.10333333333333333,
+      "grad_norm": 3.28125,
+      "learning_rate": 1.9999451693655125e-05,
+      "loss": 0.7723,
+      "step": 310
+    },
+    {
+      "epoch": 0.10666666666666667,
+      "grad_norm": 5.71875,
+      "learning_rate": 1.9997556390892623e-05,
+      "loss": 0.8172,
+      "step": 320
+    },
+    {
+      "epoch": 0.11,
+      "grad_norm": 4.84375,
+      "learning_rate": 1.9994307579030925e-05,
+      "loss": 0.8008,
+      "step": 330
+    },
+    {
+      "epoch": 0.11333333333333333,
+      "grad_norm": 2.8125,
+      "learning_rate": 1.998970569790715e-05,
+      "loss": 0.8305,
+      "step": 340
+    },
+    {
+      "epoch": 0.11666666666666667,
+      "grad_norm": 5.34375,
+      "learning_rate": 1.9983751370542334e-05,
+      "loss": 0.7742,
+      "step": 350
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 2.984375,
+      "learning_rate": 1.9976445403057095e-05,
+      "loss": 0.768,
+      "step": 360
+    },
+    {
+      "epoch": 0.12333333333333334,
+      "grad_norm": 3.34375,
+      "learning_rate": 1.9967788784562474e-05,
+      "loss": 0.7555,
+      "step": 370
+    },
+    {
+      "epoch": 0.12666666666666668,
+      "grad_norm": 3.265625,
+      "learning_rate": 1.9957782687026046e-05,
+      "loss": 0.7535,
+      "step": 380
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 4.6875,
+      "learning_rate": 1.9946428465113244e-05,
+      "loss": 0.7488,
+      "step": 390
+    },
+    {
+      "epoch": 0.13333333333333333,
+      "grad_norm": 3.0625,
+      "learning_rate": 1.9933727656003964e-05,
+      "loss": 0.7707,
+      "step": 400
+    },
+    {
+      "epoch": 0.13666666666666666,
+      "grad_norm": 6.53125,
+      "learning_rate": 1.9919681979184452e-05,
+      "loss": 0.782,
+      "step": 410
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 3.796875,
+      "learning_rate": 1.9904293336214508e-05,
+      "loss": 0.7641,
+      "step": 420
+    },
+    {
+      "epoch": 0.14333333333333334,
+      "grad_norm": 3.46875,
+      "learning_rate": 1.988756381047006e-05,
+      "loss": 0.7184,
+      "step": 430
+    },
+    {
+      "epoch": 0.14666666666666667,
+      "grad_norm": 3.34375,
+      "learning_rate": 1.9869495666861094e-05,
+      "loss": 0.743,
+      "step": 440
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 3.75,
+      "learning_rate": 1.985009135152503e-05,
+      "loss": 0.7531,
+      "step": 450
+    },
+    {
+      "epoch": 0.15333333333333332,
+      "grad_norm": 3.15625,
+      "learning_rate": 1.9829353491495545e-05,
+      "loss": 0.7668,
+      "step": 460
+    },
+    {
+      "epoch": 0.15666666666666668,
+      "grad_norm": 3.203125,
+      "learning_rate": 1.980728489434693e-05,
+      "loss": 0.7512,
+      "step": 470
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 2.8125,
+      "learning_rate": 1.978388854781397e-05,
+      "loss": 0.7367,
+      "step": 480
+    },
+    {
+      "epoch": 0.16333333333333333,
+      "grad_norm": 3.203125,
+      "learning_rate": 1.9759167619387474e-05,
+      "loss": 0.7555,
+      "step": 490
+    },
+    {
+      "epoch": 0.16666666666666666,
+      "grad_norm": 4.03125,
+      "learning_rate": 1.973312545588543e-05,
+      "loss": 0.7664,
+      "step": 500
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 4.03125,
+      "learning_rate": 1.97057655829999e-05,
+      "loss": 0.7461,
+      "step": 510
+    },
+    {
+      "epoch": 0.17333333333333334,
+      "grad_norm": 4.53125,
+      "learning_rate": 1.9677091704819714e-05,
+      "loss": 0.7902,
+      "step": 520
+    },
+    {
+      "epoch": 0.17666666666666667,
+      "grad_norm": 3.34375,
+      "learning_rate": 1.964710770332898e-05,
+      "loss": 0.7469,
+      "step": 530
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 3.609375,
+      "learning_rate": 1.961581763788152e-05,
+      "loss": 0.7453,
+      "step": 540
+    },
+    {
+      "epoch": 0.18333333333333332,
+      "grad_norm": 3.671875,
+      "learning_rate": 1.9583225744651334e-05,
+      "loss": 0.6637,
+      "step": 550
+    },
+    {
+      "epoch": 0.18666666666666668,
+      "grad_norm": 2.703125,
+      "learning_rate": 1.954933643605904e-05,
+      "loss": 3.0793,
+      "step": 560
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 2.59375,
+      "learning_rate": 1.9514154300174542e-05,
+      "loss": 0.7562,
+      "step": 570
+    },
+    {
+      "epoch": 0.19333333333333333,
+      "grad_norm": 2.828125,
+      "learning_rate": 1.947768410009586e-05,
+      "loss": 0.7289,
+      "step": 580
+    },
+    {
+      "epoch": 0.19666666666666666,
+      "grad_norm": 3.65625,
+      "learning_rate": 1.9439930773304284e-05,
+      "loss": 0.7027,
+      "step": 590
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 3.46875,
+      "learning_rate": 1.9400899430995923e-05,
+      "loss": 0.7277,
+      "step": 600
+    },
+    {
+      "epoch": 0.20333333333333334,
+      "grad_norm": 2.328125,
+      "learning_rate": 1.9360595357389735e-05,
+      "loss": 0.7387,
+      "step": 610
+    },
+    {
+      "epoch": 0.20666666666666667,
+      "grad_norm": 3.265625,
+      "learning_rate": 1.9319024009012114e-05,
+      "loss": 0.777,
+      "step": 620
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 2.84375,
+      "learning_rate": 1.927619101395818e-05,
+      "loss": 0.7016,
+      "step": 630
+    },
+    {
+      "epoch": 0.21333333333333335,
+      "grad_norm": 2.765625,
+      "learning_rate": 1.923210217112981e-05,
+      "loss": 0.6867,
+      "step": 640
+    },
+    {
+      "epoch": 0.21666666666666667,
+      "grad_norm": 3.6875,
+      "learning_rate": 1.9186763449450572e-05,
+      "loss": 0.7148,
+      "step": 650
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 5.46875,
+      "learning_rate": 1.914018098705762e-05,
+      "loss": 0.7258,
+      "step": 660
+    },
+    {
+      "epoch": 0.22333333333333333,
+      "grad_norm": 2.484375,
+      "learning_rate": 1.9092361090470688e-05,
+      "loss": 0.6934,
+      "step": 670
+    },
+    {
+      "epoch": 0.22666666666666666,
+      "grad_norm": 3.78125,
+      "learning_rate": 1.90433102337383e-05,
+      "loss": 0.7078,
+      "step": 680
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 3.359375,
+      "learning_rate": 1.8993035057561274e-05,
+      "loss": 0.7227,
+      "step": 690
+    },
+    {
+      "epoch": 0.23333333333333334,
+      "grad_norm": 3.203125,
+      "learning_rate": 1.8941542368393683e-05,
+      "loss": 0.7277,
+      "step": 700
+    },
+    {
+      "epoch": 0.23666666666666666,
+      "grad_norm": 3.25,
+      "learning_rate": 1.888883913752137e-05,
+      "loss": 0.7012,
+      "step": 710
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 4.21875,
+      "learning_rate": 1.8834932500118148e-05,
+      "loss": 0.6895,
+      "step": 720
+    },
+    {
+      "epoch": 0.24333333333333335,
+      "grad_norm": 2.59375,
+      "learning_rate": 1.8779829754279806e-05,
+      "loss": 0.6445,
+      "step": 730
+    },
+    {
+      "epoch": 0.24666666666666667,
+      "grad_norm": 12.125,
+      "learning_rate": 1.872353836003608e-05,
+      "loss": 0.709,
+      "step": 740
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 6.09375,
+      "learning_rate": 1.866606593834065e-05,
+      "loss": 0.6625,
+      "step": 750
+    },
+    {
+      "epoch": 0.25333333333333335,
+      "grad_norm": 2.734375,
+      "learning_rate": 1.860742027003944e-05,
+      "loss": 0.6477,
+      "step": 760
+    },
+    {
+      "epoch": 0.25666666666666665,
+      "grad_norm": 6.125,
+      "learning_rate": 1.854760929481715e-05,
+      "loss": 0.659,
+      "step": 770
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 2.640625,
+      "learning_rate": 1.848664111012241e-05,
+      "loss": 0.6312,
+      "step": 780
+    },
+    {
+      "epoch": 0.2633333333333333,
+      "grad_norm": 2.71875,
+      "learning_rate": 1.842452397007148e-05,
+      "loss": 0.5988,
+      "step": 790
+    },
+    {
+      "epoch": 0.26666666666666666,
+      "grad_norm": 3.328125,
+      "learning_rate": 1.836126628433077e-05,
+      "loss": 0.6215,
+      "step": 800
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 2.5,
+      "learning_rate": 1.8296876616978337e-05,
+      "loss": 0.6246,
+      "step": 810
+    },
+    {
+      "epoch": 0.2733333333333333,
+      "grad_norm": 2.984375,
+      "learning_rate": 1.8231363685344422e-05,
+      "loss": 0.6469,
+      "step": 820
+    },
+    {
+      "epoch": 0.27666666666666667,
+      "grad_norm": 2.765625,
+      "learning_rate": 1.8164736358831265e-05,
+      "loss": 0.5516,
+      "step": 830
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 4.3125,
+      "learning_rate": 1.8097003657712343e-05,
+      "loss": 0.5729,
+      "step": 840
+    },
+    {
+      "epoch": 0.2833333333333333,
+      "grad_norm": 2.96875,
+      "learning_rate": 1.8028174751911147e-05,
+      "loss": 0.5051,
+      "step": 850
+    },
+    {
+      "epoch": 0.2866666666666667,
+      "grad_norm": 3.53125,
+      "learning_rate": 1.7958258959759747e-05,
+      "loss": 0.4977,
+      "step": 860
+    },
+    {
+      "epoch": 0.29,
+      "grad_norm": 4.03125,
+      "learning_rate": 1.7887265746737224e-05,
+      "loss": 0.4945,
+      "step": 870
+    },
+    {
+      "epoch": 0.29333333333333333,
+      "grad_norm": 3.65625,
+      "learning_rate": 1.781520472418819e-05,
+      "loss": 0.5334,
+      "step": 880
+    },
+    {
+      "epoch": 0.2966666666666667,
+      "grad_norm": 2.515625,
+      "learning_rate": 1.774208564802158e-05,
+      "loss": 0.4965,
+      "step": 890
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 3.015625,
+      "learning_rate": 1.7667918417389857e-05,
+      "loss": 0.5502,
+      "step": 900
+    },
+    {
+      "epoch": 0.30333333333333334,
+      "grad_norm": 2.515625,
+      "learning_rate": 1.759271307334881e-05,
+      "loss": 0.5457,
+      "step": 910
+    },
+    {
+      "epoch": 0.30666666666666664,
+      "grad_norm": 2.5625,
+      "learning_rate": 1.7516479797498172e-05,
+      "loss": 0.4469,
+      "step": 920
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 2.53125,
+      "learning_rate": 1.7439228910603184e-05,
+      "loss": 0.5006,
+      "step": 930
+    },
+    {
+      "epoch": 0.31333333333333335,
+      "grad_norm": 2.453125,
+      "learning_rate": 1.7360970871197347e-05,
+      "loss": 0.4676,
+      "step": 940
+    },
+    {
+      "epoch": 0.31666666666666665,
+      "grad_norm": 2.71875,
+      "learning_rate": 1.7281716274166464e-05,
+      "loss": 0.4725,
+      "step": 950
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 13.3125,
+      "learning_rate": 1.720147584931431e-05,
+      "loss": 0.4918,
+      "step": 960
+    },
+    {
+      "epoch": 0.3233333333333333,
+      "grad_norm": 2.703125,
+      "learning_rate": 1.712026045990997e-05,
+      "loss": 0.4328,
+      "step": 970
+    },
+    {
+      "epoch": 0.32666666666666666,
+      "grad_norm": 2.546875,
+      "learning_rate": 1.7038081101217093e-05,
+      "loss": 0.5229,
+      "step": 980
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 2.375,
+      "learning_rate": 1.6954948899005365e-05,
+      "loss": 0.435,
+      "step": 990
+    },
+    {
+      "epoch": 0.3333333333333333,
+      "grad_norm": 3.953125,
+      "learning_rate": 1.6870875108044233e-05,
+      "loss": 0.4268,
+      "step": 1000
+    },
+    {
+      "epoch": 0.33666666666666667,
+      "grad_norm": 2.484375,
+      "learning_rate": 1.6785871110579167e-05,
+      "loss": 0.4736,
+      "step": 1010
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 8.25,
+      "learning_rate": 1.6699948414790734e-05,
+      "loss": 0.4209,
+      "step": 1020
+    },
+    {
+      "epoch": 0.3433333333333333,
+      "grad_norm": 3.15625,
+      "learning_rate": 1.661311865323652e-05,
+      "loss": 0.5115,
+      "step": 1030
+    },
+    {
+      "epoch": 0.3466666666666667,
+      "grad_norm": 2.453125,
+      "learning_rate": 1.652539358127632e-05,
+      "loss": 0.477,
+      "step": 1040
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 2.28125,
+      "learning_rate": 1.643678507548061e-05,
+      "loss": 0.4611,
+      "step": 1050
+    },
+    {
+      "epoch": 0.35333333333333333,
+      "grad_norm": 2.890625,
+      "learning_rate": 1.6347305132022677e-05,
+      "loss": 0.4627,
+      "step": 1060
+    },
+    {
+      "epoch": 0.3566666666666667,
+      "grad_norm": 3.484375,
+      "learning_rate": 1.625696586505451e-05,
+      "loss": 0.4479,
+      "step": 1070
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 3.28125,
+      "learning_rate": 1.616577950506675e-05,
+      "loss": 0.4662,
+      "step": 1080
+    },
+    {
+      "epoch": 0.36333333333333334,
+      "grad_norm": 2.375,
+      "learning_rate": 1.607375839723287e-05,
+      "loss": 0.3928,
+      "step": 1090
+    },
+    {
+      "epoch": 0.36666666666666664,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.598091499973784e-05,
+      "loss": 0.4174,
+      "step": 1100
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 2.96875,
+      "learning_rate": 1.5887261882091488e-05,
+      "loss": 0.4432,
+      "step": 1110
+    },
+    {
+      "epoch": 0.37333333333333335,
+      "grad_norm": 2.625,
+      "learning_rate": 1.5792811723426787e-05,
+      "loss": 0.4582,
+      "step": 1120
+    },
+    {
+      "epoch": 0.37666666666666665,
+      "grad_norm": 2.875,
+      "learning_rate": 1.5697577310783318e-05,
+      "loss": 0.4098,
+      "step": 1130
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 3.640625,
+      "learning_rate": 1.560157153737607e-05,
+      "loss": 0.4264,
+      "step": 1140
+    },
+    {
+      "epoch": 0.38333333333333336,
+      "grad_norm": 2.046875,
+      "learning_rate": 1.5504807400849957e-05,
+      "loss": 0.3967,
+      "step": 1150
+    },
+    {
+      "epoch": 0.38666666666666666,
+      "grad_norm": 2.390625,
+      "learning_rate": 1.5407298001520108e-05,
+      "loss": 0.4064,
+      "step": 1160
+    },
+    {
+      "epoch": 0.39,
+      "grad_norm": 2.734375,
+      "learning_rate": 1.530905654059831e-05,
+      "loss": 0.4137,
+      "step": 1170
+    },
+    {
+      "epoch": 0.3933333333333333,
+      "grad_norm": 3.203125,
+      "learning_rate": 1.5210096318405768e-05,
+      "loss": 0.5766,
+      "step": 1180
+    },
+    {
+      "epoch": 0.39666666666666667,
+      "grad_norm": 2.359375,
+      "learning_rate": 1.5110430732572454e-05,
+      "loss": 0.4457,
+      "step": 1190
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 2.375,
+      "learning_rate": 1.5010073276223295e-05,
+      "loss": 0.4275,
+      "step": 1200
+    },
+    {
+      "epoch": 0.4033333333333333,
+      "grad_norm": 3.25,
+      "learning_rate": 1.490903753615141e-05,
+      "loss": 0.4061,
+      "step": 1210
+    },
+    {
+      "epoch": 0.4066666666666667,
+      "grad_norm": 3.015625,
+      "learning_rate": 1.4807337190978666e-05,
+      "loss": 0.39,
+      "step": 1220
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 2.625,
+      "learning_rate": 1.4704986009303833e-05,
+      "loss": 0.4738,
+      "step": 1230
+    },
+    {
+      "epoch": 0.41333333333333333,
+      "grad_norm": 3.15625,
+      "learning_rate": 1.4601997847838518e-05,
+      "loss": 0.4451,
+      "step": 1240
+    },
+    {
+      "epoch": 0.4166666666666667,
+      "grad_norm": 2.421875,
+      "learning_rate": 1.4498386649531198e-05,
+      "loss": 0.4242,
+      "step": 1250
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 2.4375,
+      "learning_rate": 1.439416644167957e-05,
+      "loss": 0.4105,
+      "step": 1260
+    },
+    {
+      "epoch": 0.42333333333333334,
+      "grad_norm": 3.03125,
+      "learning_rate": 1.4289351334031461e-05,
+      "loss": 0.4256,
+      "step": 1270
+    },
+    {
+      "epoch": 0.4266666666666667,
+      "grad_norm": 2.390625,
+      "learning_rate": 1.4183955516874624e-05,
+      "loss": 0.4217,
+      "step": 1280
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 2.734375,
+      "learning_rate": 1.4077993259115568e-05,
+      "loss": 0.4049,
+      "step": 1290
+    },
+    {
+      "epoch": 0.43333333333333335,
+      "grad_norm": 3.671875,
+      "learning_rate": 1.3971478906347806e-05,
+      "loss": 0.4236,
+      "step": 1300
+    },
+    {
+      "epoch": 0.43666666666666665,
+      "grad_norm": 2.359375,
+      "learning_rate": 1.3864426878909674e-05,
+      "loss": 0.4234,
+      "step": 1310
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.375685166993204e-05,
+      "loss": 0.4369,
+      "step": 1320
+    },
+    {
+      "epoch": 0.44333333333333336,
+      "grad_norm": 2.421875,
+      "learning_rate": 1.3648767843376196e-05,
+      "loss": 0.4316,
+      "step": 1330
+    },
+    {
+      "epoch": 0.44666666666666666,
+      "grad_norm": 2.3125,
+      "learning_rate": 1.3540190032062102e-05,
+      "loss": 0.3934,
+      "step": 1340
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 1.9296875,
+      "learning_rate": 1.343113293568734e-05,
+      "loss": 0.3596,
+      "step": 1350
+    },
+    {
+      "epoch": 0.4533333333333333,
+      "grad_norm": 2.3125,
+      "learning_rate": 1.3321611318837033e-05,
+      "loss": 0.4154,
+      "step": 1360
+    },
+    {
+      "epoch": 0.45666666666666667,
+      "grad_norm": 1.9453125,
+      "learning_rate": 1.3211640008984934e-05,
+      "loss": 0.3816,
+      "step": 1370
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 2.453125,
+      "learning_rate": 1.3101233894486018e-05,
+      "loss": 0.3715,
+      "step": 1380
+    },
+    {
+      "epoch": 0.4633333333333333,
+      "grad_norm": 2.28125,
+      "learning_rate": 1.2990407922560869e-05,
+      "loss": 0.359,
+      "step": 1390
+    },
+    {
+      "epoch": 0.4666666666666667,
+      "grad_norm": 2.65625,
+      "learning_rate": 1.2879177097272033e-05,
+      "loss": 0.3576,
+      "step": 1400
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 2.296875,
+      "learning_rate": 1.2767556477492722e-05,
+      "loss": 0.3393,
+      "step": 1410
+    },
+    {
+      "epoch": 0.47333333333333333,
+      "grad_norm": 2.78125,
+      "learning_rate": 1.265556117486809e-05,
+      "loss": 0.3244,
+      "step": 1420
+    },
+    {
+      "epoch": 0.4766666666666667,
+      "grad_norm": 2.28125,
+      "learning_rate": 1.2543206351769341e-05,
+      "loss": 0.3518,
+      "step": 1430
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 2.21875,
+      "learning_rate": 1.2430507219240997e-05,
+      "loss": 0.4025,
+      "step": 1440
+    },
+    {
+      "epoch": 0.48333333333333334,
+      "grad_norm": 2.3125,
+      "learning_rate": 1.2317479034941572e-05,
+      "loss": 0.3142,
+      "step": 1450
+    },
+    {
+      "epoch": 0.4866666666666667,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.2204137101077924e-05,
+      "loss": 0.3418,
+      "step": 1460
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.2090496762333565e-05,
+      "loss": 0.2744,
+      "step": 1470
+    },
+    {
+      "epoch": 0.49333333333333335,
+      "grad_norm": 2.375,
+      "learning_rate": 1.1976573403791263e-05,
+      "loss": 0.2838,
+      "step": 1480
+    },
+    {
+      "epoch": 0.49666666666666665,
+      "grad_norm": 2.765625,
+      "learning_rate": 1.1862382448850136e-05,
+      "loss": 0.3221,
+      "step": 1490
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 3.328125,
+      "learning_rate": 1.1747939357137568e-05,
+      "loss": 0.2822,
+      "step": 1500
+    },
+    {
+      "epoch": 0.5033333333333333,
+      "grad_norm": 2.515625,
+      "learning_rate": 1.1633259622416224e-05,
+      "loss": 0.2749,
+      "step": 1510
+    },
+    {
+      "epoch": 0.5066666666666667,
+      "grad_norm": 2.4375,
+      "learning_rate": 1.151835877048645e-05,
+      "loss": 0.2843,
+      "step": 1520
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 2.046875,
+      "learning_rate": 1.1403252357084315e-05,
+      "loss": 0.2721,
+      "step": 1530
+    },
+    {
+      "epoch": 0.5133333333333333,
+      "grad_norm": 2.0625,
+      "learning_rate": 1.128795596577563e-05,
+      "loss": 0.2392,
+      "step": 1540
+    },
+    {
+      "epoch": 0.5166666666666667,
+      "grad_norm": 4.3125,
+      "learning_rate": 1.1172485205846161e-05,
+      "loss": 0.3388,
+      "step": 1550
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.1056855710188413e-05,
+      "loss": 0.2516,
+      "step": 1560
+    },
+    {
+      "epoch": 0.5233333333333333,
+      "grad_norm": 2.5625,
+      "learning_rate": 1.0941083133185146e-05,
+      "loss": 0.2571,
+      "step": 1570
+    },
+    {
+      "epoch": 0.5266666666666666,
+      "grad_norm": 1.890625,
+      "learning_rate": 1.0825183148590055e-05,
+      "loss": 0.8644,
+      "step": 1580
+    },
+    {
+      "epoch": 0.53,
+      "grad_norm": 2.328125,
+      "learning_rate": 1.0709171447405786e-05,
+      "loss": 0.2578,
+      "step": 1590
+    },
+    {
+      "epoch": 0.5333333333333333,
+      "grad_norm": 2.3125,
+      "learning_rate": 1.0593063735759619e-05,
+      "loss": 0.2648,
+      "step": 1600
+    },
+    {
+      "epoch": 0.5366666666666666,
+      "grad_norm": 2.4375,
+      "learning_rate": 1.047687573277711e-05,
+      "loss": 0.3711,
+      "step": 1610
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 1.8203125,
+      "learning_rate": 1.0360623168453982e-05,
+      "loss": 0.2442,
+      "step": 1620
+    },
+    {
+      "epoch": 0.5433333333333333,
+      "grad_norm": 1.96875,
+      "learning_rate": 1.0244321781526533e-05,
+      "loss": 0.4466,
+      "step": 1630
+    },
+    {
+      "epoch": 0.5466666666666666,
+      "grad_norm": 2.21875,
+      "learning_rate": 1.0127987317340851e-05,
+      "loss": 0.2465,
+      "step": 1640
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 2.46875,
+      "learning_rate": 1.001163552572116e-05,
+      "loss": 0.2738,
+      "step": 1650
+    },
+    {
+      "epoch": 0.5533333333333333,
+      "grad_norm": 2.015625,
+      "learning_rate": 9.895282158837545e-06,
+      "loss": 0.2808,
+      "step": 1660
+    },
+    {
+      "epoch": 0.5566666666666666,
+      "grad_norm": 2.75,
+      "learning_rate": 9.778942969073345e-06,
+      "loss": 0.2623,
+      "step": 1670
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 2.796875,
+      "learning_rate": 9.66263370689256e-06,
+      "loss": 0.2759,
+      "step": 1680
+    },
+    {
+      "epoch": 0.5633333333333334,
+      "grad_norm": 2.390625,
+      "learning_rate": 9.546370118707463e-06,
+      "loss": 0.2125,
+      "step": 1690
+    },
+    {
+      "epoch": 0.5666666666666667,
+      "grad_norm": 2.140625,
+      "learning_rate": 9.430167944746802e-06,
+      "loss": 0.2364,
+      "step": 1700
+    },
+    {
+      "epoch": 0.57,
+      "grad_norm": 2.40625,
+      "learning_rate": 9.314042916924816e-06,
+      "loss": 0.2147,
+      "step": 1710
+    },
+    {
+      "epoch": 0.5733333333333334,
+      "grad_norm": 10.625,
+      "learning_rate": 9.198010756711413e-06,
+      "loss": 0.614,
+      "step": 1720
+    },
+    {
+      "epoch": 0.5766666666666667,
+      "grad_norm": 3.390625,
+      "learning_rate": 9.082087173003686e-06,
+      "loss": 0.2452,
+      "step": 1730
+    },
+    {
+      "epoch": 0.58,
+      "grad_norm": 2.09375,
+      "learning_rate": 8.966287859999216e-06,
+      "loss": 0.3431,
+      "step": 1740
+    },
+    {
+      "epoch": 0.5833333333333334,
+      "grad_norm": 2.0625,
+      "learning_rate": 8.850628495071336e-06,
+      "loss": 0.2399,
+      "step": 1750
+    },
+    {
+      "epoch": 0.5866666666666667,
+      "grad_norm": 1.6484375,
+      "learning_rate": 8.735124736646627e-06,
+      "loss": 0.236,
+      "step": 1760
+    },
+    {
+      "epoch": 0.59,
+      "grad_norm": 18.0,
+      "learning_rate": 8.619792222085059e-06,
+      "loss": 0.4111,
+      "step": 1770
+    },
+    {
+      "epoch": 0.5933333333333334,
+      "grad_norm": 2.34375,
+      "learning_rate": 8.504646565562907e-06,
+      "loss": 0.3645,
+      "step": 1780
+    },
+    {
+      "epoch": 0.5966666666666667,
+      "grad_norm": 2.390625,
+      "learning_rate": 8.389703355958873e-06,
+      "loss": 0.5304,
+      "step": 1790
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 3.25,
+      "learning_rate": 8.274978154743574e-06,
+      "loss": 0.4774,
+      "step": 1800
+    },
+    {
+      "epoch": 0.6033333333333334,
+      "grad_norm": 1.765625,
+      "learning_rate": 8.1604864938728e-06,
+      "loss": 0.2347,
+      "step": 1810
+    },
+    {
+      "epoch": 0.6066666666666667,
+      "grad_norm": 1.609375,
+      "learning_rate": 8.046243873684694e-06,
+      "loss": 0.3877,
+      "step": 1820
+    },
+    {
+      "epoch": 0.61,
+      "grad_norm": 2.015625,
+      "learning_rate": 7.932265760801295e-06,
+      "loss": 0.2186,
+      "step": 1830
+    },
+    {
+      "epoch": 0.6133333333333333,
+      "grad_norm": 1.90625,
+      "learning_rate": 7.818567586034578e-06,
+      "loss": 0.2157,
+      "step": 1840
+    },
+    {
+      "epoch": 0.6166666666666667,
+      "grad_norm": 3.21875,
+      "learning_rate": 7.705164742297376e-06,
+      "loss": 0.4418,
+      "step": 1850
+    },
+    {
+      "epoch": 0.62,
+      "grad_norm": 2.1875,
+      "learning_rate": 7.592072582519437e-06,
+      "loss": 0.232,
+      "step": 1860
+    },
+    {
+      "epoch": 0.6233333333333333,
+      "grad_norm": 1.9296875,
+      "learning_rate": 7.4793064175688635e-06,
+      "loss": 0.2247,
+      "step": 1870
+    },
+    {
+      "epoch": 0.6266666666666667,
+      "grad_norm": 8.875,
+      "learning_rate": 7.366881514179292e-06,
+      "loss": 0.4345,
+      "step": 1880
+    },
+    {
+      "epoch": 0.63,
+      "grad_norm": 3.015625,
+      "learning_rate": 7.254813092882989e-06,
+      "loss": 0.3485,
+      "step": 1890
+    },
+    {
+      "epoch": 0.6333333333333333,
+      "grad_norm": 3.5,
+      "learning_rate": 7.143116325950266e-06,
+      "loss": 0.2207,
+      "step": 1900
+    },
+    {
+      "epoch": 0.6366666666666667,
+      "grad_norm": 3.4375,
+      "learning_rate": 7.031806335335372e-06,
+      "loss": 0.2216,
+      "step": 1910
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 4.46875,
+      "learning_rate": 6.920898190629242e-06,
+      "loss": 0.2622,
+      "step": 1920
+    },
+    {
+      "epoch": 0.6433333333333333,
+      "grad_norm": 3.203125,
+      "learning_rate": 6.8104069070193e-06,
+      "loss": 0.3646,
+      "step": 1930
+    },
+    {
+      "epoch": 0.6466666666666666,
+      "grad_norm": 2.234375,
+      "learning_rate": 6.700347443256661e-06,
+      "loss": 0.549,
+      "step": 1940
+    },
+    {
+      "epoch": 0.65,
+      "grad_norm": 1.765625,
+      "learning_rate": 6.590734699630939e-06,
+      "loss": 0.3519,
+      "step": 1950
+    },
+    {
+      "epoch": 0.6533333333333333,
+      "grad_norm": 1.96875,
+      "learning_rate": 6.481583515952983e-06,
+      "loss": 0.2032,
+      "step": 1960
+    },
+    {
+      "epoch": 0.6566666666666666,
+      "grad_norm": 2.421875,
+      "learning_rate": 6.372908669545832e-06,
+      "loss": 0.3308,
+      "step": 1970
+    },
+    {
+      "epoch": 0.66,
+      "grad_norm": 5.25,
+      "learning_rate": 6.26472487324407e-06,
+      "loss": 0.3102,
+      "step": 1980
+    },
+    {
+      "epoch": 0.6633333333333333,
+      "grad_norm": 1.9765625,
+      "learning_rate": 6.157046773401964e-06,
+      "loss": 0.2195,
+      "step": 1990
+    },
+    {
+      "epoch": 0.6666666666666666,
+      "grad_norm": 1.6953125,
+      "learning_rate": 6.049888947910569e-06,
+      "loss": 0.2017,
+      "step": 2000
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 3000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 9223372036854775807,
+  "save_steps": 2000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 7.377641417998336e+17,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-2000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c9a6c6da9f1036ae258a284992f59dfabb5196c32616e982f11533f2fa09bb9
+size 5368