Training in progress, step 419, checkpoint

.gitattributes

@@ -35,3 +35,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint-227/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-419/tokenizer.json filter=lfs diff=lfs merge=lfs -text

checkpoint-419/config.json

@@ -0,0 +1,35 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128009,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.1",
+  "use_cache": false,
+  "vocab_size": 128256
+}

checkpoint-419/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.51.1"
+}

checkpoint-419/global_step419/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd5affe994d7e40d67154027b182e11215b0eac81bcf8fcb9cb38a631fef9f33
+size 24217512847

checkpoint-419/global_step419/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73d493daf2c22d49690c63551c239585da92656eadaf2655144ffc5a5d9efa3a
+size 24217512847

checkpoint-419/global_step419/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb0d797ee33b89625f5de919e89360edecb40559719e623b1528efdde8cc4f2d
+size 151845

checkpoint-419/global_step419/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc4b0555462b1a7e77a256e3caee327e0fd0ed5c6ac8391917dfa467fdefb282
+size 151845

checkpoint-419/latest

@@ -0,0 +1 @@
+global_step419

checkpoint-419/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:132bd6e87c992b845bb1557570cef16ea2f0ee59f7bf7e6e4fd52fc0fb52cf27
+size 4976698672

checkpoint-419/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:940faecb2817abe834f81a32b6e3ae609d664802554081fa8930ed58bb18edaa
+size 4999802720

checkpoint-419/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cddeff4b187d9e98053ac3b7fa557f24646884fff54bea13653e12f2e2d86f45
+size 4915916176

checkpoint-419/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4759f3211a761b6346af14ac90262da2c0c0f8f5bcf59198d9a76e844edc9adc
+size 1168138808

checkpoint-419/model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 16060522496
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

checkpoint-419/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5760f1347dfb235ea1535097ed60f32852e195369a850d128196e21060244ece
+size 14512

checkpoint-419/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46c3975c377d1a0e4265d8759569136468ef63d7f0908f40a33f1ddcab855fa1
+size 14512

checkpoint-419/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:002f0fa934388364ec53934305a110b18b623d792c1166c5ea913eb6eb6725f1
+size 1064

checkpoint-419/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-419/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

checkpoint-419/tokenizer_config.json

@@ -0,0 +1,2064 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|end_of_text|>",
+  "tokenizer_class": "PreTrainedTokenizer"
+}

checkpoint-419/trainer_state.json

@@ -0,0 +1,2983 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.5001492091912861,
+  "eval_steps": 279,
+  "global_step": 419,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.001193673530289466,
+      "grad_norm": 8.16114303780401,
+      "learning_rate": 0.0,
+      "loss": 1.2384,
+      "step": 1
+    },
+    {
+      "epoch": 0.001193673530289466,
+      "eval_loss": 1.2247389554977417,
+      "eval_runtime": 377.0756,
+      "eval_samples_per_second": 29.405,
+      "eval_steps_per_second": 14.703,
+      "step": 1
+    },
+    {
+      "epoch": 0.002387347060578932,
+      "grad_norm": 8.47871027916886,
+      "learning_rate": 1.2500000000000002e-07,
+      "loss": 1.2206,
+      "step": 2
+    },
+    {
+      "epoch": 0.0035810205908683975,
+      "grad_norm": 9.074249796948349,
+      "learning_rate": 2.5000000000000004e-07,
+      "loss": 1.2612,
+      "step": 3
+    },
+    {
+      "epoch": 0.004774694121157864,
+      "grad_norm": 8.46395449442453,
+      "learning_rate": 3.75e-07,
+      "loss": 1.2244,
+      "step": 4
+    },
+    {
+      "epoch": 0.005968367651447329,
+      "grad_norm": 8.623468246740337,
+      "learning_rate": 5.000000000000001e-07,
+      "loss": 1.2237,
+      "step": 5
+    },
+    {
+      "epoch": 0.007162041181736795,
+      "grad_norm": 8.009333963998843,
+      "learning_rate": 6.25e-07,
+      "loss": 1.1959,
+      "step": 6
+    },
+    {
+      "epoch": 0.008355714712026261,
+      "grad_norm": 7.985626918976279,
+      "learning_rate": 7.5e-07,
+      "loss": 1.1517,
+      "step": 7
+    },
+    {
+      "epoch": 0.009549388242315727,
+      "grad_norm": 7.7914409534481095,
+      "learning_rate": 8.75e-07,
+      "loss": 1.1721,
+      "step": 8
+    },
+    {
+      "epoch": 0.010743061772605193,
+      "grad_norm": 7.439660012763994,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 1.2006,
+      "step": 9
+    },
+    {
+      "epoch": 0.011936735302894658,
+      "grad_norm": 4.8095912529024405,
+      "learning_rate": 1.125e-06,
+      "loss": 1.2075,
+      "step": 10
+    },
+    {
+      "epoch": 0.013130408833184124,
+      "grad_norm": 3.891998890477489,
+      "learning_rate": 1.25e-06,
+      "loss": 1.148,
+      "step": 11
+    },
+    {
+      "epoch": 0.01432408236347359,
+      "grad_norm": 3.259994639596157,
+      "learning_rate": 1.3750000000000002e-06,
+      "loss": 1.1158,
+      "step": 12
+    },
+    {
+      "epoch": 0.015517755893763056,
+      "grad_norm": 4.11327402736425,
+      "learning_rate": 1.5e-06,
+      "loss": 1.1558,
+      "step": 13
+    },
+    {
+      "epoch": 0.016711429424052522,
+      "grad_norm": 5.241297644735645,
+      "learning_rate": 1.6250000000000001e-06,
+      "loss": 1.112,
+      "step": 14
+    },
+    {
+      "epoch": 0.017905102954341987,
+      "grad_norm": 4.83489746805021,
+      "learning_rate": 1.75e-06,
+      "loss": 1.059,
+      "step": 15
+    },
+    {
+      "epoch": 0.019098776484631454,
+      "grad_norm": 4.926431898693722,
+      "learning_rate": 1.8750000000000003e-06,
+      "loss": 1.2208,
+      "step": 16
+    },
+    {
+      "epoch": 0.02029245001492092,
+      "grad_norm": 3.8781961339772217,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 1.1505,
+      "step": 17
+    },
+    {
+      "epoch": 0.021486123545210387,
+      "grad_norm": 2.543621117165087,
+      "learning_rate": 2.125e-06,
+      "loss": 1.1362,
+      "step": 18
+    },
+    {
+      "epoch": 0.02267979707549985,
+      "grad_norm": 2.1009860488188368,
+      "learning_rate": 2.25e-06,
+      "loss": 1.0701,
+      "step": 19
+    },
+    {
+      "epoch": 0.023873470605789315,
+      "grad_norm": 1.8927575511303283,
+      "learning_rate": 2.375e-06,
+      "loss": 1.0381,
+      "step": 20
+    },
+    {
+      "epoch": 0.025067144136078783,
+      "grad_norm": 1.7163748223593238,
+      "learning_rate": 2.5e-06,
+      "loss": 1.0771,
+      "step": 21
+    },
+    {
+      "epoch": 0.026260817666368248,
+      "grad_norm": 1.4814021734613099,
+      "learning_rate": 2.6250000000000003e-06,
+      "loss": 1.06,
+      "step": 22
+    },
+    {
+      "epoch": 0.027454491196657715,
+      "grad_norm": 1.292017848420993,
+      "learning_rate": 2.7500000000000004e-06,
+      "loss": 1.0334,
+      "step": 23
+    },
+    {
+      "epoch": 0.02864816472694718,
+      "grad_norm": 1.2760011449446051,
+      "learning_rate": 2.875e-06,
+      "loss": 0.9496,
+      "step": 24
+    },
+    {
+      "epoch": 0.029841838257236644,
+      "grad_norm": 1.5611299892461064,
+      "learning_rate": 3e-06,
+      "loss": 1.0631,
+      "step": 25
+    },
+    {
+      "epoch": 0.031035511787526112,
+      "grad_norm": 1.5071399385518096,
+      "learning_rate": 3.125e-06,
+      "loss": 1.0503,
+      "step": 26
+    },
+    {
+      "epoch": 0.03222918531781558,
+      "grad_norm": 1.4011246881564738,
+      "learning_rate": 3.2500000000000002e-06,
+      "loss": 0.9864,
+      "step": 27
+    },
+    {
+      "epoch": 0.033422858848105044,
+      "grad_norm": 1.3575328401932647,
+      "learning_rate": 3.3750000000000003e-06,
+      "loss": 1.037,
+      "step": 28
+    },
+    {
+      "epoch": 0.03461653237839451,
+      "grad_norm": 1.347922922109771,
+      "learning_rate": 3.5e-06,
+      "loss": 1.0007,
+      "step": 29
+    },
+    {
+      "epoch": 0.03581020590868397,
+      "grad_norm": 1.3032805970679422,
+      "learning_rate": 3.625e-06,
+      "loss": 1.0191,
+      "step": 30
+    },
+    {
+      "epoch": 0.03700387943897344,
+      "grad_norm": 1.2401790084146032,
+      "learning_rate": 3.7500000000000005e-06,
+      "loss": 0.9624,
+      "step": 31
+    },
+    {
+      "epoch": 0.03819755296926291,
+      "grad_norm": 1.2794209814081245,
+      "learning_rate": 3.875e-06,
+      "loss": 1.0257,
+      "step": 32
+    },
+    {
+      "epoch": 0.03939122649955237,
+      "grad_norm": 1.4446359015526915,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 1.0076,
+      "step": 33
+    },
+    {
+      "epoch": 0.04058490002984184,
+      "grad_norm": 1.3110810974574507,
+      "learning_rate": 4.125e-06,
+      "loss": 0.9771,
+      "step": 34
+    },
+    {
+      "epoch": 0.0417785735601313,
+      "grad_norm": 1.2219402025041668,
+      "learning_rate": 4.25e-06,
+      "loss": 0.9595,
+      "step": 35
+    },
+    {
+      "epoch": 0.04297224709042077,
+      "grad_norm": 1.2409147982217572,
+      "learning_rate": 4.3750000000000005e-06,
+      "loss": 0.9904,
+      "step": 36
+    },
+    {
+      "epoch": 0.04416592062071024,
+      "grad_norm": 1.3508715258107795,
+      "learning_rate": 4.5e-06,
+      "loss": 0.9698,
+      "step": 37
+    },
+    {
+      "epoch": 0.0453595941509997,
+      "grad_norm": 1.1392387957219212,
+      "learning_rate": 4.625000000000001e-06,
+      "loss": 0.9517,
+      "step": 38
+    },
+    {
+      "epoch": 0.046553267681289166,
+      "grad_norm": 1.175657348183339,
+      "learning_rate": 4.75e-06,
+      "loss": 0.9738,
+      "step": 39
+    },
+    {
+      "epoch": 0.04774694121157863,
+      "grad_norm": 1.29451100293097,
+      "learning_rate": 4.875e-06,
+      "loss": 1.015,
+      "step": 40
+    },
+    {
+      "epoch": 0.0489406147418681,
+      "grad_norm": 1.0777298510039686,
+      "learning_rate": 5e-06,
+      "loss": 0.9782,
+      "step": 41
+    },
+    {
+      "epoch": 0.050134288272157566,
+      "grad_norm": 0.9876049508091342,
+      "learning_rate": 4.999995379323966e-06,
+      "loss": 0.8986,
+      "step": 42
+    },
+    {
+      "epoch": 0.05132796180244703,
+      "grad_norm": 1.099457410326718,
+      "learning_rate": 4.999981517312945e-06,
+      "loss": 0.9397,
+      "step": 43
+    },
+    {
+      "epoch": 0.052521635332736495,
+      "grad_norm": 1.2100440309282297,
+      "learning_rate": 4.999958414018178e-06,
+      "loss": 0.9756,
+      "step": 44
+    },
+    {
+      "epoch": 0.05371530886302596,
+      "grad_norm": 1.1305259086159474,
+      "learning_rate": 4.999926069525066e-06,
+      "loss": 0.9714,
+      "step": 45
+    },
+    {
+      "epoch": 0.05490898239331543,
+      "grad_norm": 1.0989207268618686,
+      "learning_rate": 4.9998844839531746e-06,
+      "loss": 0.9344,
+      "step": 46
+    },
+    {
+      "epoch": 0.056102655923604895,
+      "grad_norm": 1.1398334118236317,
+      "learning_rate": 4.999833657456224e-06,
+      "loss": 0.9495,
+      "step": 47
+    },
+    {
+      "epoch": 0.05729632945389436,
+      "grad_norm": 1.1293435303709396,
+      "learning_rate": 4.999773590222098e-06,
+      "loss": 0.9222,
+      "step": 48
+    },
+    {
+      "epoch": 0.058490002984183824,
+      "grad_norm": 1.017087531351413,
+      "learning_rate": 4.9997042824728365e-06,
+      "loss": 0.8748,
+      "step": 49
+    },
+    {
+      "epoch": 0.05968367651447329,
+      "grad_norm": 1.1258466933903977,
+      "learning_rate": 4.999625734464639e-06,
+      "loss": 0.9609,
+      "step": 50
+    },
+    {
+      "epoch": 0.06087735004476276,
+      "grad_norm": 1.0209775313724632,
+      "learning_rate": 4.999537946487862e-06,
+      "loss": 0.9179,
+      "step": 51
+    },
+    {
+      "epoch": 0.062071023575052224,
+      "grad_norm": 1.166110386790191,
+      "learning_rate": 4.999440918867016e-06,
+      "loss": 0.8819,
+      "step": 52
+    },
+    {
+      "epoch": 0.06326469710534169,
+      "grad_norm": 1.1351149008586316,
+      "learning_rate": 4.9993346519607685e-06,
+      "loss": 0.8704,
+      "step": 53
+    },
+    {
+      "epoch": 0.06445837063563116,
+      "grad_norm": 1.2014927628973993,
+      "learning_rate": 4.9992191461619384e-06,
+      "loss": 0.8879,
+      "step": 54
+    },
+    {
+      "epoch": 0.06565204416592062,
+      "grad_norm": 1.2912944317710835,
+      "learning_rate": 4.9990944018975e-06,
+      "loss": 0.9883,
+      "step": 55
+    },
+    {
+      "epoch": 0.06684571769621009,
+      "grad_norm": 1.0990357747364996,
+      "learning_rate": 4.998960419628573e-06,
+      "loss": 0.8552,
+      "step": 56
+    },
+    {
+      "epoch": 0.06803939122649955,
+      "grad_norm": 1.1749242302529268,
+      "learning_rate": 4.99881719985043e-06,
+      "loss": 0.9289,
+      "step": 57
+    },
+    {
+      "epoch": 0.06923306475678902,
+      "grad_norm": 1.1416138380576308,
+      "learning_rate": 4.998664743092487e-06,
+      "loss": 0.8816,
+      "step": 58
+    },
+    {
+      "epoch": 0.07042673828707849,
+      "grad_norm": 1.0989269231311176,
+      "learning_rate": 4.998503049918308e-06,
+      "loss": 0.914,
+      "step": 59
+    },
+    {
+      "epoch": 0.07162041181736795,
+      "grad_norm": 1.061068993065825,
+      "learning_rate": 4.998332120925598e-06,
+      "loss": 0.884,
+      "step": 60
+    },
+    {
+      "epoch": 0.07281408534765742,
+      "grad_norm": 1.1286732153102872,
+      "learning_rate": 4.998151956746204e-06,
+      "loss": 0.8633,
+      "step": 61
+    },
+    {
+      "epoch": 0.07400775887794687,
+      "grad_norm": 1.1770687865566731,
+      "learning_rate": 4.997962558046109e-06,
+      "loss": 0.9322,
+      "step": 62
+    },
+    {
+      "epoch": 0.07520143240823635,
+      "grad_norm": 1.0614639945429885,
+      "learning_rate": 4.997763925525433e-06,
+      "loss": 0.8853,
+      "step": 63
+    },
+    {
+      "epoch": 0.07639510593852582,
+      "grad_norm": 1.0831541750852,
+      "learning_rate": 4.99755605991843e-06,
+      "loss": 0.8573,
+      "step": 64
+    },
+    {
+      "epoch": 0.07758877946881527,
+      "grad_norm": 1.0506713962732483,
+      "learning_rate": 4.997338961993483e-06,
+      "loss": 0.8369,
+      "step": 65
+    },
+    {
+      "epoch": 0.07878245299910475,
+      "grad_norm": 1.0962691367409116,
+      "learning_rate": 4.997112632553104e-06,
+      "loss": 0.9228,
+      "step": 66
+    },
+    {
+      "epoch": 0.07997612652939422,
+      "grad_norm": 1.0345560521905983,
+      "learning_rate": 4.9968770724339284e-06,
+      "loss": 0.8476,
+      "step": 67
+    },
+    {
+      "epoch": 0.08116980005968367,
+      "grad_norm": 1.0969300499561807,
+      "learning_rate": 4.996632282506714e-06,
+      "loss": 0.8793,
+      "step": 68
+    },
+    {
+      "epoch": 0.08236347358997315,
+      "grad_norm": 1.0777700101385033,
+      "learning_rate": 4.996378263676337e-06,
+      "loss": 0.8464,
+      "step": 69
+    },
+    {
+      "epoch": 0.0835571471202626,
+      "grad_norm": 1.0760968526682098,
+      "learning_rate": 4.996115016881788e-06,
+      "loss": 0.8998,
+      "step": 70
+    },
+    {
+      "epoch": 0.08475082065055208,
+      "grad_norm": 1.1678040527069657,
+      "learning_rate": 4.995842543096171e-06,
+      "loss": 0.9122,
+      "step": 71
+    },
+    {
+      "epoch": 0.08594449418084155,
+      "grad_norm": 1.07731981266689,
+      "learning_rate": 4.9955608433266934e-06,
+      "loss": 0.8386,
+      "step": 72
+    },
+    {
+      "epoch": 0.087138167711131,
+      "grad_norm": 1.297888828978233,
+      "learning_rate": 4.995269918614672e-06,
+      "loss": 0.8945,
+      "step": 73
+    },
+    {
+      "epoch": 0.08833184124142048,
+      "grad_norm": 1.1408935228222659,
+      "learning_rate": 4.994969770035523e-06,
+      "loss": 0.8548,
+      "step": 74
+    },
+    {
+      "epoch": 0.08952551477170993,
+      "grad_norm": 1.0573172294992146,
+      "learning_rate": 4.994660398698755e-06,
+      "loss": 0.8896,
+      "step": 75
+    },
+    {
+      "epoch": 0.0907191883019994,
+      "grad_norm": 1.1870512065956376,
+      "learning_rate": 4.994341805747974e-06,
+      "loss": 0.8428,
+      "step": 76
+    },
+    {
+      "epoch": 0.09191286183228888,
+      "grad_norm": 1.1767792325463364,
+      "learning_rate": 4.99401399236087e-06,
+      "loss": 0.8854,
+      "step": 77
+    },
+    {
+      "epoch": 0.09310653536257833,
+      "grad_norm": 1.1733632166017016,
+      "learning_rate": 4.993676959749221e-06,
+      "loss": 0.8359,
+      "step": 78
+    },
+    {
+      "epoch": 0.0943002088928678,
+      "grad_norm": 1.168122428967752,
+      "learning_rate": 4.993330709158879e-06,
+      "loss": 0.9055,
+      "step": 79
+    },
+    {
+      "epoch": 0.09549388242315726,
+      "grad_norm": 1.1161660017475175,
+      "learning_rate": 4.992975241869776e-06,
+      "loss": 0.8217,
+      "step": 80
+    },
+    {
+      "epoch": 0.09668755595344673,
+      "grad_norm": 1.2226228718020973,
+      "learning_rate": 4.99261055919591e-06,
+      "loss": 0.8574,
+      "step": 81
+    },
+    {
+      "epoch": 0.0978812294837362,
+      "grad_norm": 1.1584273725751542,
+      "learning_rate": 4.992236662485346e-06,
+      "loss": 0.8365,
+      "step": 82
+    },
+    {
+      "epoch": 0.09907490301402566,
+      "grad_norm": 1.1636195971387586,
+      "learning_rate": 4.9918535531202075e-06,
+      "loss": 0.8985,
+      "step": 83
+    },
+    {
+      "epoch": 0.10026857654431513,
+      "grad_norm": 1.2855272207779183,
+      "learning_rate": 4.991461232516675e-06,
+      "loss": 0.8305,
+      "step": 84
+    },
+    {
+      "epoch": 0.10146225007460459,
+      "grad_norm": 1.1308363940389612,
+      "learning_rate": 4.991059702124976e-06,
+      "loss": 0.819,
+      "step": 85
+    },
+    {
+      "epoch": 0.10265592360489406,
+      "grad_norm": 1.1543993257508203,
+      "learning_rate": 4.990648963429387e-06,
+      "loss": 0.8702,
+      "step": 86
+    },
+    {
+      "epoch": 0.10384959713518353,
+      "grad_norm": 1.349313529314071,
+      "learning_rate": 4.990229017948217e-06,
+      "loss": 0.8519,
+      "step": 87
+    },
+    {
+      "epoch": 0.10504327066547299,
+      "grad_norm": 1.09965470628552,
+      "learning_rate": 4.989799867233815e-06,
+      "loss": 0.8627,
+      "step": 88
+    },
+    {
+      "epoch": 0.10623694419576246,
+      "grad_norm": 1.1370315208165804,
+      "learning_rate": 4.98936151287255e-06,
+      "loss": 0.8496,
+      "step": 89
+    },
+    {
+      "epoch": 0.10743061772605192,
+      "grad_norm": 1.1872879419869933,
+      "learning_rate": 4.98891395648482e-06,
+      "loss": 0.8343,
+      "step": 90
+    },
+    {
+      "epoch": 0.10862429125634139,
+      "grad_norm": 1.1061958932042313,
+      "learning_rate": 4.988457199725034e-06,
+      "loss": 0.8297,
+      "step": 91
+    },
+    {
+      "epoch": 0.10981796478663086,
+      "grad_norm": 1.3027114198402223,
+      "learning_rate": 4.9879912442816126e-06,
+      "loss": 0.8866,
+      "step": 92
+    },
+    {
+      "epoch": 0.11101163831692032,
+      "grad_norm": 1.314238053408892,
+      "learning_rate": 4.987516091876979e-06,
+      "loss": 0.8491,
+      "step": 93
+    },
+    {
+      "epoch": 0.11220531184720979,
+      "grad_norm": 1.1979509083460977,
+      "learning_rate": 4.987031744267554e-06,
+      "loss": 0.8703,
+      "step": 94
+    },
+    {
+      "epoch": 0.11339898537749925,
+      "grad_norm": 1.234194872455364,
+      "learning_rate": 4.986538203243748e-06,
+      "loss": 0.8446,
+      "step": 95
+    },
+    {
+      "epoch": 0.11459265890778872,
+      "grad_norm": 1.0957514650184408,
+      "learning_rate": 4.986035470629955e-06,
+      "loss": 0.8096,
+      "step": 96
+    },
+    {
+      "epoch": 0.11578633243807819,
+      "grad_norm": 1.1234554432924009,
+      "learning_rate": 4.9855235482845466e-06,
+      "loss": 0.8044,
+      "step": 97
+    },
+    {
+      "epoch": 0.11698000596836765,
+      "grad_norm": 1.049189288237806,
+      "learning_rate": 4.9850024380998655e-06,
+      "loss": 0.8193,
+      "step": 98
+    },
+    {
+      "epoch": 0.11817367949865712,
+      "grad_norm": 1.0985847135524742,
+      "learning_rate": 4.984472142002216e-06,
+      "loss": 0.8372,
+      "step": 99
+    },
+    {
+      "epoch": 0.11936735302894658,
+      "grad_norm": 1.1124501715562143,
+      "learning_rate": 4.983932661951859e-06,
+      "loss": 0.8114,
+      "step": 100
+    },
+    {
+      "epoch": 0.12056102655923605,
+      "grad_norm": 1.1094171803008324,
+      "learning_rate": 4.9833839999430065e-06,
+      "loss": 0.8427,
+      "step": 101
+    },
+    {
+      "epoch": 0.12175470008952552,
+      "grad_norm": 1.1042304519119917,
+      "learning_rate": 4.982826158003808e-06,
+      "loss": 0.8536,
+      "step": 102
+    },
+    {
+      "epoch": 0.12294837361981498,
+      "grad_norm": 1.09018073189874,
+      "learning_rate": 4.982259138196349e-06,
+      "loss": 0.8446,
+      "step": 103
+    },
+    {
+      "epoch": 0.12414204715010445,
+      "grad_norm": 1.1105099719277685,
+      "learning_rate": 4.981682942616643e-06,
+      "loss": 0.8256,
+      "step": 104
+    },
+    {
+      "epoch": 0.12533572068039392,
+      "grad_norm": 1.2208595723131699,
+      "learning_rate": 4.981097573394618e-06,
+      "loss": 0.8081,
+      "step": 105
+    },
+    {
+      "epoch": 0.12652939421068338,
+      "grad_norm": 1.0866825223378307,
+      "learning_rate": 4.980503032694118e-06,
+      "loss": 0.7415,
+      "step": 106
+    },
+    {
+      "epoch": 0.12772306774097283,
+      "grad_norm": 1.2365667742638329,
+      "learning_rate": 4.979899322712885e-06,
+      "loss": 0.8064,
+      "step": 107
+    },
+    {
+      "epoch": 0.12891674127126232,
+      "grad_norm": 1.2028095795753095,
+      "learning_rate": 4.979286445682558e-06,
+      "loss": 0.8153,
+      "step": 108
+    },
+    {
+      "epoch": 0.13011041480155178,
+      "grad_norm": 1.2054961617763407,
+      "learning_rate": 4.978664403868664e-06,
+      "loss": 0.8302,
+      "step": 109
+    },
+    {
+      "epoch": 0.13130408833184123,
+      "grad_norm": 1.2460181051686399,
+      "learning_rate": 4.978033199570602e-06,
+      "loss": 0.8538,
+      "step": 110
+    },
+    {
+      "epoch": 0.13249776186213072,
+      "grad_norm": 1.218146180548586,
+      "learning_rate": 4.977392835121649e-06,
+      "loss": 0.8437,
+      "step": 111
+    },
+    {
+      "epoch": 0.13369143539242018,
+      "grad_norm": 1.1112170995013697,
+      "learning_rate": 4.976743312888934e-06,
+      "loss": 0.7917,
+      "step": 112
+    },
+    {
+      "epoch": 0.13488510892270963,
+      "grad_norm": 1.2250435472005108,
+      "learning_rate": 4.976084635273445e-06,
+      "loss": 0.8517,
+      "step": 113
+    },
+    {
+      "epoch": 0.1360787824529991,
+      "grad_norm": 1.248560391747664,
+      "learning_rate": 4.9754168047100095e-06,
+      "loss": 0.8364,
+      "step": 114
+    },
+    {
+      "epoch": 0.13727245598328858,
+      "grad_norm": 1.4057157658992752,
+      "learning_rate": 4.974739823667292e-06,
+      "loss": 0.8428,
+      "step": 115
+    },
+    {
+      "epoch": 0.13846612951357803,
+      "grad_norm": 1.232101520188445,
+      "learning_rate": 4.974053694647779e-06,
+      "loss": 0.8019,
+      "step": 116
+    },
+    {
+      "epoch": 0.1396598030438675,
+      "grad_norm": 1.0691739882049396,
+      "learning_rate": 4.973358420187776e-06,
+      "loss": 0.7876,
+      "step": 117
+    },
+    {
+      "epoch": 0.14085347657415698,
+      "grad_norm": 1.0990717855699714,
+      "learning_rate": 4.972654002857392e-06,
+      "loss": 0.8062,
+      "step": 118
+    },
+    {
+      "epoch": 0.14204715010444643,
+      "grad_norm": 1.1914019305541128,
+      "learning_rate": 4.9719404452605345e-06,
+      "loss": 0.824,
+      "step": 119
+    },
+    {
+      "epoch": 0.1432408236347359,
+      "grad_norm": 1.248534127601636,
+      "learning_rate": 4.971217750034899e-06,
+      "loss": 0.8595,
+      "step": 120
+    },
+    {
+      "epoch": 0.14443449716502538,
+      "grad_norm": 1.1363883709126494,
+      "learning_rate": 4.970485919851958e-06,
+      "loss": 0.7774,
+      "step": 121
+    },
+    {
+      "epoch": 0.14562817069531483,
+      "grad_norm": 1.192101050796341,
+      "learning_rate": 4.969744957416951e-06,
+      "loss": 0.7547,
+      "step": 122
+    },
+    {
+      "epoch": 0.1468218442256043,
+      "grad_norm": 1.10790877586813,
+      "learning_rate": 4.968994865468876e-06,
+      "loss": 0.8087,
+      "step": 123
+    },
+    {
+      "epoch": 0.14801551775589375,
+      "grad_norm": 1.5631572663942805,
+      "learning_rate": 4.968235646780479e-06,
+      "loss": 0.8512,
+      "step": 124
+    },
+    {
+      "epoch": 0.14920919128618323,
+      "grad_norm": 1.2566894341004975,
+      "learning_rate": 4.967467304158242e-06,
+      "loss": 0.8073,
+      "step": 125
+    },
+    {
+      "epoch": 0.1504028648164727,
+      "grad_norm": 1.2197632966265681,
+      "learning_rate": 4.9666898404423755e-06,
+      "loss": 0.7886,
+      "step": 126
+    },
+    {
+      "epoch": 0.15159653834676215,
+      "grad_norm": 1.1342051167019487,
+      "learning_rate": 4.965903258506806e-06,
+      "loss": 0.795,
+      "step": 127
+    },
+    {
+      "epoch": 0.15279021187705163,
+      "grad_norm": 1.109864145510677,
+      "learning_rate": 4.965107561259166e-06,
+      "loss": 0.8079,
+      "step": 128
+    },
+    {
+      "epoch": 0.1539838854073411,
+      "grad_norm": 1.1953603077930386,
+      "learning_rate": 4.964302751640782e-06,
+      "loss": 0.7621,
+      "step": 129
+    },
+    {
+      "epoch": 0.15517755893763055,
+      "grad_norm": 1.1382695927374318,
+      "learning_rate": 4.9634888326266665e-06,
+      "loss": 0.7612,
+      "step": 130
+    },
+    {
+      "epoch": 0.15637123246792003,
+      "grad_norm": 1.1079042802908126,
+      "learning_rate": 4.962665807225503e-06,
+      "loss": 0.7882,
+      "step": 131
+    },
+    {
+      "epoch": 0.1575649059982095,
+      "grad_norm": 1.1881329570436796,
+      "learning_rate": 4.961833678479639e-06,
+      "loss": 0.7843,
+      "step": 132
+    },
+    {
+      "epoch": 0.15875857952849895,
+      "grad_norm": 1.219136491476124,
+      "learning_rate": 4.960992449465073e-06,
+      "loss": 0.7609,
+      "step": 133
+    },
+    {
+      "epoch": 0.15995225305878844,
+      "grad_norm": 1.1300470130912046,
+      "learning_rate": 4.9601421232914435e-06,
+      "loss": 0.7713,
+      "step": 134
+    },
+    {
+      "epoch": 0.1611459265890779,
+      "grad_norm": 1.2443157752142076,
+      "learning_rate": 4.959282703102013e-06,
+      "loss": 0.7944,
+      "step": 135
+    },
+    {
+      "epoch": 0.16233960011936735,
+      "grad_norm": 1.2537008022769138,
+      "learning_rate": 4.958414192073665e-06,
+      "loss": 0.785,
+      "step": 136
+    },
+    {
+      "epoch": 0.1635332736496568,
+      "grad_norm": 1.1669056420133548,
+      "learning_rate": 4.957536593416887e-06,
+      "loss": 0.7743,
+      "step": 137
+    },
+    {
+      "epoch": 0.1647269471799463,
+      "grad_norm": 1.3588053892144505,
+      "learning_rate": 4.956649910375757e-06,
+      "loss": 0.7943,
+      "step": 138
+    },
+    {
+      "epoch": 0.16592062071023575,
+      "grad_norm": 1.4775062989961492,
+      "learning_rate": 4.955754146227934e-06,
+      "loss": 0.761,
+      "step": 139
+    },
+    {
+      "epoch": 0.1671142942405252,
+      "grad_norm": 1.3286915838523645,
+      "learning_rate": 4.954849304284649e-06,
+      "loss": 0.8195,
+      "step": 140
+    },
+    {
+      "epoch": 0.1683079677708147,
+      "grad_norm": 1.698625154491963,
+      "learning_rate": 4.9539353878906855e-06,
+      "loss": 0.7483,
+      "step": 141
+    },
+    {
+      "epoch": 0.16950164130110415,
+      "grad_norm": 1.5149991463502988,
+      "learning_rate": 4.953012400424373e-06,
+      "loss": 0.7382,
+      "step": 142
+    },
+    {
+      "epoch": 0.1706953148313936,
+      "grad_norm": 1.8018380897847472,
+      "learning_rate": 4.952080345297573e-06,
+      "loss": 0.8065,
+      "step": 143
+    },
+    {
+      "epoch": 0.1718889883616831,
+      "grad_norm": 1.974668240659228,
+      "learning_rate": 4.951139225955666e-06,
+      "loss": 0.7603,
+      "step": 144
+    },
+    {
+      "epoch": 0.17308266189197255,
+      "grad_norm": 1.3490501205848957,
+      "learning_rate": 4.950189045877536e-06,
+      "loss": 0.7718,
+      "step": 145
+    },
+    {
+      "epoch": 0.174276335422262,
+      "grad_norm": 2.0029868054669704,
+      "learning_rate": 4.9492298085755645e-06,
+      "loss": 0.7727,
+      "step": 146
+    },
+    {
+      "epoch": 0.17547000895255147,
+      "grad_norm": 1.2543801528507452,
+      "learning_rate": 4.94826151759561e-06,
+      "loss": 0.8365,
+      "step": 147
+    },
+    {
+      "epoch": 0.17666368248284095,
+      "grad_norm": 1.3454571859146043,
+      "learning_rate": 4.9472841765169995e-06,
+      "loss": 0.8399,
+      "step": 148
+    },
+    {
+      "epoch": 0.1778573560131304,
+      "grad_norm": 1.104388470919368,
+      "learning_rate": 4.946297788952515e-06,
+      "loss": 0.7765,
+      "step": 149
+    },
+    {
+      "epoch": 0.17905102954341987,
+      "grad_norm": 1.2845752585530912,
+      "learning_rate": 4.945302358548379e-06,
+      "loss": 0.8399,
+      "step": 150
+    },
+    {
+      "epoch": 0.18024470307370935,
+      "grad_norm": 1.38666460741284,
+      "learning_rate": 4.944297888984239e-06,
+      "loss": 0.8114,
+      "step": 151
+    },
+    {
+      "epoch": 0.1814383766039988,
+      "grad_norm": 1.294939717819466,
+      "learning_rate": 4.943284383973159e-06,
+      "loss": 0.7287,
+      "step": 152
+    },
+    {
+      "epoch": 0.18263205013428827,
+      "grad_norm": 1.0350534121793518,
+      "learning_rate": 4.9422618472616005e-06,
+      "loss": 0.7641,
+      "step": 153
+    },
+    {
+      "epoch": 0.18382572366457775,
+      "grad_norm": 1.2604270595475044,
+      "learning_rate": 4.941230282629413e-06,
+      "loss": 0.856,
+      "step": 154
+    },
+    {
+      "epoch": 0.1850193971948672,
+      "grad_norm": 1.4459296581352967,
+      "learning_rate": 4.940189693889819e-06,
+      "loss": 0.8014,
+      "step": 155
+    },
+    {
+      "epoch": 0.18621307072515667,
+      "grad_norm": 1.4006999891427907,
+      "learning_rate": 4.939140084889393e-06,
+      "loss": 0.7809,
+      "step": 156
+    },
+    {
+      "epoch": 0.18740674425544612,
+      "grad_norm": 1.1961002337638094,
+      "learning_rate": 4.938081459508061e-06,
+      "loss": 0.6918,
+      "step": 157
+    },
+    {
+      "epoch": 0.1886004177857356,
+      "grad_norm": 2.311534488689422,
+      "learning_rate": 4.937013821659074e-06,
+      "loss": 0.7907,
+      "step": 158
+    },
+    {
+      "epoch": 0.18979409131602507,
+      "grad_norm": 1.2653651062834335,
+      "learning_rate": 4.935937175288999e-06,
+      "loss": 0.7393,
+      "step": 159
+    },
+    {
+      "epoch": 0.19098776484631452,
+      "grad_norm": 1.2203924343875887,
+      "learning_rate": 4.934851524377703e-06,
+      "loss": 0.779,
+      "step": 160
+    },
+    {
+      "epoch": 0.192181438376604,
+      "grad_norm": 1.4196639804720872,
+      "learning_rate": 4.933756872938338e-06,
+      "loss": 0.7693,
+      "step": 161
+    },
+    {
+      "epoch": 0.19337511190689347,
+      "grad_norm": 1.5375267721740755,
+      "learning_rate": 4.93265322501733e-06,
+      "loss": 0.7428,
+      "step": 162
+    },
+    {
+      "epoch": 0.19456878543718292,
+      "grad_norm": 1.1285713164450155,
+      "learning_rate": 4.931540584694356e-06,
+      "loss": 0.7569,
+      "step": 163
+    },
+    {
+      "epoch": 0.1957624589674724,
+      "grad_norm": 1.5604293357086574,
+      "learning_rate": 4.930418956082339e-06,
+      "loss": 0.8006,
+      "step": 164
+    },
+    {
+      "epoch": 0.19695613249776187,
+      "grad_norm": 1.3070516179192826,
+      "learning_rate": 4.929288343327423e-06,
+      "loss": 0.7604,
+      "step": 165
+    },
+    {
+      "epoch": 0.19814980602805132,
+      "grad_norm": 1.1644097946917475,
+      "learning_rate": 4.928148750608965e-06,
+      "loss": 0.775,
+      "step": 166
+    },
+    {
+      "epoch": 0.19934347955834078,
+      "grad_norm": 1.482255753807923,
+      "learning_rate": 4.927000182139516e-06,
+      "loss": 0.716,
+      "step": 167
+    },
+    {
+      "epoch": 0.20053715308863027,
+      "grad_norm": 1.3635195623605272,
+      "learning_rate": 4.925842642164806e-06,
+      "loss": 0.7361,
+      "step": 168
+    },
+    {
+      "epoch": 0.20173082661891972,
+      "grad_norm": 1.1101114682804836,
+      "learning_rate": 4.924676134963729e-06,
+      "loss": 0.6757,
+      "step": 169
+    },
+    {
+      "epoch": 0.20292450014920918,
+      "grad_norm": 2.3068740765877895,
+      "learning_rate": 4.923500664848327e-06,
+      "loss": 0.7764,
+      "step": 170
+    },
+    {
+      "epoch": 0.20411817367949867,
+      "grad_norm": 1.5055501328011494,
+      "learning_rate": 4.922316236163771e-06,
+      "loss": 0.6933,
+      "step": 171
+    },
+    {
+      "epoch": 0.20531184720978812,
+      "grad_norm": 1.2582812088169284,
+      "learning_rate": 4.921122853288353e-06,
+      "loss": 0.7629,
+      "step": 172
+    },
+    {
+      "epoch": 0.20650552074007758,
+      "grad_norm": 1.2992244424010069,
+      "learning_rate": 4.9199205206334595e-06,
+      "loss": 0.7659,
+      "step": 173
+    },
+    {
+      "epoch": 0.20769919427036707,
+      "grad_norm": 1.299795847544305,
+      "learning_rate": 4.918709242643563e-06,
+      "loss": 0.6963,
+      "step": 174
+    },
+    {
+      "epoch": 0.20889286780065652,
+      "grad_norm": 1.469667567259723,
+      "learning_rate": 4.917489023796202e-06,
+      "loss": 0.71,
+      "step": 175
+    },
+    {
+      "epoch": 0.21008654133094598,
+      "grad_norm": 1.2820417691715331,
+      "learning_rate": 4.9162598686019655e-06,
+      "loss": 0.7541,
+      "step": 176
+    },
+    {
+      "epoch": 0.21128021486123547,
+      "grad_norm": 1.472385187997062,
+      "learning_rate": 4.915021781604476e-06,
+      "loss": 0.7827,
+      "step": 177
+    },
+    {
+      "epoch": 0.21247388839152492,
+      "grad_norm": 1.5990993905513917,
+      "learning_rate": 4.913774767380372e-06,
+      "loss": 0.7331,
+      "step": 178
+    },
+    {
+      "epoch": 0.21366756192181438,
+      "grad_norm": 1.3084227582090084,
+      "learning_rate": 4.912518830539292e-06,
+      "loss": 0.7344,
+      "step": 179
+    },
+    {
+      "epoch": 0.21486123545210384,
+      "grad_norm": 1.2612182619687637,
+      "learning_rate": 4.9112539757238594e-06,
+      "loss": 0.7576,
+      "step": 180
+    },
+    {
+      "epoch": 0.21605490898239332,
+      "grad_norm": 1.4927518252334036,
+      "learning_rate": 4.909980207609661e-06,
+      "loss": 0.7997,
+      "step": 181
+    },
+    {
+      "epoch": 0.21724858251268278,
+      "grad_norm": 1.37544016331359,
+      "learning_rate": 4.908697530905231e-06,
+      "loss": 0.7331,
+      "step": 182
+    },
+    {
+      "epoch": 0.21844225604297224,
+      "grad_norm": 1.2623947407294713,
+      "learning_rate": 4.907405950352039e-06,
+      "loss": 0.7672,
+      "step": 183
+    },
+    {
+      "epoch": 0.21963592957326172,
+      "grad_norm": 1.3338183562733208,
+      "learning_rate": 4.906105470724464e-06,
+      "loss": 0.7652,
+      "step": 184
+    },
+    {
+      "epoch": 0.22082960310355118,
+      "grad_norm": 1.3526700045070938,
+      "learning_rate": 4.904796096829781e-06,
+      "loss": 0.7627,
+      "step": 185
+    },
+    {
+      "epoch": 0.22202327663384064,
+      "grad_norm": 1.296782122552344,
+      "learning_rate": 4.903477833508145e-06,
+      "loss": 0.7076,
+      "step": 186
+    },
+    {
+      "epoch": 0.22321695016413012,
+      "grad_norm": 1.174967837710992,
+      "learning_rate": 4.902150685632571e-06,
+      "loss": 0.7471,
+      "step": 187
+    },
+    {
+      "epoch": 0.22441062369441958,
+      "grad_norm": 1.2483337231732115,
+      "learning_rate": 4.900814658108914e-06,
+      "loss": 0.7849,
+      "step": 188
+    },
+    {
+      "epoch": 0.22560429722470904,
+      "grad_norm": 1.2446901896593106,
+      "learning_rate": 4.899469755875855e-06,
+      "loss": 0.6872,
+      "step": 189
+    },
+    {
+      "epoch": 0.2267979707549985,
+      "grad_norm": 1.4671891738307608,
+      "learning_rate": 4.89811598390488e-06,
+      "loss": 0.7305,
+      "step": 190
+    },
+    {
+      "epoch": 0.22799164428528798,
+      "grad_norm": 1.2943348672718462,
+      "learning_rate": 4.896753347200261e-06,
+      "loss": 0.7107,
+      "step": 191
+    },
+    {
+      "epoch": 0.22918531781557744,
+      "grad_norm": 1.5157144950189583,
+      "learning_rate": 4.895381850799042e-06,
+      "loss": 0.7293,
+      "step": 192
+    },
+    {
+      "epoch": 0.2303789913458669,
+      "grad_norm": 1.115807759982719,
+      "learning_rate": 4.894001499771015e-06,
+      "loss": 0.7373,
+      "step": 193
+    },
+    {
+      "epoch": 0.23157266487615638,
+      "grad_norm": 1.3481941616422346,
+      "learning_rate": 4.892612299218704e-06,
+      "loss": 0.7352,
+      "step": 194
+    },
+    {
+      "epoch": 0.23276633840644584,
+      "grad_norm": 1.519744205896213,
+      "learning_rate": 4.891214254277345e-06,
+      "loss": 0.7576,
+      "step": 195
+    },
+    {
+      "epoch": 0.2339600119367353,
+      "grad_norm": 1.553584724264315,
+      "learning_rate": 4.889807370114868e-06,
+      "loss": 0.743,
+      "step": 196
+    },
+    {
+      "epoch": 0.23515368546702478,
+      "grad_norm": 1.2874162250928223,
+      "learning_rate": 4.888391651931879e-06,
+      "loss": 0.7409,
+      "step": 197
+    },
+    {
+      "epoch": 0.23634735899731424,
+      "grad_norm": 1.3626573772815251,
+      "learning_rate": 4.886967104961637e-06,
+      "loss": 0.7002,
+      "step": 198
+    },
+    {
+      "epoch": 0.2375410325276037,
+      "grad_norm": 1.621088987920121,
+      "learning_rate": 4.885533734470039e-06,
+      "loss": 0.772,
+      "step": 199
+    },
+    {
+      "epoch": 0.23873470605789315,
+      "grad_norm": 1.5804682649909156,
+      "learning_rate": 4.884091545755596e-06,
+      "loss": 0.7044,
+      "step": 200
+    },
+    {
+      "epoch": 0.23992837958818264,
+      "grad_norm": 1.4195593038907255,
+      "learning_rate": 4.8826405441494185e-06,
+      "loss": 0.7534,
+      "step": 201
+    },
+    {
+      "epoch": 0.2411220531184721,
+      "grad_norm": 1.9356808705926651,
+      "learning_rate": 4.881180735015193e-06,
+      "loss": 0.7244,
+      "step": 202
+    },
+    {
+      "epoch": 0.24231572664876155,
+      "grad_norm": 1.5708687361924116,
+      "learning_rate": 4.879712123749164e-06,
+      "loss": 0.7388,
+      "step": 203
+    },
+    {
+      "epoch": 0.24350940017905104,
+      "grad_norm": 1.3661682626467346,
+      "learning_rate": 4.878234715780112e-06,
+      "loss": 0.7067,
+      "step": 204
+    },
+    {
+      "epoch": 0.2447030737093405,
+      "grad_norm": 1.251758147953428,
+      "learning_rate": 4.876748516569337e-06,
+      "loss": 0.7255,
+      "step": 205
+    },
+    {
+      "epoch": 0.24589674723962995,
+      "grad_norm": 1.3703942882089086,
+      "learning_rate": 4.875253531610634e-06,
+      "loss": 0.7314,
+      "step": 206
+    },
+    {
+      "epoch": 0.24709042076991944,
+      "grad_norm": 1.461368128735752,
+      "learning_rate": 4.873749766430277e-06,
+      "loss": 0.7777,
+      "step": 207
+    },
+    {
+      "epoch": 0.2482840943002089,
+      "grad_norm": 1.315919212217038,
+      "learning_rate": 4.872237226586995e-06,
+      "loss": 0.7831,
+      "step": 208
+    },
+    {
+      "epoch": 0.24947776783049835,
+      "grad_norm": 1.3208386704868256,
+      "learning_rate": 4.870715917671953e-06,
+      "loss": 0.8244,
+      "step": 209
+    },
+    {
+      "epoch": 0.25067144136078784,
+      "grad_norm": 1.2373714760363816,
+      "learning_rate": 4.869185845308732e-06,
+      "loss": 0.7324,
+      "step": 210
+    },
+    {
+      "epoch": 0.25186511489107727,
+      "grad_norm": 1.4088409747127422,
+      "learning_rate": 4.867647015153306e-06,
+      "loss": 0.7611,
+      "step": 211
+    },
+    {
+      "epoch": 0.25305878842136675,
+      "grad_norm": 1.4077268297947585,
+      "learning_rate": 4.866099432894023e-06,
+      "loss": 0.747,
+      "step": 212
+    },
+    {
+      "epoch": 0.25425246195165624,
+      "grad_norm": 1.4827933791399774,
+      "learning_rate": 4.864543104251587e-06,
+      "loss": 0.7137,
+      "step": 213
+    },
+    {
+      "epoch": 0.25544613548194567,
+      "grad_norm": 1.3508633525171672,
+      "learning_rate": 4.862978034979028e-06,
+      "loss": 0.724,
+      "step": 214
+    },
+    {
+      "epoch": 0.25663980901223515,
+      "grad_norm": 1.465268489212778,
+      "learning_rate": 4.861404230861688e-06,
+      "loss": 0.6241,
+      "step": 215
+    },
+    {
+      "epoch": 0.25783348254252464,
+      "grad_norm": 1.3300886293569927,
+      "learning_rate": 4.8598216977172005e-06,
+      "loss": 0.7014,
+      "step": 216
+    },
+    {
+      "epoch": 0.25902715607281407,
+      "grad_norm": 1.41602797636404,
+      "learning_rate": 4.858230441395462e-06,
+      "loss": 0.7039,
+      "step": 217
+    },
+    {
+      "epoch": 0.26022082960310355,
+      "grad_norm": 1.3664225014511933,
+      "learning_rate": 4.856630467778617e-06,
+      "loss": 0.7305,
+      "step": 218
+    },
+    {
+      "epoch": 0.26141450313339304,
+      "grad_norm": 1.1175176998330205,
+      "learning_rate": 4.855021782781033e-06,
+      "loss": 0.717,
+      "step": 219
+    },
+    {
+      "epoch": 0.26260817666368247,
+      "grad_norm": 1.2713889824942233,
+      "learning_rate": 4.85340439234928e-06,
+      "loss": 0.7671,
+      "step": 220
+    },
+    {
+      "epoch": 0.26380185019397195,
+      "grad_norm": 1.1393102182324073,
+      "learning_rate": 4.851778302462108e-06,
+      "loss": 0.7044,
+      "step": 221
+    },
+    {
+      "epoch": 0.26499552372426144,
+      "grad_norm": 1.3923915333803638,
+      "learning_rate": 4.850143519130424e-06,
+      "loss": 0.6752,
+      "step": 222
+    },
+    {
+      "epoch": 0.26618919725455087,
+      "grad_norm": 1.5055776063455715,
+      "learning_rate": 4.8485000483972724e-06,
+      "loss": 0.7206,
+      "step": 223
+    },
+    {
+      "epoch": 0.26738287078484035,
+      "grad_norm": 1.2346499117786998,
+      "learning_rate": 4.846847896337809e-06,
+      "loss": 0.7367,
+      "step": 224
+    },
+    {
+      "epoch": 0.26857654431512984,
+      "grad_norm": 1.1697769400742972,
+      "learning_rate": 4.8451870690592815e-06,
+      "loss": 0.6837,
+      "step": 225
+    },
+    {
+      "epoch": 0.26977021784541927,
+      "grad_norm": 1.4627787971102764,
+      "learning_rate": 4.843517572701005e-06,
+      "loss": 0.7714,
+      "step": 226
+    },
+    {
+      "epoch": 0.27096389137570875,
+      "grad_norm": 1.461677358159491,
+      "learning_rate": 4.841839413434343e-06,
+      "loss": 0.7164,
+      "step": 227
+    },
+    {
+      "epoch": 0.2721575649059982,
+      "grad_norm": 1.247477301704973,
+      "learning_rate": 4.840152597462679e-06,
+      "loss": 0.7369,
+      "step": 228
+    },
+    {
+      "epoch": 0.27335123843628767,
+      "grad_norm": 1.5335526823557455,
+      "learning_rate": 4.838457131021394e-06,
+      "loss": 0.7414,
+      "step": 229
+    },
+    {
+      "epoch": 0.27454491196657715,
+      "grad_norm": 1.416196356493793,
+      "learning_rate": 4.836753020377854e-06,
+      "loss": 0.7233,
+      "step": 230
+    },
+    {
+      "epoch": 0.2757385854968666,
+      "grad_norm": 1.2402844158110742,
+      "learning_rate": 4.835040271831371e-06,
+      "loss": 0.6921,
+      "step": 231
+    },
+    {
+      "epoch": 0.27693225902715607,
+      "grad_norm": 1.325405626630407,
+      "learning_rate": 4.833318891713189e-06,
+      "loss": 0.7448,
+      "step": 232
+    },
+    {
+      "epoch": 0.27812593255744555,
+      "grad_norm": 1.636164206386521,
+      "learning_rate": 4.8315888863864615e-06,
+      "loss": 0.7257,
+      "step": 233
+    },
+    {
+      "epoch": 0.279319606087735,
+      "grad_norm": 1.294477892152405,
+      "learning_rate": 4.829850262246223e-06,
+      "loss": 0.7275,
+      "step": 234
+    },
+    {
+      "epoch": 0.28051327961802447,
+      "grad_norm": 1.2534851560902933,
+      "learning_rate": 4.828103025719368e-06,
+      "loss": 0.6744,
+      "step": 235
+    },
+    {
+      "epoch": 0.28170695314831395,
+      "grad_norm": 1.3361743741123135,
+      "learning_rate": 4.8263471832646305e-06,
+      "loss": 0.7542,
+      "step": 236
+    },
+    {
+      "epoch": 0.2829006266786034,
+      "grad_norm": 1.7794879323390358,
+      "learning_rate": 4.824582741372551e-06,
+      "loss": 0.6397,
+      "step": 237
+    },
+    {
+      "epoch": 0.28409430020889287,
+      "grad_norm": 1.3658199624392589,
+      "learning_rate": 4.8228097065654615e-06,
+      "loss": 0.7149,
+      "step": 238
+    },
+    {
+      "epoch": 0.28528797373918235,
+      "grad_norm": 1.3072375367316251,
+      "learning_rate": 4.821028085397458e-06,
+      "loss": 0.6894,
+      "step": 239
+    },
+    {
+      "epoch": 0.2864816472694718,
+      "grad_norm": 1.6554740415887155,
+      "learning_rate": 4.819237884454375e-06,
+      "loss": 0.6779,
+      "step": 240
+    },
+    {
+      "epoch": 0.28767532079976127,
+      "grad_norm": 1.3979259089374587,
+      "learning_rate": 4.8174391103537655e-06,
+      "loss": 0.6782,
+      "step": 241
+    },
+    {
+      "epoch": 0.28886899433005075,
+      "grad_norm": 1.2115058986142737,
+      "learning_rate": 4.815631769744869e-06,
+      "loss": 0.7638,
+      "step": 242
+    },
+    {
+      "epoch": 0.2900626678603402,
+      "grad_norm": 1.306426889473089,
+      "learning_rate": 4.8138158693085955e-06,
+      "loss": 0.7819,
+      "step": 243
+    },
+    {
+      "epoch": 0.29125634139062967,
+      "grad_norm": 1.3589954968814129,
+      "learning_rate": 4.8119914157574945e-06,
+      "loss": 0.7585,
+      "step": 244
+    },
+    {
+      "epoch": 0.29245001492091915,
+      "grad_norm": 1.3781497816322819,
+      "learning_rate": 4.810158415835733e-06,
+      "loss": 0.7605,
+      "step": 245
+    },
+    {
+      "epoch": 0.2936436884512086,
+      "grad_norm": 1.2584577307454725,
+      "learning_rate": 4.8083168763190705e-06,
+      "loss": 0.6997,
+      "step": 246
+    },
+    {
+      "epoch": 0.29483736198149807,
+      "grad_norm": 1.6933326451549058,
+      "learning_rate": 4.8064668040148315e-06,
+      "loss": 0.7097,
+      "step": 247
+    },
+    {
+      "epoch": 0.2960310355117875,
+      "grad_norm": 1.6252538060447355,
+      "learning_rate": 4.8046082057618855e-06,
+      "loss": 0.6976,
+      "step": 248
+    },
+    {
+      "epoch": 0.297224709042077,
+      "grad_norm": 1.2648878328803557,
+      "learning_rate": 4.802741088430616e-06,
+      "loss": 0.6786,
+      "step": 249
+    },
+    {
+      "epoch": 0.29841838257236647,
+      "grad_norm": 1.291377081492189,
+      "learning_rate": 4.800865458922899e-06,
+      "loss": 0.6653,
+      "step": 250
+    },
+    {
+      "epoch": 0.2996120561026559,
+      "grad_norm": 1.2451565458636964,
+      "learning_rate": 4.798981324172075e-06,
+      "loss": 0.6818,
+      "step": 251
+    },
+    {
+      "epoch": 0.3008057296329454,
+      "grad_norm": 1.3859233273705183,
+      "learning_rate": 4.797088691142926e-06,
+      "loss": 0.6952,
+      "step": 252
+    },
+    {
+      "epoch": 0.30199940316323487,
+      "grad_norm": 1.1554071712447842,
+      "learning_rate": 4.795187566831645e-06,
+      "loss": 0.6942,
+      "step": 253
+    },
+    {
+      "epoch": 0.3031930766935243,
+      "grad_norm": 1.4487233844413288,
+      "learning_rate": 4.793277958265819e-06,
+      "loss": 0.7843,
+      "step": 254
+    },
+    {
+      "epoch": 0.3043867502238138,
+      "grad_norm": 1.2016328771893767,
+      "learning_rate": 4.79135987250439e-06,
+      "loss": 0.6929,
+      "step": 255
+    },
+    {
+      "epoch": 0.30558042375410327,
+      "grad_norm": 1.3029418364965668,
+      "learning_rate": 4.789433316637644e-06,
+      "loss": 0.7128,
+      "step": 256
+    },
+    {
+      "epoch": 0.3067740972843927,
+      "grad_norm": 1.2067002392705943,
+      "learning_rate": 4.787498297787172e-06,
+      "loss": 0.6715,
+      "step": 257
+    },
+    {
+      "epoch": 0.3079677708146822,
+      "grad_norm": 1.2825737988071968,
+      "learning_rate": 4.785554823105849e-06,
+      "loss": 0.759,
+      "step": 258
+    },
+    {
+      "epoch": 0.30916144434497167,
+      "grad_norm": 1.405657134321189,
+      "learning_rate": 4.783602899777811e-06,
+      "loss": 0.7039,
+      "step": 259
+    },
+    {
+      "epoch": 0.3103551178752611,
+      "grad_norm": 1.1939358415349064,
+      "learning_rate": 4.7816425350184195e-06,
+      "loss": 0.7093,
+      "step": 260
+    },
+    {
+      "epoch": 0.3115487914055506,
+      "grad_norm": 1.3894017831081242,
+      "learning_rate": 4.779673736074245e-06,
+      "loss": 0.6904,
+      "step": 261
+    },
+    {
+      "epoch": 0.31274246493584007,
+      "grad_norm": 1.2508482757860593,
+      "learning_rate": 4.7776965102230325e-06,
+      "loss": 0.6636,
+      "step": 262
+    },
+    {
+      "epoch": 0.3139361384661295,
+      "grad_norm": 1.2118929615337841,
+      "learning_rate": 4.775710864773677e-06,
+      "loss": 0.6291,
+      "step": 263
+    },
+    {
+      "epoch": 0.315129811996419,
+      "grad_norm": 1.154157208619186,
+      "learning_rate": 4.7737168070662e-06,
+      "loss": 0.6878,
+      "step": 264
+    },
+    {
+      "epoch": 0.31632348552670847,
+      "grad_norm": 1.4600719790063776,
+      "learning_rate": 4.771714344471716e-06,
+      "loss": 0.7356,
+      "step": 265
+    },
+    {
+      "epoch": 0.3175171590569979,
+      "grad_norm": 1.225295091264663,
+      "learning_rate": 4.769703484392411e-06,
+      "loss": 0.6846,
+      "step": 266
+    },
+    {
+      "epoch": 0.3187108325872874,
+      "grad_norm": 1.1751238538711366,
+      "learning_rate": 4.767684234261509e-06,
+      "loss": 0.6645,
+      "step": 267
+    },
+    {
+      "epoch": 0.31990450611757687,
+      "grad_norm": 1.3442391337993869,
+      "learning_rate": 4.765656601543252e-06,
+      "loss": 0.6808,
+      "step": 268
+    },
+    {
+      "epoch": 0.3210981796478663,
+      "grad_norm": 1.1916211123088165,
+      "learning_rate": 4.763620593732867e-06,
+      "loss": 0.7086,
+      "step": 269
+    },
+    {
+      "epoch": 0.3222918531781558,
+      "grad_norm": 1.3687179869347033,
+      "learning_rate": 4.76157621835654e-06,
+      "loss": 0.69,
+      "step": 270
+    },
+    {
+      "epoch": 0.3234855267084452,
+      "grad_norm": 1.5140697413879123,
+      "learning_rate": 4.759523482971389e-06,
+      "loss": 0.7073,
+      "step": 271
+    },
+    {
+      "epoch": 0.3246792002387347,
+      "grad_norm": 1.135600038359386,
+      "learning_rate": 4.757462395165432e-06,
+      "loss": 0.6983,
+      "step": 272
+    },
+    {
+      "epoch": 0.3258728737690242,
+      "grad_norm": 1.6760735732966718,
+      "learning_rate": 4.755392962557565e-06,
+      "loss": 0.6473,
+      "step": 273
+    },
+    {
+      "epoch": 0.3270665472993136,
+      "grad_norm": 1.7254698749814177,
+      "learning_rate": 4.753315192797531e-06,
+      "loss": 0.6899,
+      "step": 274
+    },
+    {
+      "epoch": 0.3282602208296031,
+      "grad_norm": 1.3969063976705576,
+      "learning_rate": 4.751229093565891e-06,
+      "loss": 0.7048,
+      "step": 275
+    },
+    {
+      "epoch": 0.3294538943598926,
+      "grad_norm": 1.2932215683409531,
+      "learning_rate": 4.749134672573995e-06,
+      "loss": 0.6992,
+      "step": 276
+    },
+    {
+      "epoch": 0.330647567890182,
+      "grad_norm": 1.4225599138183258,
+      "learning_rate": 4.747031937563956e-06,
+      "loss": 0.6946,
+      "step": 277
+    },
+    {
+      "epoch": 0.3318412414204715,
+      "grad_norm": 1.4712564671318273,
+      "learning_rate": 4.74492089630862e-06,
+      "loss": 0.7052,
+      "step": 278
+    },
+    {
+      "epoch": 0.333034914950761,
+      "grad_norm": 1.3823801943977407,
+      "learning_rate": 4.7428015566115365e-06,
+      "loss": 0.6771,
+      "step": 279
+    },
+    {
+      "epoch": 0.333034914950761,
+      "eval_loss": 0.6846477389335632,
+      "eval_runtime": 412.9026,
+      "eval_samples_per_second": 26.854,
+      "eval_steps_per_second": 13.427,
+      "step": 279
+    },
+    {
+      "epoch": 0.3342285884810504,
+      "grad_norm": 1.3371211095928401,
+      "learning_rate": 4.7406739263069316e-06,
+      "loss": 0.6376,
+      "step": 280
+    },
+    {
+      "epoch": 0.3354222620113399,
+      "grad_norm": 1.5260384792192485,
+      "learning_rate": 4.7385380132596785e-06,
+      "loss": 0.6634,
+      "step": 281
+    },
+    {
+      "epoch": 0.3366159355416294,
+      "grad_norm": 1.3406401765758174,
+      "learning_rate": 4.736393825365265e-06,
+      "loss": 0.6478,
+      "step": 282
+    },
+    {
+      "epoch": 0.3378096090719188,
+      "grad_norm": 1.4024574369692429,
+      "learning_rate": 4.734241370549771e-06,
+      "loss": 0.6762,
+      "step": 283
+    },
+    {
+      "epoch": 0.3390032826022083,
+      "grad_norm": 1.3508634062720946,
+      "learning_rate": 4.732080656769833e-06,
+      "loss": 0.691,
+      "step": 284
+    },
+    {
+      "epoch": 0.3401969561324978,
+      "grad_norm": 1.3448634784208517,
+      "learning_rate": 4.729911692012617e-06,
+      "loss": 0.6807,
+      "step": 285
+    },
+    {
+      "epoch": 0.3413906296627872,
+      "grad_norm": 1.2847227723152743,
+      "learning_rate": 4.727734484295791e-06,
+      "loss": 0.6949,
+      "step": 286
+    },
+    {
+      "epoch": 0.3425843031930767,
+      "grad_norm": 1.488731080042999,
+      "learning_rate": 4.725549041667492e-06,
+      "loss": 0.7249,
+      "step": 287
+    },
+    {
+      "epoch": 0.3437779767233662,
+      "grad_norm": 1.2793855202844717,
+      "learning_rate": 4.723355372206297e-06,
+      "loss": 0.6776,
+      "step": 288
+    },
+    {
+      "epoch": 0.3449716502536556,
+      "grad_norm": 1.362290881047155,
+      "learning_rate": 4.721153484021196e-06,
+      "loss": 0.6896,
+      "step": 289
+    },
+    {
+      "epoch": 0.3461653237839451,
+      "grad_norm": 1.1578966118704863,
+      "learning_rate": 4.718943385251558e-06,
+      "loss": 0.6374,
+      "step": 290
+    },
+    {
+      "epoch": 0.34735899731423453,
+      "grad_norm": 1.3438024523340593,
+      "learning_rate": 4.716725084067102e-06,
+      "loss": 0.655,
+      "step": 291
+    },
+    {
+      "epoch": 0.348552670844524,
+      "grad_norm": 1.3312081355869356,
+      "learning_rate": 4.714498588667871e-06,
+      "loss": 0.6728,
+      "step": 292
+    },
+    {
+      "epoch": 0.3497463443748135,
+      "grad_norm": 1.211234849537357,
+      "learning_rate": 4.712263907284195e-06,
+      "loss": 0.6946,
+      "step": 293
+    },
+    {
+      "epoch": 0.35094001790510293,
+      "grad_norm": 1.285835930408837,
+      "learning_rate": 4.710021048176665e-06,
+      "loss": 0.6584,
+      "step": 294
+    },
+    {
+      "epoch": 0.3521336914353924,
+      "grad_norm": 1.1822973184872547,
+      "learning_rate": 4.707770019636101e-06,
+      "loss": 0.6994,
+      "step": 295
+    },
+    {
+      "epoch": 0.3533273649656819,
+      "grad_norm": 1.312513533249853,
+      "learning_rate": 4.7055108299835224e-06,
+      "loss": 0.6707,
+      "step": 296
+    },
+    {
+      "epoch": 0.35452103849597133,
+      "grad_norm": 1.3800556921412532,
+      "learning_rate": 4.7032434875701176e-06,
+      "loss": 0.7198,
+      "step": 297
+    },
+    {
+      "epoch": 0.3557147120262608,
+      "grad_norm": 1.3436446527064017,
+      "learning_rate": 4.700968000777207e-06,
+      "loss": 0.6852,
+      "step": 298
+    },
+    {
+      "epoch": 0.3569083855565503,
+      "grad_norm": 1.3083999180241308,
+      "learning_rate": 4.698684378016223e-06,
+      "loss": 0.7055,
+      "step": 299
+    },
+    {
+      "epoch": 0.35810205908683973,
+      "grad_norm": 1.4093191802031717,
+      "learning_rate": 4.6963926277286695e-06,
+      "loss": 0.6408,
+      "step": 300
+    },
+    {
+      "epoch": 0.3592957326171292,
+      "grad_norm": 1.3201210764453963,
+      "learning_rate": 4.694092758386095e-06,
+      "loss": 0.6683,
+      "step": 301
+    },
+    {
+      "epoch": 0.3604894061474187,
+      "grad_norm": 1.4917658289317903,
+      "learning_rate": 4.69178477849006e-06,
+      "loss": 0.6616,
+      "step": 302
+    },
+    {
+      "epoch": 0.36168307967770813,
+      "grad_norm": 1.2202456890838416,
+      "learning_rate": 4.689468696572108e-06,
+      "loss": 0.6135,
+      "step": 303
+    },
+    {
+      "epoch": 0.3628767532079976,
+      "grad_norm": 1.8840891333384444,
+      "learning_rate": 4.687144521193729e-06,
+      "loss": 0.6383,
+      "step": 304
+    },
+    {
+      "epoch": 0.3640704267382871,
+      "grad_norm": 1.5404380338601549,
+      "learning_rate": 4.684812260946331e-06,
+      "loss": 0.6271,
+      "step": 305
+    },
+    {
+      "epoch": 0.36526410026857653,
+      "grad_norm": 1.5375462640413233,
+      "learning_rate": 4.682471924451212e-06,
+      "loss": 0.7098,
+      "step": 306
+    },
+    {
+      "epoch": 0.366457773798866,
+      "grad_norm": 1.4972752667736051,
+      "learning_rate": 4.68012352035952e-06,
+      "loss": 0.6788,
+      "step": 307
+    },
+    {
+      "epoch": 0.3676514473291555,
+      "grad_norm": 1.3986877871483208,
+      "learning_rate": 4.677767057352226e-06,
+      "loss": 0.7058,
+      "step": 308
+    },
+    {
+      "epoch": 0.36884512085944493,
+      "grad_norm": 1.5735298046244075,
+      "learning_rate": 4.675402544140093e-06,
+      "loss": 0.6266,
+      "step": 309
+    },
+    {
+      "epoch": 0.3700387943897344,
+      "grad_norm": 1.424858174397913,
+      "learning_rate": 4.673029989463639e-06,
+      "loss": 0.689,
+      "step": 310
+    },
+    {
+      "epoch": 0.3712324679200239,
+      "grad_norm": 1.422563152472999,
+      "learning_rate": 4.670649402093112e-06,
+      "loss": 0.687,
+      "step": 311
+    },
+    {
+      "epoch": 0.37242614145031333,
+      "grad_norm": 1.2227545404530318,
+      "learning_rate": 4.6682607908284475e-06,
+      "loss": 0.6504,
+      "step": 312
+    },
+    {
+      "epoch": 0.3736198149806028,
+      "grad_norm": 1.2601523472303733,
+      "learning_rate": 4.665864164499245e-06,
+      "loss": 0.6418,
+      "step": 313
+    },
+    {
+      "epoch": 0.37481348851089225,
+      "grad_norm": 1.2234768466153,
+      "learning_rate": 4.6634595319647334e-06,
+      "loss": 0.6118,
+      "step": 314
+    },
+    {
+      "epoch": 0.37600716204118173,
+      "grad_norm": 1.4660753549186738,
+      "learning_rate": 4.661046902113734e-06,
+      "loss": 0.6375,
+      "step": 315
+    },
+    {
+      "epoch": 0.3772008355714712,
+      "grad_norm": 1.2990252600909762,
+      "learning_rate": 4.658626283864631e-06,
+      "loss": 0.6934,
+      "step": 316
+    },
+    {
+      "epoch": 0.37839450910176065,
+      "grad_norm": 1.214170689120582,
+      "learning_rate": 4.656197686165339e-06,
+      "loss": 0.6842,
+      "step": 317
+    },
+    {
+      "epoch": 0.37958818263205013,
+      "grad_norm": 1.1648083720499496,
+      "learning_rate": 4.653761117993269e-06,
+      "loss": 0.6221,
+      "step": 318
+    },
+    {
+      "epoch": 0.3807818561623396,
+      "grad_norm": 2.0374447610747493,
+      "learning_rate": 4.651316588355294e-06,
+      "loss": 0.765,
+      "step": 319
+    },
+    {
+      "epoch": 0.38197552969262905,
+      "grad_norm": 1.2913848875836433,
+      "learning_rate": 4.648864106287718e-06,
+      "loss": 0.6742,
+      "step": 320
+    },
+    {
+      "epoch": 0.38316920322291853,
+      "grad_norm": 1.3826403559076854,
+      "learning_rate": 4.646403680856241e-06,
+      "loss": 0.6509,
+      "step": 321
+    },
+    {
+      "epoch": 0.384362876753208,
+      "grad_norm": 1.391962878825795,
+      "learning_rate": 4.6439353211559255e-06,
+      "loss": 0.626,
+      "step": 322
+    },
+    {
+      "epoch": 0.38555655028349745,
+      "grad_norm": 1.2366810288776844,
+      "learning_rate": 4.641459036311166e-06,
+      "loss": 0.6482,
+      "step": 323
+    },
+    {
+      "epoch": 0.38675022381378693,
+      "grad_norm": 1.1608380993062588,
+      "learning_rate": 4.638974835475647e-06,
+      "loss": 0.7045,
+      "step": 324
+    },
+    {
+      "epoch": 0.3879438973440764,
+      "grad_norm": 1.681395154022434,
+      "learning_rate": 4.636482727832322e-06,
+      "loss": 0.6485,
+      "step": 325
+    },
+    {
+      "epoch": 0.38913757087436585,
+      "grad_norm": 1.5499853764668938,
+      "learning_rate": 4.633982722593367e-06,
+      "loss": 0.6341,
+      "step": 326
+    },
+    {
+      "epoch": 0.39033124440465533,
+      "grad_norm": 1.3506158071666676,
+      "learning_rate": 4.631474829000152e-06,
+      "loss": 0.6152,
+      "step": 327
+    },
+    {
+      "epoch": 0.3915249179349448,
+      "grad_norm": 1.32895937053141,
+      "learning_rate": 4.628959056323211e-06,
+      "loss": 0.7004,
+      "step": 328
+    },
+    {
+      "epoch": 0.39271859146523425,
+      "grad_norm": 1.241985846489561,
+      "learning_rate": 4.626435413862198e-06,
+      "loss": 0.6713,
+      "step": 329
+    },
+    {
+      "epoch": 0.39391226499552373,
+      "grad_norm": 1.3605066292508516,
+      "learning_rate": 4.623903910945861e-06,
+      "loss": 0.6243,
+      "step": 330
+    },
+    {
+      "epoch": 0.3951059385258132,
+      "grad_norm": 1.5681694169794416,
+      "learning_rate": 4.621364556932005e-06,
+      "loss": 0.6273,
+      "step": 331
+    },
+    {
+      "epoch": 0.39629961205610265,
+      "grad_norm": 1.4092893288151016,
+      "learning_rate": 4.618817361207455e-06,
+      "loss": 0.6649,
+      "step": 332
+    },
+    {
+      "epoch": 0.39749328558639213,
+      "grad_norm": 1.223446383808155,
+      "learning_rate": 4.616262333188023e-06,
+      "loss": 0.6036,
+      "step": 333
+    },
+    {
+      "epoch": 0.39868695911668156,
+      "grad_norm": 1.2916912487028882,
+      "learning_rate": 4.613699482318476e-06,
+      "loss": 0.5678,
+      "step": 334
+    },
+    {
+      "epoch": 0.39988063264697105,
+      "grad_norm": 1.2168147001895975,
+      "learning_rate": 4.611128818072496e-06,
+      "loss": 0.6649,
+      "step": 335
+    },
+    {
+      "epoch": 0.40107430617726053,
+      "grad_norm": 1.3679248232676404,
+      "learning_rate": 4.608550349952648e-06,
+      "loss": 0.6122,
+      "step": 336
+    },
+    {
+      "epoch": 0.40226797970754996,
+      "grad_norm": 1.5932039919961867,
+      "learning_rate": 4.605964087490346e-06,
+      "loss": 0.6927,
+      "step": 337
+    },
+    {
+      "epoch": 0.40346165323783945,
+      "grad_norm": 1.303172463186514,
+      "learning_rate": 4.603370040245814e-06,
+      "loss": 0.6052,
+      "step": 338
+    },
+    {
+      "epoch": 0.40465532676812893,
+      "grad_norm": 1.284260746698017,
+      "learning_rate": 4.600768217808053e-06,
+      "loss": 0.616,
+      "step": 339
+    },
+    {
+      "epoch": 0.40584900029841836,
+      "grad_norm": 1.2715127048886254,
+      "learning_rate": 4.598158629794807e-06,
+      "loss": 0.641,
+      "step": 340
+    },
+    {
+      "epoch": 0.40704267382870785,
+      "grad_norm": 1.3727093879029966,
+      "learning_rate": 4.5955412858525234e-06,
+      "loss": 0.6575,
+      "step": 341
+    },
+    {
+      "epoch": 0.40823634735899733,
+      "grad_norm": 1.3972056862082822,
+      "learning_rate": 4.592916195656322e-06,
+      "loss": 0.6202,
+      "step": 342
+    },
+    {
+      "epoch": 0.40943002088928676,
+      "grad_norm": 1.226422767344469,
+      "learning_rate": 4.590283368909955e-06,
+      "loss": 0.6373,
+      "step": 343
+    },
+    {
+      "epoch": 0.41062369441957625,
+      "grad_norm": 1.2214362910423493,
+      "learning_rate": 4.587642815345775e-06,
+      "loss": 0.6513,
+      "step": 344
+    },
+    {
+      "epoch": 0.41181736794986573,
+      "grad_norm": 1.2724022723275898,
+      "learning_rate": 4.584994544724695e-06,
+      "loss": 0.6435,
+      "step": 345
+    },
+    {
+      "epoch": 0.41301104148015516,
+      "grad_norm": 1.90952619576874,
+      "learning_rate": 4.582338566836157e-06,
+      "loss": 0.6323,
+      "step": 346
+    },
+    {
+      "epoch": 0.41420471501044465,
+      "grad_norm": 1.2579926244712305,
+      "learning_rate": 4.579674891498089e-06,
+      "loss": 0.6446,
+      "step": 347
+    },
+    {
+      "epoch": 0.41539838854073413,
+      "grad_norm": 1.382956229673369,
+      "learning_rate": 4.577003528556878e-06,
+      "loss": 0.6749,
+      "step": 348
+    },
+    {
+      "epoch": 0.41659206207102356,
+      "grad_norm": 1.383601549667179,
+      "learning_rate": 4.574324487887326e-06,
+      "loss": 0.6641,
+      "step": 349
+    },
+    {
+      "epoch": 0.41778573560131305,
+      "grad_norm": 1.1622294774191468,
+      "learning_rate": 4.571637779392615e-06,
+      "loss": 0.636,
+      "step": 350
+    },
+    {
+      "epoch": 0.41897940913160253,
+      "grad_norm": 1.428456878926952,
+      "learning_rate": 4.568943413004274e-06,
+      "loss": 0.6163,
+      "step": 351
+    },
+    {
+      "epoch": 0.42017308266189196,
+      "grad_norm": 1.4910910058961493,
+      "learning_rate": 4.566241398682138e-06,
+      "loss": 0.6447,
+      "step": 352
+    },
+    {
+      "epoch": 0.42136675619218145,
+      "grad_norm": 1.3541150547126044,
+      "learning_rate": 4.563531746414311e-06,
+      "loss": 0.6351,
+      "step": 353
+    },
+    {
+      "epoch": 0.42256042972247093,
+      "grad_norm": 1.4205816255695827,
+      "learning_rate": 4.560814466217137e-06,
+      "loss": 0.7153,
+      "step": 354
+    },
+    {
+      "epoch": 0.42375410325276036,
+      "grad_norm": 1.4948619172954853,
+      "learning_rate": 4.558089568135151e-06,
+      "loss": 0.6028,
+      "step": 355
+    },
+    {
+      "epoch": 0.42494777678304985,
+      "grad_norm": 1.3766011505260616,
+      "learning_rate": 4.55535706224105e-06,
+      "loss": 0.5708,
+      "step": 356
+    },
+    {
+      "epoch": 0.4261414503133393,
+      "grad_norm": 1.371263391588093,
+      "learning_rate": 4.5526169586356535e-06,
+      "loss": 0.626,
+      "step": 357
+    },
+    {
+      "epoch": 0.42733512384362876,
+      "grad_norm": 1.472168514490173,
+      "learning_rate": 4.5498692674478685e-06,
+      "loss": 0.6118,
+      "step": 358
+    },
+    {
+      "epoch": 0.42852879737391825,
+      "grad_norm": 1.3555505111118311,
+      "learning_rate": 4.5471139988346445e-06,
+      "loss": 0.6824,
+      "step": 359
+    },
+    {
+      "epoch": 0.4297224709042077,
+      "grad_norm": 1.1158326109881502,
+      "learning_rate": 4.544351162980947e-06,
+      "loss": 0.6629,
+      "step": 360
+    },
+    {
+      "epoch": 0.43091614443449716,
+      "grad_norm": 1.4225920267037269,
+      "learning_rate": 4.541580770099709e-06,
+      "loss": 0.7106,
+      "step": 361
+    },
+    {
+      "epoch": 0.43210981796478665,
+      "grad_norm": 1.519185463886708,
+      "learning_rate": 4.538802830431803e-06,
+      "loss": 0.6279,
+      "step": 362
+    },
+    {
+      "epoch": 0.4333034914950761,
+      "grad_norm": 1.2133200333323961,
+      "learning_rate": 4.5360173542459965e-06,
+      "loss": 0.692,
+      "step": 363
+    },
+    {
+      "epoch": 0.43449716502536556,
+      "grad_norm": 1.3310636716566095,
+      "learning_rate": 4.533224351838914e-06,
+      "loss": 0.6554,
+      "step": 364
+    },
+    {
+      "epoch": 0.43569083855565505,
+      "grad_norm": 1.7294024871950497,
+      "learning_rate": 4.530423833535004e-06,
+      "loss": 0.5876,
+      "step": 365
+    },
+    {
+      "epoch": 0.4368845120859445,
+      "grad_norm": 2.0358887652438677,
+      "learning_rate": 4.527615809686498e-06,
+      "loss": 0.6288,
+      "step": 366
+    },
+    {
+      "epoch": 0.43807818561623396,
+      "grad_norm": 1.213647251606033,
+      "learning_rate": 4.524800290673369e-06,
+      "loss": 0.6305,
+      "step": 367
+    },
+    {
+      "epoch": 0.43927185914652345,
+      "grad_norm": 1.501478488863323,
+      "learning_rate": 4.521977286903299e-06,
+      "loss": 0.6929,
+      "step": 368
+    },
+    {
+      "epoch": 0.4404655326768129,
+      "grad_norm": 1.463837293971756,
+      "learning_rate": 4.5191468088116355e-06,
+      "loss": 0.6421,
+      "step": 369
+    },
+    {
+      "epoch": 0.44165920620710236,
+      "grad_norm": 1.7942748893980314,
+      "learning_rate": 4.516308866861357e-06,
+      "loss": 0.6246,
+      "step": 370
+    },
+    {
+      "epoch": 0.44285287973739185,
+      "grad_norm": 1.2126836888579313,
+      "learning_rate": 4.513463471543032e-06,
+      "loss": 0.6348,
+      "step": 371
+    },
+    {
+      "epoch": 0.4440465532676813,
+      "grad_norm": 1.5786268577595093,
+      "learning_rate": 4.510610633374781e-06,
+      "loss": 0.6345,
+      "step": 372
+    },
+    {
+      "epoch": 0.44524022679797076,
+      "grad_norm": 1.8316828103496652,
+      "learning_rate": 4.507750362902236e-06,
+      "loss": 0.6511,
+      "step": 373
+    },
+    {
+      "epoch": 0.44643390032826025,
+      "grad_norm": 1.2827659220880676,
+      "learning_rate": 4.504882670698504e-06,
+      "loss": 0.6105,
+      "step": 374
+    },
+    {
+      "epoch": 0.4476275738585497,
+      "grad_norm": 1.251328046029592,
+      "learning_rate": 4.502007567364126e-06,
+      "loss": 0.6489,
+      "step": 375
+    },
+    {
+      "epoch": 0.44882124738883916,
+      "grad_norm": 1.466361287788854,
+      "learning_rate": 4.499125063527038e-06,
+      "loss": 0.6257,
+      "step": 376
+    },
+    {
+      "epoch": 0.4500149209191286,
+      "grad_norm": 1.6779457947298428,
+      "learning_rate": 4.496235169842535e-06,
+      "loss": 0.5505,
+      "step": 377
+    },
+    {
+      "epoch": 0.4512085944494181,
+      "grad_norm": 1.2259448186587079,
+      "learning_rate": 4.493337896993226e-06,
+      "loss": 0.6456,
+      "step": 378
+    },
+    {
+      "epoch": 0.45240226797970756,
+      "grad_norm": 1.2873284134809548,
+      "learning_rate": 4.490433255688998e-06,
+      "loss": 0.675,
+      "step": 379
+    },
+    {
+      "epoch": 0.453595941509997,
+      "grad_norm": 1.5992223985195466,
+      "learning_rate": 4.4875212566669764e-06,
+      "loss": 0.6342,
+      "step": 380
+    },
+    {
+      "epoch": 0.4547896150402865,
+      "grad_norm": 1.3955795063874046,
+      "learning_rate": 4.484601910691484e-06,
+      "loss": 0.5658,
+      "step": 381
+    },
+    {
+      "epoch": 0.45598328857057596,
+      "grad_norm": 1.3171512803658922,
+      "learning_rate": 4.481675228554005e-06,
+      "loss": 0.618,
+      "step": 382
+    },
+    {
+      "epoch": 0.4571769621008654,
+      "grad_norm": 1.368596334205501,
+      "learning_rate": 4.478741221073136e-06,
+      "loss": 0.7001,
+      "step": 383
+    },
+    {
+      "epoch": 0.4583706356311549,
+      "grad_norm": 1.496955355802238,
+      "learning_rate": 4.475799899094557e-06,
+      "loss": 0.6279,
+      "step": 384
+    },
+    {
+      "epoch": 0.45956430916144436,
+      "grad_norm": 1.3311940792099912,
+      "learning_rate": 4.472851273490985e-06,
+      "loss": 0.6454,
+      "step": 385
+    },
+    {
+      "epoch": 0.4607579826917338,
+      "grad_norm": 1.4781859135515831,
+      "learning_rate": 4.4698953551621335e-06,
+      "loss": 0.6582,
+      "step": 386
+    },
+    {
+      "epoch": 0.4619516562220233,
+      "grad_norm": 1.2467840318252372,
+      "learning_rate": 4.466932155034677e-06,
+      "loss": 0.6245,
+      "step": 387
+    },
+    {
+      "epoch": 0.46314532975231276,
+      "grad_norm": 1.3165106445001906,
+      "learning_rate": 4.4639616840622055e-06,
+      "loss": 0.6133,
+      "step": 388
+    },
+    {
+      "epoch": 0.4643390032826022,
+      "grad_norm": 1.3734383781343862,
+      "learning_rate": 4.460983953225185e-06,
+      "loss": 0.6335,
+      "step": 389
+    },
+    {
+      "epoch": 0.4655326768128917,
+      "grad_norm": 1.2352789136351765,
+      "learning_rate": 4.457998973530921e-06,
+      "loss": 0.6234,
+      "step": 390
+    },
+    {
+      "epoch": 0.46672635034318116,
+      "grad_norm": 1.3013245788706025,
+      "learning_rate": 4.455006756013511e-06,
+      "loss": 0.655,
+      "step": 391
+    },
+    {
+      "epoch": 0.4679200238734706,
+      "grad_norm": 1.306859336801617,
+      "learning_rate": 4.45200731173381e-06,
+      "loss": 0.6748,
+      "step": 392
+    },
+    {
+      "epoch": 0.4691136974037601,
+      "grad_norm": 1.3547776286359348,
+      "learning_rate": 4.449000651779386e-06,
+      "loss": 0.6288,
+      "step": 393
+    },
+    {
+      "epoch": 0.47030737093404956,
+      "grad_norm": 1.2770604039839548,
+      "learning_rate": 4.445986787264482e-06,
+      "loss": 0.599,
+      "step": 394
+    },
+    {
+      "epoch": 0.471501044464339,
+      "grad_norm": 1.29086918908413,
+      "learning_rate": 4.442965729329968e-06,
+      "loss": 0.6539,
+      "step": 395
+    },
+    {
+      "epoch": 0.4726947179946285,
+      "grad_norm": 1.1098582590328354,
+      "learning_rate": 4.43993748914331e-06,
+      "loss": 0.6321,
+      "step": 396
+    },
+    {
+      "epoch": 0.47388839152491796,
+      "grad_norm": 1.3633067255021907,
+      "learning_rate": 4.436902077898522e-06,
+      "loss": 0.604,
+      "step": 397
+    },
+    {
+      "epoch": 0.4750820650552074,
+      "grad_norm": 1.36600583395753,
+      "learning_rate": 4.433859506816124e-06,
+      "loss": 0.7084,
+      "step": 398
+    },
+    {
+      "epoch": 0.4762757385854969,
+      "grad_norm": 1.293392283768407,
+      "learning_rate": 4.430809787143105e-06,
+      "loss": 0.593,
+      "step": 399
+    },
+    {
+      "epoch": 0.4774694121157863,
+      "grad_norm": 1.5157608273336003,
+      "learning_rate": 4.427752930152879e-06,
+      "loss": 0.6764,
+      "step": 400
+    },
+    {
+      "epoch": 0.4786630856460758,
+      "grad_norm": 1.488324378469982,
+      "learning_rate": 4.424688947145241e-06,
+      "loss": 0.5455,
+      "step": 401
+    },
+    {
+      "epoch": 0.4798567591763653,
+      "grad_norm": 1.302352660679449,
+      "learning_rate": 4.4216178494463305e-06,
+      "loss": 0.6464,
+      "step": 402
+    },
+    {
+      "epoch": 0.4810504327066547,
+      "grad_norm": 1.2124487454708892,
+      "learning_rate": 4.418539648408585e-06,
+      "loss": 0.6263,
+      "step": 403
+    },
+    {
+      "epoch": 0.4822441062369442,
+      "grad_norm": 1.7100013098870857,
+      "learning_rate": 4.415454355410701e-06,
+      "loss": 0.6308,
+      "step": 404
+    },
+    {
+      "epoch": 0.4834377797672337,
+      "grad_norm": 1.429362592378676,
+      "learning_rate": 4.41236198185759e-06,
+      "loss": 0.6272,
+      "step": 405
+    },
+    {
+      "epoch": 0.4846314532975231,
+      "grad_norm": 1.468813200306582,
+      "learning_rate": 4.409262539180337e-06,
+      "loss": 0.6665,
+      "step": 406
+    },
+    {
+      "epoch": 0.4858251268278126,
+      "grad_norm": 2.0737362408199487,
+      "learning_rate": 4.4061560388361564e-06,
+      "loss": 0.615,
+      "step": 407
+    },
+    {
+      "epoch": 0.4870188003581021,
+      "grad_norm": 1.367681046177691,
+      "learning_rate": 4.403042492308357e-06,
+      "loss": 0.5975,
+      "step": 408
+    },
+    {
+      "epoch": 0.4882124738883915,
+      "grad_norm": 1.3930985977006256,
+      "learning_rate": 4.399921911106288e-06,
+      "loss": 0.6478,
+      "step": 409
+    },
+    {
+      "epoch": 0.489406147418681,
+      "grad_norm": 1.1931289592139445,
+      "learning_rate": 4.396794306765306e-06,
+      "loss": 0.6683,
+      "step": 410
+    },
+    {
+      "epoch": 0.4905998209489705,
+      "grad_norm": 1.4270063255266063,
+      "learning_rate": 4.393659690846729e-06,
+      "loss": 0.5593,
+      "step": 411
+    },
+    {
+      "epoch": 0.4917934944792599,
+      "grad_norm": 1.1992396152146196,
+      "learning_rate": 4.390518074937791e-06,
+      "loss": 0.6197,
+      "step": 412
+    },
+    {
+      "epoch": 0.4929871680095494,
+      "grad_norm": 1.2931882830447148,
+      "learning_rate": 4.387369470651605e-06,
+      "loss": 0.5987,
+      "step": 413
+    },
+    {
+      "epoch": 0.4941808415398389,
+      "grad_norm": 1.3870381593601393,
+      "learning_rate": 4.384213889627115e-06,
+      "loss": 0.566,
+      "step": 414
+    },
+    {
+      "epoch": 0.4953745150701283,
+      "grad_norm": 1.450139028407346,
+      "learning_rate": 4.381051343529054e-06,
+      "loss": 0.6401,
+      "step": 415
+    },
+    {
+      "epoch": 0.4965681886004178,
+      "grad_norm": 1.511602442791746,
+      "learning_rate": 4.377881844047905e-06,
+      "loss": 0.5967,
+      "step": 416
+    },
+    {
+      "epoch": 0.4977618621307073,
+      "grad_norm": 1.2080612008668248,
+      "learning_rate": 4.374705402899849e-06,
+      "loss": 0.5858,
+      "step": 417
+    },
+    {
+      "epoch": 0.4989555356609967,
+      "grad_norm": 1.2307479615208436,
+      "learning_rate": 4.371522031826733e-06,
+      "loss": 0.6072,
+      "step": 418
+    },
+    {
+      "epoch": 0.5001492091912861,
+      "grad_norm": 1.250346505274662,
+      "learning_rate": 4.368331742596018e-06,
+      "loss": 0.619,
+      "step": 419
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 1674,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 419,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 171296686080000.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-419/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de16ea2bd057d62e3a1660882a22cff76073b1ca1a9e6a5b2b563a24cf6ab122
+size 9016

checkpoint-419/zero_to_fp32.py

@@ -0,0 +1,674 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+#   python zero_to_fp32.py . output_dir/
+#   or
+#   python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import json
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in tqdm(param_shapes.items(), desc='Gathering Sharded Weights'):
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+                                               output_dir,
+                                               max_shard_size="5GB",
+                                               safe_serialization=False,
+                                               tag=None,
+                                               exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_dir``: directory to the pytorch fp32 state_dict output files
+        - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+        - ``safe_serialization``:  whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+    # Dependency pre-check
+    if safe_serialization:
+        try:
+            from safetensors.torch import save_file
+        except ImportError:
+            print('If you want to use `safe_serialization`, please `pip install safetensors`')
+            raise
+    if max_shard_size is not None:
+        try:
+            from huggingface_hub import split_torch_state_dict_into_shards
+        except ImportError:
+            print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+            raise
+
+    # Convert zero checkpoint to state_dict
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+
+    # Shard the model if it is too big.
+    weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+    if max_shard_size is not None:
+        filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+        state_dict_split = split_torch_state_dict_into_shards(state_dict,
+                                                              filename_pattern=filename_pattern,
+                                                              max_shard_size=max_shard_size)
+    else:
+        from collections import namedtuple
+        StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+        state_dict_split = StateDictSplit(is_sharded=False,
+                                          filename_to_tensors={weights_name: list(state_dict.keys())})
+
+    # Save the model
+    filename_to_tensors = state_dict_split.filename_to_tensors.items()
+    for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+        shard = {tensor: state_dict[tensor].contiguous() for tensor in tensors}
+        output_path = os.path.join(output_dir, shard_file)
+        if safe_serialization:
+            save_file(shard, output_path, metadata={"format": "pt"})
+        else:
+            torch.save(shard, output_path)
+
+    # Save index if sharded
+    if state_dict_split.is_sharded:
+        index = {
+            "metadata": state_dict_split.metadata,
+            "weight_map": state_dict_split.tensor_to_filename,
+        }
+        save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+        save_index_file = os.path.join(output_dir, save_index_file)
+        with open(save_index_file, "w", encoding="utf-8") as f:
+            content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+            f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument("output_dir",
+                        type=str,
+                        help="directory to the pytorch fp32 state_dict output files"
+                        "(e.g. path/checkpoint-12-output/)")
+    parser.add_argument(
+        "--max_shard_size",
+        type=str,
+        default="5GB",
+        help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+        "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+        "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+        "without CPU OOM issues.")
+    parser.add_argument(
+        "--safe_serialization",
+        default=False,
+        action='store_true',
+        help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_dir,
+                                               max_shard_size=args.max_shard_size,
+                                               safe_serialization=args.safe_serialization,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)