Training in progress, step 100, checkpoint

last-checkpoint/config.json

@@ -1,28 +1,30 @@
 {
-  "_name_or_path": "unsloth/OpenHermes-2.5-Mistral-7B",
+  "_name_or_path": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
   "architectures": [
-    "MistralForCausalLM"
+    "LlamaForCausalLM"
   ],
+  "attention_bias": false,
   "attention_dropout": 0.0,
   "bos_token_id": 1,
-  "eos_token_id": 32000,
-  "head_dim": 128,
+  "eos_token_id": 2,
+  "head_dim": 64,
   "hidden_act": "silu",
-  "hidden_size": 4096,
+  "hidden_size": 2048,
   "initializer_range": 0.02,
-  "intermediate_size": 14336,
-  "max_position_embeddings": 32768,
-  "model_type": "mistral",
+  "intermediate_size": 5632,
+  "max_position_embeddings": 2048,
+  "mlp_bias": false,
+  "model_type": "llama",
   "num_attention_heads": 32,
-  "num_hidden_layers": 32,
-  "num_key_value_heads": 8,
-  "pad_token_id": 0,
+  "num_hidden_layers": 22,
+  "num_key_value_heads": 4,
+  "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
   "rope_theta": 10000.0,
-  "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.48.1",
   "use_cache": false,
-  "vocab_size": 32002
+  "vocab_size": 32000
 }

last-checkpoint/generation_config.json

@@ -1,7 +1,8 @@
 {
-  "_from_model_config": true,
   "bos_token_id": 1,
   "do_sample": true,
-  "eos_token_id": 32000,
+  "eos_token_id": 2,
+  "max_length": 2048,
+  "pad_token_id": 0,
   "transformers_version": "4.48.1"
 }

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e9ce8744c1e71a49bf9bff3ccecfc98f1e92613ac7fabd94737ad873ca1a94de
-size 250490408
+oid sha256:9f733e9041aab06f212569d2073af6e6c0acf30fcc84bec18f30feab19bc086f
+size 2200119864

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d9ba45f2201a4a373dfbdc728ab19e675bfdfebf3a6c2523b065cadd5c719946
-size 14710155092
+oid sha256:2e7f6e0595a900a9079a5615b8fdd966aefa419bcc59d072afe009c68faa40b1
+size 2235234810

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f20d155a14d34cd1bb6d04e5de90f3224906e1758821edd752a8f1a9085a2db
-size 15024
+oid sha256:5e139d403fa01e97c7f0b1fb6f366f88e99b3ef2d1090658accfac79d4e4bf7a
+size 15984

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ad18e870a176ca75a54f6620f83de92bdfd5a91302744d90bff8e5feae2fe0c5
-size 15024
+oid sha256:163db0385f3f2c6cc1cdde3fd1262ef8c70bc95cb0742071833b93242de76dd2
+size 15984

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9009a52be47b75834407dc5e146ed5360e6f23a35bff27bab34ef6fb47df1661
-size 15024
+oid sha256:a3f2223c650d0a4e2baa05aba88577beb8cf1341342d6523706f17972a2f481c
+size 15984

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7fb494a08fd57e7c6f63f06826c872164986e81b271996be0496671f713bdcc3
-size 15024
+oid sha256:f06016ff924c2dbb5f6bd90ff3c51224f11735ecac5d42e1b0d6eb800acc7b67
+size 15984

last-checkpoint/rng_state_4.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e76900c3a49367b21bcd4b796fdf1376987ed660638bdf98ccd45d1a87f5c184
+oid sha256:7099cb24dc8750eaaa6a5df6cead1dfa156e46f7a3a478fb7ea1f45a9599d5c1
 size 15984

last-checkpoint/rng_state_5.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4a446f1cd24dab7caff9cadff49a53cd1909d2fb5bba7af3112c5b9d54f4308b
+oid sha256:19562befb91856d315c73b5e1fe6c1554bd4ffd5d803c7f9588b2b2a18b10eb7
 size 15984

last-checkpoint/rng_state_6.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:507ef075351dc4050f0467f54693ae3336a7708023dc0aefe5fd3e986b17b4a8
+oid sha256:b055646d6b89d5f656bbcc0f7774bff592c8649c36beead05e7d98a901842815
 size 15984

last-checkpoint/rng_state_7.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:de7dde0309a85b36fb0802e6f2c30acf442b986826853119d1478a511a694e77
+oid sha256:6d80f9631b5929172f6acf476e00b42058e6cff03360fb2e0c295ebe290c9cd6
 size 15984

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9cd11d413bc67bf01de9a1a006e9e7655be307353028b25f5b3c299e5b6b7a44
+oid sha256:3a60c7d771c1fd156acee762fba03c724cb41829a3f71df370ecd1d20b134982
 size 1064

last-checkpoint/special_tokens_map.json

@@ -7,14 +7,14 @@
     "single_word": false
   },
   "eos_token": {
-    "content": "<|im_end|>",
+    "content": "</s>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "pad_token": {
-    "content": "<unk>",
+    "content": "</s>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

last-checkpoint/tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:04222cd76979c181cd3f72c3bf6982fe2a09d9f4b8f23d82902efde18f1d0668
-size 3506125
+oid sha256:c88bda6bdd84543eadebdf4bd2ec325ae43b71a3ff1fead6a766562c00b29bd8
+size 3619016

last-checkpoint/tokenizer.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
-size 493443
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

last-checkpoint/tokenizer_config.json

@@ -1,7 +1,7 @@
 {
   "add_bos_token": true,
   "add_eos_token": false,
-  "add_prefix_space": true,
+  "add_prefix_space": null,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",
@@ -26,39 +26,20 @@
       "rstrip": false,
       "single_word": false,
       "special": true
-    },
-    "32000": {
-      "content": "<|im_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32001": {
-      "content": "<|im_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
     }
   },
-  "additional_special_tokens": [],
   "bos_token": "<s>",
-  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "<|im_end|>",
+  "eos_token": "</s>",
   "extra_special_tokens": {},
-  "legacy": true,
-  "model_max_length": 32768,
-  "pad_token": "<unk>",
+  "legacy": false,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
   "padding_side": "right",
   "sp_model_kwargs": {},
-  "spaces_between_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",
-  "trust_remote_code": false,
   "unk_token": "<unk>",
-  "use_default_system_prompt": true,
+  "use_default_system_prompt": false,
   "use_fast": true
 }

last-checkpoint/trainer_state.json

@@ -1,174 +1,96 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.9047619047619047,
+  "epoch": 12.5,
   "eval_steps": 200,
-  "global_step": 200,
+  "global_step": 100,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.009523809523809525,
-      "eval_loss": 1.873080849647522,
-      "eval_runtime": 27.6632,
-      "eval_samples_per_second": 54.26,
-      "eval_steps_per_second": 6.796,
+      "epoch": 0.125,
+      "eval_loss": 2.0389645099639893,
+      "eval_runtime": 3.3031,
+      "eval_samples_per_second": 454.427,
+      "eval_steps_per_second": 4.844,
       "step": 1
     },
     {
-      "epoch": 0.09523809523809523,
-      "grad_norm": 25.75,
-      "learning_rate": 6.666666666666667e-05,
-      "loss": 2.7082,
+      "epoch": 1.25,
+      "grad_norm": 2.234375,
+      "learning_rate": 0.00019863613034027224,
+      "loss": 1.8385,
       "step": 10
     },
     {
-      "epoch": 0.19047619047619047,
-      "grad_norm": 466.0,
-      "learning_rate": 0.00013333333333333334,
-      "loss": 3.6654,
+      "epoch": 2.5,
+      "grad_norm": 1.90625,
+      "learning_rate": 0.0001879473751206489,
+      "loss": 1.1862,
       "step": 20
     },
     {
-      "epoch": 0.2857142857142857,
-      "grad_norm": 2040.0,
-      "learning_rate": 0.0002,
-      "loss": 9.544,
+      "epoch": 3.75,
+      "grad_norm": 1.375,
+      "learning_rate": 0.00016772815716257412,
+      "loss": 0.615,
       "step": 30
     },
     {
-      "epoch": 0.38095238095238093,
-      "grad_norm": 1632.0,
-      "learning_rate": 0.00019984815164333163,
-      "loss": 9.2338,
+      "epoch": 5.0,
+      "grad_norm": 0.82421875,
+      "learning_rate": 0.00014016954246529696,
+      "loss": 0.2473,
       "step": 40
     },
     {
-      "epoch": 0.47619047619047616,
-      "grad_norm": 83.0,
-      "learning_rate": 0.00019939306773179497,
-      "loss": 8.3396,
+      "epoch": 6.25,
+      "grad_norm": 0.498046875,
+      "learning_rate": 0.00010825793454723325,
+      "loss": 0.0731,
       "step": 50
     },
     {
-      "epoch": 0.5714285714285714,
-      "grad_norm": 37.25,
-      "learning_rate": 0.00019863613034027224,
-      "loss": 7.7295,
+      "epoch": 7.5,
+      "grad_norm": 0.29296875,
+      "learning_rate": 7.54514512859201e-05,
+      "loss": 0.0275,
       "step": 60
     },
     {
-      "epoch": 0.6666666666666666,
-      "grad_norm": 109.5,
-      "learning_rate": 0.00019757963826274357,
-      "loss": 7.6581,
+      "epoch": 8.75,
+      "grad_norm": 0.11962890625,
+      "learning_rate": 4.530518418775733e-05,
+      "loss": 0.0112,
       "step": 70
     },
     {
-      "epoch": 0.7619047619047619,
-      "grad_norm": 95.5,
-      "learning_rate": 0.00019622680003092503,
-      "loss": 7.5321,
+      "epoch": 10.0,
+      "grad_norm": 0.06787109375,
+      "learning_rate": 2.1085949060360654e-05,
+      "loss": 0.0078,
       "step": 80
     },
     {
-      "epoch": 0.8571428571428571,
-      "grad_norm": 76.5,
-      "learning_rate": 0.00019458172417006347,
-      "loss": 7.4379,
+      "epoch": 11.25,
+      "grad_norm": 0.061767578125,
+      "learning_rate": 5.418275829936537e-06,
+      "loss": 0.0072,
       "step": 90
     },
     {
-      "epoch": 0.9523809523809523,
-      "grad_norm": 34.25,
-      "learning_rate": 0.00019264940672148018,
-      "loss": 7.4215,
+      "epoch": 12.5,
+      "grad_norm": 0.0615234375,
+      "learning_rate": 0.0,
+      "loss": 0.007,
       "step": 100
-    },
-    {
-      "epoch": 1.0476190476190477,
-      "grad_norm": 25.75,
-      "learning_rate": 0.00019043571606975777,
-      "loss": 7.3074,
-      "step": 110
-    },
-    {
-      "epoch": 1.1428571428571428,
-      "grad_norm": 27.0,
-      "learning_rate": 0.0001879473751206489,
-      "loss": 7.2047,
-      "step": 120
-    },
-    {
-      "epoch": 1.2380952380952381,
-      "grad_norm": 16.75,
-      "learning_rate": 0.00018519194088383273,
-      "loss": 7.2054,
-      "step": 130
-    },
-    {
-      "epoch": 1.3333333333333333,
-      "grad_norm": 312.0,
-      "learning_rate": 0.0001821777815225245,
-      "loss": 7.2831,
-      "step": 140
-    },
-    {
-      "epoch": 1.4285714285714286,
-      "grad_norm": 134.0,
-      "learning_rate": 0.00017891405093963938,
-      "loss": 7.1299,
-      "step": 150
-    },
-    {
-      "epoch": 1.5238095238095237,
-      "grad_norm": 255.0,
-      "learning_rate": 0.00017541066097768963,
-      "loss": 7.0289,
-      "step": 160
-    },
-    {
-      "epoch": 1.619047619047619,
-      "grad_norm": 21.125,
-      "learning_rate": 0.00017167825131684513,
-      "loss": 7.0257,
-      "step": 170
-    },
-    {
-      "epoch": 1.7142857142857144,
-      "grad_norm": 18.5,
-      "learning_rate": 0.00016772815716257412,
-      "loss": 6.9214,
-      "step": 180
-    },
-    {
-      "epoch": 1.8095238095238095,
-      "grad_norm": 25.25,
-      "learning_rate": 0.00016357237482099684,
-      "loss": 6.8652,
-      "step": 190
-    },
-    {
-      "epoch": 1.9047619047619047,
-      "grad_norm": 12.125,
-      "learning_rate": 0.00015922352526649803,
-      "loss": 6.745,
-      "step": 200
-    },
-    {
-      "epoch": 1.9047619047619047,
-      "eval_loss": 6.904326438903809,
-      "eval_runtime": 26.9851,
-      "eval_samples_per_second": 55.623,
-      "eval_steps_per_second": 6.967,
-      "step": 200
     }
   ],
   "logging_steps": 10,
-  "max_steps": 600,
+  "max_steps": 100,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 6,
+  "num_train_epochs": 13,
   "save_steps": 200,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -177,13 +99,13 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 1.3980142170800128e+17,
-  "train_batch_size": 2,
+  "total_flos": 1.2203604110737408e+17,
+  "train_batch_size": 12,
   "trial_name": null,
   "trial_params": null
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a62c95db7d808ffe6624a24b5b202f0f59342fc3d11220ea4577943c4efbf06e
-size 6776
+oid sha256:917005e54e6a74be4d5bc3c0309ff36483f7719e494b126c6cee70ee375d0421
+size 7032