Model save

README.md

@@ -2,10 +2,6 @@
 license: gemma
 base_model: google/gemma-2-9b
 tags:
-- easylm
-- trl
-- sft
-- generated_from_trainer
 - trl
 - sft
 - generated_from_trainer
@@ -23,7 +19,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [google/gemma-2-9b](https://huggingface.co/google/gemma-2-9b) on the alpaca_farm dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.0189
+- Loss: 1.4481
 
 ## Model description
 
@@ -43,13 +39,13 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 2e-05
-- train_batch_size: 2
-- eval_batch_size: 2
+- train_batch_size: 1
+- eval_batch_size: 1
 - seed: 42
 - distributed_type: multi-GPU
 - num_devices: 4
-- total_train_batch_size: 8
-- total_eval_batch_size: 8
+- total_train_batch_size: 4
+- total_eval_batch_size: 4
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - num_epochs: 3
@@ -58,9 +54,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| No log        | 1.0   | 13   | 0.7451          |
-| No log        | 2.0   | 26   | 0.8544          |
-| No log        | 3.0   | 39   | 1.0189          |
+| 0.9381        | 1.0   | 2500 | 0.9376          |
+| 0.4124        | 2.0   | 5000 | 1.0478          |
+| 0.1515        | 3.0   | 7500 | 1.4481          |
 
 
 ### Framework versions

all_results.json

@@ -1,14 +1,9 @@
 {
     "epoch": 3.0,
-    "eval_loss": 1.0189313888549805,
-    "eval_runtime": 1.2033,
-    "eval_samples": 100,
-    "eval_samples_per_second": 83.104,
-    "eval_steps_per_second": 10.804,
-    "total_flos": 2667373869924352.0,
-    "train_loss": 0.4380333729279347,
-    "train_runtime": 62.4476,
-    "train_samples": 100,
-    "train_samples_per_second": 4.804,
-    "train_steps_per_second": 0.625
+    "total_flos": 2.3924465601190298e+17,
+    "train_loss": 0.5169153635660807,
+    "train_runtime": 5042.7608,
+    "train_samples": 10000,
+    "train_samples_per_second": 5.949,
+    "train_steps_per_second": 1.487
 }

model-00001-of-00005.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:73be13f0027911e41266b06be2f0964f1769a87506d6ca16654821085a1194e6
+oid sha256:d5961b1d69d275c39dc069772d844b00dc76548033be4945b533f969d48cf2d2
 size 4903351912

model-00002-of-00005.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:22dad6692d8bf3170c55407b9beb1f31a83108383ba0b162882382dfc30297d5
+oid sha256:d9412bf9310b91d24ab66be912aaaca7e1b4a948ae143bf3a3cc544989ac5acf
 size 4947570872

model-00003-of-00005.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:069d4a9c1dc6ecd4696dbf8a7ba5e92b86e3d581317f2c14567f0bc33764c7ca
+oid sha256:76ad58c2416f8d3e641c4e0a048d7e79be909b3504d20083fe875ccc83e67442
 size 4962221464

model-00004-of-00005.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c18ea4aecc59553ba832c282834bc40d8021246d59f01789fee1524e564cc55f
+oid sha256:f37d1089517faab0a6bd850d428abecdbe471a6fdd34d39e24e0d6092c17ea3d
 size 3670322200

model-00005-of-00005.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:625618147894e35d0006e2761ed5343576f9c1ba0ef73232ecef83b4652dae0b
+oid sha256:005757341b2e5052d7c5f78e1807a373a49bdd0938d82a2328f33cad9a6579a5
 size 1835008128

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:62aac1caf8a9d4c3f0bbcea6f3b568dc4c31697217cfb0a518a27db2e4da992a
-size 17518624
+oid sha256:7da53ca29fb16f6b2489482fc0bc6a394162cdab14d12764a1755ebc583fea79
+size 17518525

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 3.0,
-    "total_flos": 2667373869924352.0,
-    "train_loss": 0.4380333729279347,
-    "train_runtime": 62.4476,
-    "train_samples": 100,
-    "train_samples_per_second": 4.804,
-    "train_steps_per_second": 0.625
+    "total_flos": 2.3924465601190298e+17,
+    "train_loss": 0.5169153635660807,
+    "train_runtime": 5042.7608,
+    "train_samples": 10000,
+    "train_samples_per_second": 5.949,
+    "train_steps_per_second": 1.487
 }

trainer_state.json

@@ -3,47 +3,152 @@
   "best_model_checkpoint": null,
   "epoch": 3.0,
   "eval_steps": 500,
-  "global_step": 39,
+  "global_step": 7500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
+    {
+      "epoch": 0.2,
+      "grad_norm": 5.575812339782715,
+      "learning_rate": 1.9781476007338058e-05,
+      "loss": 0.9324,
+      "step": 500
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 4.583652973175049,
+      "learning_rate": 1.913545457642601e-05,
+      "loss": 0.9619,
+      "step": 1000
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 3.9456448554992676,
+      "learning_rate": 1.8090169943749477e-05,
+      "loss": 0.979,
+      "step": 1500
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 5.700883865356445,
+      "learning_rate": 1.6691306063588583e-05,
+      "loss": 0.9591,
+      "step": 2000
+    },
     {
       "epoch": 1.0,
-      "eval_loss": 0.7451236844062805,
-      "eval_runtime": 1.2183,
-      "eval_samples_per_second": 82.081,
-      "eval_steps_per_second": 10.67,
-      "step": 13
+      "grad_norm": 3.7766757011413574,
+      "learning_rate": 1.5000000000000002e-05,
+      "loss": 0.9381,
+      "step": 2500
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.937586784362793,
+      "eval_runtime": 43.9207,
+      "eval_samples_per_second": 45.537,
+      "eval_steps_per_second": 11.384,
+      "step": 2500
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 4.477228164672852,
+      "learning_rate": 1.3090169943749475e-05,
+      "loss": 0.4559,
+      "step": 3000
+    },
+    {
+      "epoch": 1.4,
+      "grad_norm": 4.878340721130371,
+      "learning_rate": 1.1045284632676535e-05,
+      "loss": 0.4521,
+      "step": 3500
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 3.9909627437591553,
+      "learning_rate": 8.954715367323468e-06,
+      "loss": 0.4468,
+      "step": 4000
+    },
+    {
+      "epoch": 1.8,
+      "grad_norm": 2.667670726776123,
+      "learning_rate": 6.909830056250527e-06,
+      "loss": 0.4321,
+      "step": 4500
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 4.369731903076172,
+      "learning_rate": 5.000000000000003e-06,
+      "loss": 0.4124,
+      "step": 5000
     },
     {
       "epoch": 2.0,
-      "eval_loss": 0.8544089794158936,
-      "eval_runtime": 1.1946,
-      "eval_samples_per_second": 83.713,
-      "eval_steps_per_second": 10.883,
-      "step": 26
+      "eval_loss": 1.0478131771087646,
+      "eval_runtime": 41.0932,
+      "eval_samples_per_second": 48.67,
+      "eval_steps_per_second": 12.167,
+      "step": 5000
+    },
+    {
+      "epoch": 2.2,
+      "grad_norm": 2.2593352794647217,
+      "learning_rate": 3.308693936411421e-06,
+      "loss": 0.1657,
+      "step": 5500
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 2.3517982959747314,
+      "learning_rate": 1.9098300562505266e-06,
+      "loss": 0.1616,
+      "step": 6000
+    },
+    {
+      "epoch": 2.6,
+      "grad_norm": 2.2167694568634033,
+      "learning_rate": 8.645454235739903e-07,
+      "loss": 0.153,
+      "step": 6500
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 2.2378008365631104,
+      "learning_rate": 2.1852399266194312e-07,
+      "loss": 0.1521,
+      "step": 7000
+    },
+    {
+      "epoch": 3.0,
+      "grad_norm": 1.863773226737976,
+      "learning_rate": 0.0,
+      "loss": 0.1515,
+      "step": 7500
     },
     {
       "epoch": 3.0,
-      "eval_loss": 1.0189313888549805,
-      "eval_runtime": 1.2024,
-      "eval_samples_per_second": 83.167,
-      "eval_steps_per_second": 10.812,
-      "step": 39
+      "eval_loss": 1.4480849504470825,
+      "eval_runtime": 41.4492,
+      "eval_samples_per_second": 48.252,
+      "eval_steps_per_second": 12.063,
+      "step": 7500
     },
     {
       "epoch": 3.0,
-      "step": 39,
-      "total_flos": 2667373869924352.0,
-      "train_loss": 0.4380333729279347,
-      "train_runtime": 62.4476,
-      "train_samples_per_second": 4.804,
-      "train_steps_per_second": 0.625
+      "step": 7500,
+      "total_flos": 2.3924465601190298e+17,
+      "train_loss": 0.5169153635660807,
+      "train_runtime": 5042.7608,
+      "train_samples_per_second": 5.949,
+      "train_steps_per_second": 1.487
     }
   ],
   "logging_steps": 500,
-  "max_steps": 39,
+  "max_steps": 7500,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 3,
   "save_steps": 500,
@@ -59,8 +164,8 @@
       "attributes": {}
     }
   },
-  "total_flos": 2667373869924352.0,
-  "train_batch_size": 2,
+  "total_flos": 2.3924465601190298e+17,
+  "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9ce09131e321f54043af13f56c4236677dd675ebddec7aef867688091507c91a
+oid sha256:1e0430645b73c3ab0a8f9201865b6bc0ad3d630c39c2224f820bdb6c9006a4db
 size 6520