Jessiecs/llama-2-7b-a3-1

README.md

@@ -50,8 +50,8 @@ The following hyperparameters were used during training:
 
 ### Framework versions
 
-- PEFT 0.8.2
+- PEFT 0.9.1.dev0
 - Transformers 4.39.0.dev0
 - Pytorch 2.1.0+cu121
-- Datasets 2.17.1
+- Datasets 2.18.0
 - Tokenizers 0.15.2

adapter_config.json

@@ -19,14 +19,9 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "o_proj",
-    "gate_proj",
-    "v_proj",
-    "down_proj",
-    "q_proj",
-    "k_proj",
-    "up_proj"
+    "base_layer"
   ],
   "task_type": "CAUSAL_LM",
+  "use_dora": false,
   "use_rslora": false
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6d96377ef3b44b73b41c9e397b84240de3c2182ce8439c2003588edfe11babc6
-size 159967880
+oid sha256:f7000b23c914cd224db2ce8978cd9850ea53b3ea53b1ae1810a8e5b5aafd9799
+size 319941280

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 0.01,
-    "total_flos": 1332162927353856.0,
-    "train_loss": 1.5293153256177903,
-    "train_runtime": 150.2786,
-    "train_samples_per_second": 0.532,
-    "train_steps_per_second": 0.133
+    "total_flos": 1201627119648768.0,
+    "train_loss": 1.4025826781988144,
+    "train_runtime": 63.619,
+    "train_samples_per_second": 1.257,
+    "train_steps_per_second": 0.314
 }

runs/Mar03_23-45-18_806464d2401f/events.out.tfevents.1709509519.806464d2401f.1315.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd103f01c228160c61232247d56d86c92efd9997dd973462b089275b98aff4b7
+size 5094

runs/Mar03_23-51-27_806464d2401f/events.out.tfevents.1709509887.806464d2401f.1315.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc50255424634284e358127d57733d452f9d7bf6fdb96685f85a981cdbf8a9d5
+size 9582

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 0.01,
-    "total_flos": 1332162927353856.0,
-    "train_loss": 1.5293153256177903,
-    "train_runtime": 150.2786,
-    "train_samples_per_second": 0.532,
-    "train_steps_per_second": 0.133
+    "total_flos": 1201627119648768.0,
+    "train_loss": 1.4025826781988144,
+    "train_runtime": 63.619,
+    "train_samples_per_second": 1.257,
+    "train_steps_per_second": 0.314
 }

trainer_state.json

@@ -1,7 +1,7 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.00812842918106076,
+  "epoch": 0.0081276033729554,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,152 +10,152 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 1.7647329568862915,
+      "grad_norm": 1.8456366062164307,
       "learning_rate": 0.0001,
-      "loss": 1.7092,
+      "loss": 1.7067,
       "step": 1
     },
     {
       "epoch": 0.0,
-      "grad_norm": 1.3001192808151245,
+      "grad_norm": 2.7521228790283203,
       "learning_rate": 0.0002,
-      "loss": 1.2963,
+      "loss": 2.4493,
       "step": 2
     },
     {
       "epoch": 0.0,
-      "grad_norm": 1.5390220880508423,
+      "grad_norm": 1.7731740474700928,
       "learning_rate": 0.00018888888888888888,
-      "loss": 1.5465,
+      "loss": 1.7336,
       "step": 3
     },
     {
       "epoch": 0.0,
-      "grad_norm": 2.012969493865967,
+      "grad_norm": 2.415656089782715,
       "learning_rate": 0.00017777777777777779,
-      "loss": 1.9059,
+      "loss": 2.0648,
       "step": 4
     },
     {
       "epoch": 0.0,
-      "grad_norm": 1.1633769273757935,
+      "grad_norm": 0.8457527756690979,
       "learning_rate": 0.0001666666666666667,
-      "loss": 1.7834,
+      "loss": 1.1381,
       "step": 5
     },
     {
       "epoch": 0.0,
-      "grad_norm": 0.9096754789352417,
+      "grad_norm": 1.1292682886123657,
       "learning_rate": 0.00015555555555555556,
-      "loss": 1.0758,
+      "loss": 1.7235,
       "step": 6
     },
     {
       "epoch": 0.0,
-      "grad_norm": 0.9768149852752686,
+      "grad_norm": 0.8633381128311157,
       "learning_rate": 0.00014444444444444444,
-      "loss": 0.931,
+      "loss": 1.0572,
       "step": 7
     },
     {
       "epoch": 0.0,
-      "grad_norm": 1.853431224822998,
+      "grad_norm": 0.8930997848510742,
       "learning_rate": 0.00013333333333333334,
-      "loss": 1.6621,
+      "loss": 1.1119,
       "step": 8
     },
     {
       "epoch": 0.0,
-      "grad_norm": 1.3012102842330933,
+      "grad_norm": 1.5441715717315674,
       "learning_rate": 0.00012222222222222224,
-      "loss": 1.8367,
+      "loss": 1.6177,
       "step": 9
     },
     {
       "epoch": 0.0,
-      "grad_norm": 1.1715010404586792,
+      "grad_norm": 0.9879918694496155,
       "learning_rate": 0.00011111111111111112,
-      "loss": 1.4482,
+      "loss": 1.2242,
       "step": 10
     },
     {
       "epoch": 0.0,
-      "grad_norm": 0.9172013401985168,
+      "grad_norm": 0.7914152145385742,
       "learning_rate": 0.0001,
-      "loss": 1.3422,
+      "loss": 1.2375,
       "step": 11
     },
     {
       "epoch": 0.0,
-      "grad_norm": 1.617510437965393,
+      "grad_norm": 0.8946468234062195,
       "learning_rate": 8.888888888888889e-05,
-      "loss": 1.4845,
+      "loss": 0.9588,
       "step": 12
     },
     {
       "epoch": 0.01,
-      "grad_norm": 1.1238973140716553,
+      "grad_norm": 0.7223942279815674,
       "learning_rate": 7.777777777777778e-05,
-      "loss": 1.4094,
+      "loss": 0.9085,
       "step": 13
     },
     {
       "epoch": 0.01,
-      "grad_norm": 1.2244164943695068,
+      "grad_norm": 1.2389971017837524,
       "learning_rate": 6.666666666666667e-05,
-      "loss": 1.8395,
+      "loss": 1.0339,
       "step": 14
     },
     {
       "epoch": 0.01,
-      "grad_norm": 3.9478447437286377,
+      "grad_norm": 1.125956654548645,
       "learning_rate": 5.555555555555556e-05,
-      "loss": 1.7928,
+      "loss": 1.4494,
       "step": 15
     },
     {
       "epoch": 0.01,
-      "grad_norm": 1.8160783052444458,
+      "grad_norm": 1.5024690628051758,
       "learning_rate": 4.4444444444444447e-05,
-      "loss": 1.226,
+      "loss": 1.3538,
       "step": 16
     },
     {
       "epoch": 0.01,
-      "grad_norm": 1.9035887718200684,
+      "grad_norm": 2.010714054107666,
       "learning_rate": 3.3333333333333335e-05,
-      "loss": 1.599,
+      "loss": 1.2944,
       "step": 17
     },
     {
       "epoch": 0.01,
-      "grad_norm": 1.3164148330688477,
+      "grad_norm": 2.901399850845337,
       "learning_rate": 2.2222222222222223e-05,
-      "loss": 1.2311,
+      "loss": 1.3021,
       "step": 18
     },
     {
       "epoch": 0.01,
-      "grad_norm": 1.200212836265564,
+      "grad_norm": 3.5073766708374023,
       "learning_rate": 1.1111111111111112e-05,
-      "loss": 1.926,
+      "loss": 1.2913,
       "step": 19
     },
     {
       "epoch": 0.01,
-      "grad_norm": 1.5367623567581177,
+      "grad_norm": 1.580161690711975,
       "learning_rate": 0.0,
-      "loss": 1.5407,
+      "loss": 1.3949,
       "step": 20
     },
     {
       "epoch": 0.01,
       "step": 20,
-      "total_flos": 1332162927353856.0,
-      "train_loss": 1.5293153256177903,
-      "train_runtime": 150.2786,
-      "train_samples_per_second": 0.532,
-      "train_steps_per_second": 0.133
+      "total_flos": 1201627119648768.0,
+      "train_loss": 1.4025826781988144,
+      "train_runtime": 63.619,
+      "train_samples_per_second": 1.257,
+      "train_steps_per_second": 0.314
     }
   ],
   "logging_steps": 1,
@@ -163,7 +163,7 @@
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 500,
-  "total_flos": 1332162927353856.0,
+  "total_flos": 1201627119648768.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:399b61aeb844ac4532af315b56fb1fc66c1e5f62d42132308aac5f34a33b4848
+oid sha256:5f796c4f9a98cf6e61c7158bd8fa4e75db4e5279ef8cec5f1004e53fd513cb9d
 size 4920