fkjsdahfjkasf/Test-SFTLlama-3-8B

README.md

@@ -40,7 +40,7 @@ More information needed
 The following hyperparameters were used during training:
 - learning_rate: 0.0002
 - train_batch_size: 4
-- eval_batch_size: 8
+- eval_batch_size: 4
 - seed: 1337
 - gradient_accumulation_steps: 4
 - total_train_batch_size: 16

adapter_config.json

@@ -15,21 +15,21 @@
   "megatron_config": null,
   "megatron_core": "megatron.core",
   "modules_to_save": [
-    "lm_head",
-    "embed_tokens"
+    "embed_tokens",
+    "lm_head"
   ],
   "peft_type": "LORA",
   "r": 16,
   "rank_pattern": {},
   "revision": "unsloth",
   "target_modules": [
-    "o_proj",
-    "v_proj",
     "q_proj",
+    "k_proj",
+    "gate_proj",
+    "o_proj",
     "down_proj",
     "up_proj",
-    "k_proj",
-    "gate_proj"
+    "v_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4ca9fdab4201ded288d8601a9d158b50c62453b6cd8d41920f6811895f3825cb
+oid sha256:4e9f5d5f3504664faf8ff0644f67875957495be6aa92402ba1fd27edaaf91938
 size 4370526560

runs/Jun10_00-16-48_150-136-147-171/events.out.tfevents.1717978611.150-136-147-171.14731.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:155799e1ff24cbdd150030de973f84cb2518eff671058a9a57b8bcffb6e58b9a
+size 5693

trainer_state.json

@@ -10,18 +10,18 @@
   "log_history": [
     {
       "epoch": 1.0,
-      "num_input_tokens_seen": 65536,
+      "num_input_tokens_seen": 131072,
       "step": 1,
-      "total_flos": 3174119881310208.0,
-      "train_loss": 0.4201101064682007,
-      "train_runtime": 27.3689,
-      "train_samples_per_second": 0.292,
-      "train_steps_per_second": 0.037
+      "total_flos": 6348239762620416.0,
+      "train_loss": 2.5150043964385986,
+      "train_runtime": 47.1012,
+      "train_samples_per_second": 0.34,
+      "train_steps_per_second": 0.021
     }
   ],
   "logging_steps": 5,
   "max_steps": 1,
-  "num_input_tokens_seen": 65536,
+  "num_input_tokens_seen": 131072,
   "num_train_epochs": 1,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -36,7 +36,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 3174119881310208.0,
+  "total_flos": 6348239762620416.0,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:091dbaea9a965c70089e5b0c0b35eb398f4f56aa913087c407400c4fc8875de1
+oid sha256:a45267d968446bd97aaa0a3e2de91a0c88bffbaea2477221bd003801bd50f9d3
 size 5112