End of training

README.md

@@ -43,11 +43,11 @@ early_stopping_patience: null
 eval_max_new_tokens: 128
 eval_table_size: null
 evals_per_epoch: 1
-flash_attention: false
+flash_attention: true
 fp16: null
 fsdp: null
 fsdp_config: null
-gradient_accumulation_steps: 8
+gradient_accumulation_steps: 32
 gradient_checkpointing: true
 group_by_length: false
 hub_model_id: error577/a61d034e-86de-4b0d-9bd2-31e33368fb4b
@@ -59,14 +59,14 @@ load_in_4bit: true
 load_in_8bit: false
 local_rank: null
 logging_steps: 1
-lora_alpha: 16
+lora_alpha: 64
 lora_dropout: 0.05
 lora_fan_in_fan_out: null
 lora_model_dir: null
 lora_r: 32
 lora_target_linear: true
 lr_scheduler: cosine
-max_steps: 100
+max_steps: 150
 micro_batch_size: 1
 mlflow_experiment_name: /tmp/dc7503eebb726953_train_data.json
 model_type: AutoModelForCausalLM
@@ -103,7 +103,7 @@ xformers_attention: null
 
 This model is a fine-tuned version of [unsloth/Qwen2.5-Coder-1.5B-Instruct](https://huggingface.co/unsloth/Qwen2.5-Coder-1.5B-Instruct) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: nan
+- Loss: 0.9791
 
 ## Model description
 
@@ -126,22 +126,21 @@ The following hyperparameters were used during training:
 - train_batch_size: 1
 - eval_batch_size: 1
 - seed: 42
-- gradient_accumulation_steps: 8
-- total_train_batch_size: 8
+- gradient_accumulation_steps: 32
+- total_train_batch_size: 32
 - optimizer: Use OptimizerNames.ADAMW_BNB with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_steps: 10
-- training_steps: 100
+- training_steps: 150
 
 ### Training results
 
 | Training Loss | Epoch  | Step | Validation Loss |
 |:-------------:|:------:|:----:|:---------------:|
-| 3.2583        | 0.0001 | 1    | nan             |
-| 24.1046       | 0.0030 | 25   | nan             |
-| 0.355         | 0.0061 | 50   | nan             |
-| 2.123         | 0.0091 | 75   | nan             |
-| 0.0           | 0.0122 | 100  | nan             |
+| 0.9356        | 0.0005 | 1    | 1.4002          |
+| 0.8597        | 0.0185 | 38   | 1.0362          |
+| 0.7329        | 0.0371 | 76   | 1.0017          |
+| 0.7452        | 0.0556 | 114  | 0.9791          |
 
 
 ### Framework versions

adapter_config.json

@@ -10,7 +10,7 @@
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
-  "lora_alpha": 16,
+  "lora_alpha": 64,
   "lora_dropout": 0.05,
   "megatron_config": null,
   "megatron_core": "megatron.core",
@@ -20,13 +20,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "gate_proj",
+    "up_proj",
     "v_proj",
-    "q_proj",
-    "down_proj",
+    "gate_proj",
     "o_proj",
     "k_proj",
-    "up_proj"
+    "down_proj",
+    "q_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:63ec74f995aa9048c4822888d685bdb31bcd39af27bb4ac29ac2d99f340f9535
+oid sha256:1ddd44b249280688d97a8e97301c8a4b1cf65a564085f7cfefdf8ca02edbb062
 size 147859242

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7dbab7c6ea2c36c6d7523df20e159e4fb7ab1785c95d096667084d635efd1669
+oid sha256:ea8c39a373768c321ead30969479aa45c3a779655bab2f5485015b0b089ad428
 size 147770496

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7aab35b6e85cc2ea075fe59fc082d102bfb0df246bba39d4e5bf62d4c38db4ac
+oid sha256:0f7ef9edaee929dc90d8ca5057d512ea7538f832777a370fbf9c487dff3330db
 size 6776