Upload folder using huggingface_hub

adapter_config.json

@@ -10,20 +10,20 @@
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
-  "lora_alpha": 16,
-  "lora_dropout": 0.05,
+  "lora_alpha": 64,
+  "lora_dropout": 0.0001,
   "megatron_config": null,
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 8,
+  "r": 32,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "o_proj",
-    "gate_up_proj",
+    "down_proj",
     "qkv_proj",
-    "down_proj"
+    "gate_up_proj",
+    "o_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:09dd6cf05506f7922ad70f74fe6934c8336e697e67448b49f05a9b4a7a6566de
-size 50365768
+oid sha256:170e9283396f794ac39c141ef58fc732a915723bcc48acda06109764aede853c
+size 201361312

added_tokens.json

@@ -2,7 +2,6 @@
   "<|assistant|>": 32001,
   "<|endoftext|>": 32000,
   "<|end|>": 32007,
-  "<|pad|>": 32011,
   "<|placeholder1|>": 32002,
   "<|placeholder2|>": 32003,
   "<|placeholder3|>": 32004,

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:31c2c06a3aa791be14685b5d59899a877c559219322620a519bd1b84ef483910
-size 100878458
+oid sha256:aba8d5f3fb425d691ad00a11ff612f2c4ce2ef2f2350b1aa78ac024098d151a6
+size 402868986

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:936b713051d3954452ec3bf4371217942ece2f8826b34ec15d76739514f6eb2c
+oid sha256:3df287feaf25c6bbc3e39d1e8402382f635590ca96adbe728944eb6f0edd1fc9
 size 14244

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a492b6d65e6e851d97e2025279f088a6a581867a0b4272350b086ae52aef4d06
+oid sha256:50d23b4f208a9403528cc4590d75da0ba9842779b9cd25a1b5978ffbe9bcceb1
 size 1064

special_tokens_map.json

@@ -7,14 +7,14 @@
     "single_word": false
   },
   "eos_token": {
-    "content": "<|end|>",
+    "content": "<|endoftext|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "pad_token": {
-    "content": "<|pad|>",
+    "content": "<|endoftext|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer.json

@@ -1,21 +1,7 @@
 {
   "version": "1.0",
-  "truncation": {
-    "direction": "Right",
-    "max_length": 2048,
-    "strategy": "LongestFirst",
-    "stride": 0
-  },
-  "padding": {
-    "strategy": {
-      "Fixed": 2048
-    },
-    "direction": "Left",
-    "pad_to_multiple_of": null,
-    "pad_id": 32011,
-    "pad_type_id": 0,
-    "pad_token": "<|pad|>"
-  },
+  "truncation": null,
+  "padding": null,
   "added_tokens": [
     {
       "id": 0,
@@ -112,7 +98,7 @@
       "content": "<|end|>",
       "single_word": false,
       "lstrip": false,
-      "rstrip": false,
+      "rstrip": true,
       "normalized": false,
       "special": true
     },
@@ -142,15 +128,6 @@
       "rstrip": true,
       "normalized": false,
       "special": true
-    },
-    {
-      "id": 32011,
-      "content": "<|pad|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
     }
   ],
   "normalizer": {

tokenizer_config.json

@@ -87,7 +87,7 @@
       "content": "<|end|>",
       "lstrip": false,
       "normalized": false,
-      "rstrip": false,
+      "rstrip": true,
       "single_word": false,
       "special": true
     },
@@ -114,24 +114,16 @@
       "rstrip": true,
       "single_word": false,
       "special": true
-    },
-    "32011": {
-      "content": "<|pad|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
     }
   },
   "bos_token": "<s>",
   "chat_template": "{% for message in messages %}{% if message['role'] == 'system' and message['content'] %}{{'<|system|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'user' %}{{'<|user|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "<|end|>",
+  "eos_token": "<|endoftext|>",
   "extra_special_tokens": {},
   "legacy": false,
   "model_max_length": 131072,
-  "pad_token": "<|pad|>",
+  "pad_token": "<|endoftext|>",
   "padding_side": "left",
   "sp_model_kwargs": {},
   "tokenizer_class": "LlamaTokenizer",

trainer_state.json

@@ -1,50 +1,482 @@
 {
-  "best_metric": 0.7067741751670837,
-  "best_model_checkpoint": "./phi3_finetuned/checkpoint-16",
-  "epoch": 1.8235294117647058,
+  "best_metric": 0.6163371205329895,
+  "best_model_checkpoint": "//outputs/task7_microsoft/Phi-3.5-mini-instruct/checkpoint-200",
+  "epoch": 22.235294117647058,
   "eval_steps": 500,
-  "global_step": 16,
+  "global_step": 200,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
+    {
+      "epoch": 0.5882352941176471,
+      "grad_norm": 0.564612090587616,
+      "learning_rate": 5e-06,
+      "loss": 0.8053,
+      "step": 5
+    },
     {
       "epoch": 1.0,
-      "eval_loss": 0.7619180083274841,
-      "eval_runtime": 28.359,
-      "eval_samples_per_second": 0.529,
-      "eval_steps_per_second": 0.282,
+      "eval_loss": 0.7743130326271057,
+      "eval_runtime": 3.5233,
+      "eval_samples_per_second": 4.257,
+      "eval_steps_per_second": 0.568,
       "step": 9
     },
     {
       "epoch": 1.1176470588235294,
-      "grad_norm": 0.53853839635849,
-      "learning_rate": 0.000225,
-      "loss": 0.8601,
+      "grad_norm": 0.5536892414093018,
+      "learning_rate": 1e-05,
+      "loss": 0.7727,
       "step": 10
     },
     {
-      "epoch": 1.8235294117647058,
-      "eval_loss": 0.7067741751670837,
-      "eval_runtime": 28.3045,
-      "eval_samples_per_second": 0.53,
-      "eval_steps_per_second": 0.283,
-      "step": 16
+      "epoch": 1.7058823529411766,
+      "grad_norm": 0.3953665494918823,
+      "learning_rate": 9.98292246503335e-06,
+      "loss": 0.7726,
+      "step": 15
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.7348855137825012,
+      "eval_runtime": 3.3629,
+      "eval_samples_per_second": 4.46,
+      "eval_steps_per_second": 0.595,
+      "step": 18
+    },
+    {
+      "epoch": 2.235294117647059,
+      "grad_norm": 0.32548508048057556,
+      "learning_rate": 9.931806517013612e-06,
+      "loss": 0.7178,
+      "step": 20
+    },
+    {
+      "epoch": 2.8235294117647056,
+      "grad_norm": 0.27749133110046387,
+      "learning_rate": 9.847001329696653e-06,
+      "loss": 0.6607,
+      "step": 25
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.7112905383110046,
+      "eval_runtime": 3.3607,
+      "eval_samples_per_second": 4.463,
+      "eval_steps_per_second": 0.595,
+      "step": 27
+    },
+    {
+      "epoch": 3.3529411764705883,
+      "grad_norm": 0.28755590319633484,
+      "learning_rate": 9.729086208503174e-06,
+      "loss": 0.7081,
+      "step": 30
+    },
+    {
+      "epoch": 3.9411764705882355,
+      "grad_norm": 0.25980502367019653,
+      "learning_rate": 9.578866633275289e-06,
+      "loss": 0.6063,
+      "step": 35
+    },
+    {
+      "epoch": 4.0,
+      "eval_loss": 0.6943528056144714,
+      "eval_runtime": 3.3629,
+      "eval_samples_per_second": 4.46,
+      "eval_steps_per_second": 0.595,
+      "step": 36
+    },
+    {
+      "epoch": 4.470588235294118,
+      "grad_norm": 0.23154301941394806,
+      "learning_rate": 9.397368756032445e-06,
+      "loss": 0.6561,
+      "step": 40
+    },
+    {
+      "epoch": 5.0,
+      "grad_norm": 0.30559542775154114,
+      "learning_rate": 9.185832391312644e-06,
+      "loss": 0.6935,
+      "step": 45
+    },
+    {
+      "epoch": 5.0,
+      "eval_loss": 0.6810200214385986,
+      "eval_runtime": 3.3611,
+      "eval_samples_per_second": 4.463,
+      "eval_steps_per_second": 0.595,
+      "step": 45
+    },
+    {
+      "epoch": 5.588235294117647,
+      "grad_norm": 0.21162718534469604,
+      "learning_rate": 8.94570254698197e-06,
+      "loss": 0.6829,
+      "step": 50
+    },
+    {
+      "epoch": 6.0,
+      "eval_loss": 0.6704084277153015,
+      "eval_runtime": 3.3625,
+      "eval_samples_per_second": 4.461,
+      "eval_steps_per_second": 0.595,
+      "step": 54
+    },
+    {
+      "epoch": 6.117647058823529,
+      "grad_norm": 0.26222917437553406,
+      "learning_rate": 8.67861955336566e-06,
+      "loss": 0.6021,
+      "step": 55
+    },
+    {
+      "epoch": 6.705882352941177,
+      "grad_norm": 0.23411308228969574,
+      "learning_rate": 8.386407858128707e-06,
+      "loss": 0.6483,
+      "step": 60
+    },
+    {
+      "epoch": 7.0,
+      "eval_loss": 0.6606718897819519,
+      "eval_runtime": 3.3601,
+      "eval_samples_per_second": 4.464,
+      "eval_steps_per_second": 0.595,
+      "step": 63
+    },
+    {
+      "epoch": 7.235294117647059,
+      "grad_norm": 0.18744103610515594,
+      "learning_rate": 8.071063563448341e-06,
+      "loss": 0.5817,
+      "step": 65
+    },
+    {
+      "epoch": 7.823529411764706,
+      "grad_norm": 0.18960484862327576,
+      "learning_rate": 7.734740790612137e-06,
+      "loss": 0.6352,
+      "step": 70
+    },
+    {
+      "epoch": 8.0,
+      "eval_loss": 0.6521106958389282,
+      "eval_runtime": 3.3613,
+      "eval_samples_per_second": 4.463,
+      "eval_steps_per_second": 0.595,
+      "step": 72
+    },
+    {
+      "epoch": 8.352941176470589,
+      "grad_norm": 0.15531951189041138,
+      "learning_rate": 7.379736965185369e-06,
+      "loss": 0.5719,
+      "step": 75
+    },
+    {
+      "epoch": 8.941176470588236,
+      "grad_norm": 0.34726396203041077,
+      "learning_rate": 7.008477123264849e-06,
+      "loss": 0.6186,
+      "step": 80
+    },
+    {
+      "epoch": 9.0,
+      "eval_loss": 0.6448661088943481,
+      "eval_runtime": 3.3624,
+      "eval_samples_per_second": 4.461,
+      "eval_steps_per_second": 0.595,
+      "step": 81
+    },
+    {
+      "epoch": 9.470588235294118,
+      "grad_norm": 0.1773035228252411,
+      "learning_rate": 6.6234973460234184e-06,
+      "loss": 0.6052,
+      "step": 85
+    },
+    {
+      "epoch": 10.0,
+      "grad_norm": 0.2170713096857071,
+      "learning_rate": 6.227427435703997e-06,
+      "loss": 0.5415,
+      "step": 90
+    },
+    {
+      "epoch": 10.0,
+      "eval_loss": 0.6390407681465149,
+      "eval_runtime": 3.3658,
+      "eval_samples_per_second": 4.457,
+      "eval_steps_per_second": 0.594,
+      "step": 90
+    },
+    {
+      "epoch": 10.588235294117647,
+      "grad_norm": 0.2540779709815979,
+      "learning_rate": 5.82297295140367e-06,
+      "loss": 0.6305,
+      "step": 95
+    },
+    {
+      "epoch": 11.0,
+      "eval_loss": 0.6332173943519592,
+      "eval_runtime": 3.3622,
+      "eval_samples_per_second": 4.461,
+      "eval_steps_per_second": 0.595,
+      "step": 99
+    },
+    {
+      "epoch": 11.117647058823529,
+      "grad_norm": 0.2432163953781128,
+      "learning_rate": 5.412896727361663e-06,
+      "loss": 0.5547,
+      "step": 100
+    },
+    {
+      "epoch": 11.705882352941176,
+      "grad_norm": 0.2414003312587738,
+      "learning_rate": 5e-06,
+      "loss": 0.5385,
+      "step": 105
+    },
+    {
+      "epoch": 12.0,
+      "eval_loss": 0.6285383701324463,
+      "eval_runtime": 3.3638,
+      "eval_samples_per_second": 4.459,
+      "eval_steps_per_second": 0.595,
+      "step": 108
+    },
+    {
+      "epoch": 12.235294117647058,
+      "grad_norm": 0.2067604809999466,
+      "learning_rate": 4.587103272638339e-06,
+      "loss": 0.536,
+      "step": 110
+    },
+    {
+      "epoch": 12.823529411764707,
+      "grad_norm": 0.29979485273361206,
+      "learning_rate": 4.17702704859633e-06,
+      "loss": 0.5896,
+      "step": 115
+    },
+    {
+      "epoch": 13.0,
+      "eval_loss": 0.6254769563674927,
+      "eval_runtime": 3.3694,
+      "eval_samples_per_second": 4.452,
+      "eval_steps_per_second": 0.594,
+      "step": 117
+    },
+    {
+      "epoch": 13.352941176470589,
+      "grad_norm": 0.1513441950082779,
+      "learning_rate": 3.7725725642960047e-06,
+      "loss": 0.5415,
+      "step": 120
+    },
+    {
+      "epoch": 13.941176470588236,
+      "grad_norm": 0.2250215709209442,
+      "learning_rate": 3.3765026539765832e-06,
+      "loss": 0.5612,
+      "step": 125
+    },
+    {
+      "epoch": 14.0,
+      "eval_loss": 0.6232194900512695,
+      "eval_runtime": 3.3613,
+      "eval_samples_per_second": 4.463,
+      "eval_steps_per_second": 0.595,
+      "step": 126
+    },
+    {
+      "epoch": 14.470588235294118,
+      "grad_norm": 0.21195632219314575,
+      "learning_rate": 2.991522876735154e-06,
+      "loss": 0.5624,
+      "step": 130
+    },
+    {
+      "epoch": 15.0,
+      "grad_norm": 0.4384087026119232,
+      "learning_rate": 2.6202630348146323e-06,
+      "loss": 0.5871,
+      "step": 135
+    },
+    {
+      "epoch": 15.0,
+      "eval_loss": 0.6213398575782776,
+      "eval_runtime": 3.3593,
+      "eval_samples_per_second": 4.465,
+      "eval_steps_per_second": 0.595,
+      "step": 135
+    },
+    {
+      "epoch": 15.588235294117647,
+      "grad_norm": 0.23890897631645203,
+      "learning_rate": 2.265259209387867e-06,
+      "loss": 0.5352,
+      "step": 140
+    },
+    {
+      "epoch": 16.0,
+      "eval_loss": 0.6193457841873169,
+      "eval_runtime": 3.3601,
+      "eval_samples_per_second": 4.464,
+      "eval_steps_per_second": 0.595,
+      "step": 144
+    },
+    {
+      "epoch": 16.11764705882353,
+      "grad_norm": 0.24785251915454865,
+      "learning_rate": 1.928936436551661e-06,
+      "loss": 0.5998,
+      "step": 145
+    },
+    {
+      "epoch": 16.705882352941178,
+      "grad_norm": 0.21428382396697998,
+      "learning_rate": 1.6135921418712959e-06,
+      "loss": 0.5564,
+      "step": 150
+    },
+    {
+      "epoch": 17.0,
+      "eval_loss": 0.618452787399292,
+      "eval_runtime": 3.3625,
+      "eval_samples_per_second": 4.461,
+      "eval_steps_per_second": 0.595,
+      "step": 153
+    },
+    {
+      "epoch": 17.235294117647058,
+      "grad_norm": 0.19924059510231018,
+      "learning_rate": 1.321380446634342e-06,
+      "loss": 0.4868,
+      "step": 155
+    },
+    {
+      "epoch": 17.823529411764707,
+      "grad_norm": 0.16416364908218384,
+      "learning_rate": 1.0542974530180327e-06,
+      "loss": 0.6029,
+      "step": 160
+    },
+    {
+      "epoch": 18.0,
+      "eval_loss": 0.6172903776168823,
+      "eval_runtime": 3.3616,
+      "eval_samples_per_second": 4.462,
+      "eval_steps_per_second": 0.595,
+      "step": 162
+    },
+    {
+      "epoch": 18.352941176470587,
+      "grad_norm": 0.21794988214969635,
+      "learning_rate": 8.141676086873574e-07,
+      "loss": 0.4832,
+      "step": 165
+    },
+    {
+      "epoch": 18.941176470588236,
+      "grad_norm": 0.27910733222961426,
+      "learning_rate": 6.026312439675553e-07,
+      "loss": 0.5107,
+      "step": 170
+    },
+    {
+      "epoch": 19.0,
+      "eval_loss": 0.617369532585144,
+      "eval_runtime": 3.3609,
+      "eval_samples_per_second": 4.463,
+      "eval_steps_per_second": 0.595,
+      "step": 171
+    },
+    {
+      "epoch": 19.470588235294116,
+      "grad_norm": 0.21645767986774445,
+      "learning_rate": 4.211333667247125e-07,
+      "loss": 0.5692,
+      "step": 175
+    },
+    {
+      "epoch": 20.0,
+      "grad_norm": 0.39115971326828003,
+      "learning_rate": 2.7091379149682683e-07,
+      "loss": 0.5808,
+      "step": 180
+    },
+    {
+      "epoch": 20.0,
+      "eval_loss": 0.6167533993721008,
+      "eval_runtime": 3.3616,
+      "eval_samples_per_second": 4.462,
+      "eval_steps_per_second": 0.595,
+      "step": 180
+    },
+    {
+      "epoch": 20.58823529411765,
+      "grad_norm": 0.26653149724006653,
+      "learning_rate": 1.5299867030334815e-07,
+      "loss": 0.5835,
+      "step": 185
+    },
+    {
+      "epoch": 21.0,
+      "eval_loss": 0.6167729496955872,
+      "eval_runtime": 3.3615,
+      "eval_samples_per_second": 4.462,
+      "eval_steps_per_second": 0.595,
+      "step": 189
+    },
+    {
+      "epoch": 21.11764705882353,
+      "grad_norm": 0.27125898003578186,
+      "learning_rate": 6.819348298638839e-08,
+      "loss": 0.5515,
+      "step": 190
+    },
+    {
+      "epoch": 21.705882352941178,
+      "grad_norm": 0.20525327324867249,
+      "learning_rate": 1.7077534966650767e-08,
+      "loss": 0.5211,
+      "step": 195
+    },
+    {
+      "epoch": 22.0,
+      "eval_loss": 0.6173871159553528,
+      "eval_runtime": 3.3629,
+      "eval_samples_per_second": 4.46,
+      "eval_steps_per_second": 0.595,
+      "step": 198
+    },
+    {
+      "epoch": 22.235294117647058,
+      "grad_norm": 0.19269497692584991,
+      "learning_rate": 0.0,
+      "loss": 0.5147,
+      "step": 200
     },
     {
-      "epoch": 1.8235294117647058,
-      "step": 16,
-      "total_flos": 1.1336724214972416e+16,
-      "train_loss": 0.8289451897144318,
-      "train_runtime": 1583.6827,
-      "train_samples_per_second": 0.17,
-      "train_steps_per_second": 0.01
+      "epoch": 22.235294117647058,
+      "eval_loss": 0.6163371205329895,
+      "eval_runtime": 3.3629,
+      "eval_samples_per_second": 4.46,
+      "eval_steps_per_second": 0.595,
+      "step": 200
     }
   ],
-  "logging_steps": 10,
-  "max_steps": 16,
+  "logging_steps": 5,
+  "max_steps": 200,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 2,
+  "num_train_epochs": 25,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -58,7 +490,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1.1336724214972416e+16,
+  "total_flos": 1.0471069689549005e+17,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:08d02580fa61a9f76991a83deddc72fb2290b7a3aa24cadbb04f537cefeeef71
-size 5304
+oid sha256:354b637532320af3e4fc7a75a7a30ab3076b3ef28de912201c92125d861c2822
+size 5624