Hugging Face's logo

VERSIL91
/
1dfee041-1016-4808-ba36-103e1b92cd54

PEFT
Safetensors
mistral
axolotl
Generated from Trainer
Model card Files Community
VERSIL91 commited on
Commit
3e7cd5e
·
verified ·
1 Parent(s): 362060c

End of training

Browse files
Files changed (9) hide show
  1. README.md +17 -32
  2. adapter_config.json +3 -3
  3. last-checkpoint/adapter_config.json +3 -3
  4. last-checkpoint/optimizer.pt +2 -2
  5. last-checkpoint/rng_state.pth +1 -1
  6. last-checkpoint/scheduler.pt +1 -1
  7. last-checkpoint/trainer_state.json +51 -689
  8. last-checkpoint/training_args.bin +1 -1
  9. training_args.bin +1 -1
README.md CHANGED
@@ -6,7 +6,7 @@ tags:
6
  - axolotl
7
  - generated_from_trainer
8
  model-index:
9
- - name: 8cdb6276-aeed-49f9-8ffa-9eea475835ec
10
  results: []
11
  ---
12
 
@@ -18,12 +18,6 @@ should probably proofread and complete it, then remove this comment. -->
18
 
19
  axolotl version: `0.4.1`
20
  ```yaml
21
- accelerate_config:
22
- dynamo_backend: inductor
23
- mixed_precision: bf16
24
- num_machines: 1
25
- num_processes: auto
26
- use_cpu: false
27
  adapter: lora
28
  base_model: echarlaix/tiny-random-mistral
29
  bf16: auto
@@ -45,7 +39,6 @@ datasets:
45
  system_prompt: ''
46
  debug: null
47
  deepspeed: null
48
- device_map: auto
49
  early_stopping_patience: null
50
  eval_max_new_tokens: 128
51
  eval_table_size: null
@@ -54,14 +47,16 @@ flash_attention: false
54
  fp16: null
55
  fsdp: null
56
  fsdp_config: null
57
- gradient_accumulation_steps: 16
58
- gradient_checkpointing: true
59
  group_by_length: false
60
  hub_model_id: null
61
  hub_repo: null
62
  hub_strategy: checkpoint
63
  hub_token: null
64
- learning_rate: 0.0001
 
 
65
  local_rank: null
66
  logging_steps: 1
67
  lora_alpha: 16
@@ -70,13 +65,8 @@ lora_fan_in_fan_out: null
70
  lora_model_dir: null
71
  lora_r: 8
72
  lora_target_linear: true
73
- lora_target_modules:
74
- - q_proj
75
- - v_proj
76
  lr_scheduler: cosine
77
- max_memory:
78
- 0: 70GiB
79
- max_steps: 100
80
  micro_batch_size: 2
81
  mlflow_experiment_name: /tmp/2e4f87c3b388cefd_train_data.json
82
  model_type: AutoModelForCausalLM
@@ -84,9 +74,6 @@ num_epochs: 1
84
  optimizer: adamw_bnb_8bit
85
  output_dir: miner_id_24
86
  pad_to_sequence_len: true
87
- quantization_config:
88
- llm_int8_enable_fp32_cpu_offload: true
89
- load_in_8bit: true
90
  resume_from_checkpoint: null
91
  s2_attention: null
92
  sample_packing: false
@@ -97,14 +84,13 @@ special_tokens:
97
  strict: false
98
  tf32: false
99
  tokenizer_type: AutoTokenizer
100
- torch_compile: true
101
  train_on_inputs: false
102
  trust_remote_code: true
103
  val_set_size: 0.05
104
  wandb_entity: null
105
  wandb_mode: online
106
  wandb_name: 8cdb6276-aeed-49f9-8ffa-9eea475835ec
107
- wandb_project: Gradients-On-Demand
108
  wandb_run: your_name
109
  wandb_runid: 8cdb6276-aeed-49f9-8ffa-9eea475835ec
110
  warmup_steps: 10
@@ -115,7 +101,7 @@ xformers_attention: null
115
 
116
  </details><br>
117
 
118
- # 8cdb6276-aeed-49f9-8ffa-9eea475835ec
119
 
120
  This model is a fine-tuned version of [echarlaix/tiny-random-mistral](https://huggingface.co/echarlaix/tiny-random-mistral) on the None dataset.
121
  It achieves the following results on the evaluation set:
@@ -138,26 +124,25 @@ More information needed
138
  ### Training hyperparameters
139
 
140
  The following hyperparameters were used during training:
141
- - learning_rate: 0.0001
142
  - train_batch_size: 2
143
  - eval_batch_size: 2
144
  - seed: 42
145
- - gradient_accumulation_steps: 16
146
- - total_train_batch_size: 32
147
  - optimizer: Use OptimizerNames.ADAMW_BNB with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
148
  - lr_scheduler_type: cosine
149
  - lr_scheduler_warmup_steps: 10
150
- - training_steps: 100
151
 
152
  ### Training results
153
 
154
  | Training Loss | Epoch | Step | Validation Loss |
155
  |:-------------:|:------:|:----:|:---------------:|
156
- | 0.0 | 0.0038 | 1 | nan |
157
- | 0.0 | 0.0946 | 25 | nan |
158
- | 0.0 | 0.1892 | 50 | nan |
159
- | 0.0 | 0.2838 | 75 | nan |
160
- | 0.0 | 0.3784 | 100 | nan |
161
 
162
 
163
  ### Framework versions
 
6
  - axolotl
7
  - generated_from_trainer
8
  model-index:
9
+ - name: 7e5f69a5-09fd-4c79-abe8-7f4ff3f6b7e5
10
  results: []
11
  ---
12
 
 
18
 
19
  axolotl version: `0.4.1`
20
  ```yaml
 
 
 
 
 
 
21
  adapter: lora
22
  base_model: echarlaix/tiny-random-mistral
23
  bf16: auto
 
39
  system_prompt: ''
40
  debug: null
41
  deepspeed: null
 
42
  early_stopping_patience: null
43
  eval_max_new_tokens: 128
44
  eval_table_size: null
 
47
  fp16: null
48
  fsdp: null
49
  fsdp_config: null
50
+ gradient_accumulation_steps: 4
51
+ gradient_checkpointing: false
52
  group_by_length: false
53
  hub_model_id: null
54
  hub_repo: null
55
  hub_strategy: checkpoint
56
  hub_token: null
57
+ learning_rate: 0.0002
58
+ load_in_4bit: false
59
+ load_in_8bit: false
60
  local_rank: null
61
  logging_steps: 1
62
  lora_alpha: 16
 
65
  lora_model_dir: null
66
  lora_r: 8
67
  lora_target_linear: true
 
 
 
68
  lr_scheduler: cosine
69
+ max_steps: 10
 
 
70
  micro_batch_size: 2
71
  mlflow_experiment_name: /tmp/2e4f87c3b388cefd_train_data.json
72
  model_type: AutoModelForCausalLM
 
74
  optimizer: adamw_bnb_8bit
75
  output_dir: miner_id_24
76
  pad_to_sequence_len: true
 
 
 
77
  resume_from_checkpoint: null
78
  s2_attention: null
79
  sample_packing: false
 
84
  strict: false
85
  tf32: false
86
  tokenizer_type: AutoTokenizer
 
87
  train_on_inputs: false
88
  trust_remote_code: true
89
  val_set_size: 0.05
90
  wandb_entity: null
91
  wandb_mode: online
92
  wandb_name: 8cdb6276-aeed-49f9-8ffa-9eea475835ec
93
+ wandb_project: Birthday-SN56-7-Gradients-On-Demand
94
  wandb_run: your_name
95
  wandb_runid: 8cdb6276-aeed-49f9-8ffa-9eea475835ec
96
  warmup_steps: 10
 
101
 
102
  </details><br>
103
 
104
+ # 7e5f69a5-09fd-4c79-abe8-7f4ff3f6b7e5
105
 
106
  This model is a fine-tuned version of [echarlaix/tiny-random-mistral](https://huggingface.co/echarlaix/tiny-random-mistral) on the None dataset.
107
  It achieves the following results on the evaluation set:
 
124
  ### Training hyperparameters
125
 
126
  The following hyperparameters were used during training:
127
+ - learning_rate: 0.0002
128
  - train_batch_size: 2
129
  - eval_batch_size: 2
130
  - seed: 42
131
+ - gradient_accumulation_steps: 4
132
+ - total_train_batch_size: 8
133
  - optimizer: Use OptimizerNames.ADAMW_BNB with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
134
  - lr_scheduler_type: cosine
135
  - lr_scheduler_warmup_steps: 10
136
+ - training_steps: 10
137
 
138
  ### Training results
139
 
140
  | Training Loss | Epoch | Step | Validation Loss |
141
  |:-------------:|:------:|:----:|:---------------:|
142
+ | 0.0 | 0.0009 | 1 | nan |
143
+ | 0.0 | 0.0028 | 3 | nan |
144
+ | 0.0 | 0.0057 | 6 | nan |
145
+ | 0.0 | 0.0085 | 9 | nan |
 
146
 
147
 
148
  ### Framework versions
adapter_config.json CHANGED
@@ -20,13 +20,13 @@
20
  "rank_pattern": {},
21
  "revision": null,
22
  "target_modules": [
23
- "gate_proj",
24
- "v_proj",
25
  "k_proj",
26
  "o_proj",
 
27
  "down_proj",
28
  "up_proj",
29
- "q_proj"
 
30
  ],
31
  "task_type": "CAUSAL_LM",
32
  "use_dora": false,
 
20
  "rank_pattern": {},
21
  "revision": null,
22
  "target_modules": [
 
 
23
  "k_proj",
24
  "o_proj",
25
+ "gate_proj",
26
  "down_proj",
27
  "up_proj",
28
+ "q_proj",
29
+ "v_proj"
30
  ],
31
  "task_type": "CAUSAL_LM",
32
  "use_dora": false,
last-checkpoint/adapter_config.json CHANGED
@@ -20,13 +20,13 @@
20
  "rank_pattern": {},
21
  "revision": null,
22
  "target_modules": [
23
- "gate_proj",
24
- "v_proj",
25
  "k_proj",
26
  "o_proj",
 
27
  "down_proj",
28
  "up_proj",
29
- "q_proj"
 
30
  ],
31
  "task_type": "CAUSAL_LM",
32
  "use_dora": false,
 
20
  "rank_pattern": {},
21
  "revision": null,
22
  "target_modules": [
 
 
23
  "k_proj",
24
  "o_proj",
25
+ "gate_proj",
26
  "down_proj",
27
  "up_proj",
28
+ "q_proj",
29
+ "v_proj"
30
  ],
31
  "task_type": "CAUSAL_LM",
32
  "use_dora": false,
last-checkpoint/optimizer.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d9a94694054557f16b87aecce198941be522358d1af0ee78b964d87b97e4d230
3
- size 71654
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bbe10c074c53af62af0964d6b70d2e6528b906557b640a82231880d56e53359d
3
+ size 71718
last-checkpoint/rng_state.pth CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d6516708bef1a97795da04892fd3df3bf2305ac978924d9e7083e4eed6ef52b1
3
  size 14244
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c12066a9c624fe38430ff3feea2dc6451e9f1a920255c11680c737a33d2c53a0
3
  size 14244
last-checkpoint/scheduler.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:49d60a69e2379be2053e816cbaff31e6c931b5922dd86c71c9eaf473299cbf62
3
  size 1064
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bb578e75c11a81e85dda67a691f96ba4793a02960f1409fd3e1511aac873491a
3
  size 1064
last-checkpoint/trainer_state.json CHANGED
@@ -1,759 +1,121 @@
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
- "epoch": 0.3784295175023652,
5
- "eval_steps": 25,
6
- "global_step": 100,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
- "epoch": 0.003784295175023652,
13
  "grad_norm": NaN,
14
- "learning_rate": 1e-05,
15
  "loss": 0.0,
16
  "step": 1
17
  },
18
  {
19
- "epoch": 0.003784295175023652,
20
  "eval_loss": NaN,
21
- "eval_runtime": 22.8133,
22
- "eval_samples_per_second": 19.506,
23
- "eval_steps_per_second": 9.775,
24
  "step": 1
25
  },
26
  {
27
- "epoch": 0.007568590350047304,
28
- "grad_norm": NaN,
29
- "learning_rate": 2e-05,
30
- "loss": 0.0,
31
- "step": 2
32
- },
33
- {
34
- "epoch": 0.011352885525070956,
35
- "grad_norm": NaN,
36
- "learning_rate": 3e-05,
37
- "loss": 0.0,
38
- "step": 3
39
- },
40
- {
41
- "epoch": 0.015137180700094607,
42
  "grad_norm": NaN,
43
  "learning_rate": 4e-05,
44
  "loss": 0.0,
45
- "step": 4
46
- },
47
- {
48
- "epoch": 0.01892147587511826,
49
- "grad_norm": NaN,
50
- "learning_rate": 5e-05,
51
- "loss": 0.0,
52
- "step": 5
53
  },
54
  {
55
- "epoch": 0.02270577105014191,
56
  "grad_norm": NaN,
57
  "learning_rate": 6e-05,
58
  "loss": 0.0,
59
- "step": 6
60
  },
61
  {
62
- "epoch": 0.026490066225165563,
63
- "grad_norm": NaN,
64
- "learning_rate": 7e-05,
65
- "loss": 0.0,
66
- "step": 7
 
67
  },
68
  {
69
- "epoch": 0.030274361400189215,
70
  "grad_norm": NaN,
71
  "learning_rate": 8e-05,
72
  "loss": 0.0,
73
- "step": 8
74
- },
75
- {
76
- "epoch": 0.03405865657521287,
77
- "grad_norm": NaN,
78
- "learning_rate": 9e-05,
79
- "loss": 0.0,
80
- "step": 9
81
  },
82
  {
83
- "epoch": 0.03784295175023652,
84
  "grad_norm": NaN,
85
  "learning_rate": 0.0001,
86
  "loss": 0.0,
87
- "step": 10
88
- },
89
- {
90
- "epoch": 0.041627246925260174,
91
- "grad_norm": NaN,
92
- "learning_rate": 9.99695413509548e-05,
93
- "loss": 0.0,
94
- "step": 11
95
- },
96
- {
97
- "epoch": 0.04541154210028382,
98
- "grad_norm": NaN,
99
- "learning_rate": 9.987820251299122e-05,
100
- "loss": 0.0,
101
- "step": 12
102
- },
103
- {
104
- "epoch": 0.04919583727530748,
105
- "grad_norm": NaN,
106
- "learning_rate": 9.972609476841367e-05,
107
- "loss": 0.0,
108
- "step": 13
109
- },
110
- {
111
- "epoch": 0.052980132450331126,
112
- "grad_norm": NaN,
113
- "learning_rate": 9.951340343707852e-05,
114
- "loss": 0.0,
115
- "step": 14
116
- },
117
- {
118
- "epoch": 0.05676442762535478,
119
- "grad_norm": NaN,
120
- "learning_rate": 9.924038765061042e-05,
121
- "loss": 0.0,
122
- "step": 15
123
- },
124
- {
125
- "epoch": 0.06054872280037843,
126
- "grad_norm": NaN,
127
- "learning_rate": 9.890738003669029e-05,
128
- "loss": 0.0,
129
- "step": 16
130
- },
131
- {
132
- "epoch": 0.06433301797540208,
133
- "grad_norm": NaN,
134
- "learning_rate": 9.851478631379982e-05,
135
- "loss": 0.0,
136
- "step": 17
137
- },
138
- {
139
- "epoch": 0.06811731315042574,
140
- "grad_norm": NaN,
141
- "learning_rate": 9.806308479691595e-05,
142
- "loss": 0.0,
143
- "step": 18
144
- },
145
- {
146
- "epoch": 0.07190160832544938,
147
- "grad_norm": NaN,
148
- "learning_rate": 9.755282581475769e-05,
149
- "loss": 0.0,
150
- "step": 19
151
- },
152
- {
153
- "epoch": 0.07568590350047304,
154
- "grad_norm": NaN,
155
- "learning_rate": 9.698463103929542e-05,
156
- "loss": 0.0,
157
- "step": 20
158
- },
159
- {
160
- "epoch": 0.07947019867549669,
161
- "grad_norm": NaN,
162
- "learning_rate": 9.635919272833938e-05,
163
- "loss": 0.0,
164
- "step": 21
165
- },
166
- {
167
- "epoch": 0.08325449385052035,
168
- "grad_norm": NaN,
169
- "learning_rate": 9.567727288213005e-05,
170
- "loss": 0.0,
171
- "step": 22
172
- },
173
- {
174
- "epoch": 0.08703878902554399,
175
- "grad_norm": NaN,
176
- "learning_rate": 9.493970231495835e-05,
177
- "loss": 0.0,
178
- "step": 23
179
- },
180
- {
181
- "epoch": 0.09082308420056764,
182
- "grad_norm": NaN,
183
- "learning_rate": 9.414737964294636e-05,
184
- "loss": 0.0,
185
- "step": 24
186
- },
187
- {
188
- "epoch": 0.0946073793755913,
189
- "grad_norm": NaN,
190
- "learning_rate": 9.330127018922194e-05,
191
- "loss": 0.0,
192
- "step": 25
193
- },
194
- {
195
- "epoch": 0.0946073793755913,
196
- "eval_loss": NaN,
197
- "eval_runtime": 1.6494,
198
- "eval_samples_per_second": 269.802,
199
- "eval_steps_per_second": 135.204,
200
- "step": 25
201
- },
202
- {
203
- "epoch": 0.09839167455061495,
204
- "grad_norm": NaN,
205
- "learning_rate": 9.24024048078213e-05,
206
- "loss": 0.0,
207
- "step": 26
208
- },
209
- {
210
- "epoch": 0.1021759697256386,
211
- "grad_norm": NaN,
212
- "learning_rate": 9.145187862775209e-05,
213
- "loss": 0.0,
214
- "step": 27
215
- },
216
- {
217
- "epoch": 0.10596026490066225,
218
- "grad_norm": NaN,
219
- "learning_rate": 9.045084971874738e-05,
220
- "loss": 0.0,
221
- "step": 28
222
- },
223
- {
224
- "epoch": 0.1097445600756859,
225
- "grad_norm": NaN,
226
- "learning_rate": 8.940053768033609e-05,
227
- "loss": 0.0,
228
- "step": 29
229
- },
230
- {
231
- "epoch": 0.11352885525070956,
232
- "grad_norm": NaN,
233
- "learning_rate": 8.83022221559489e-05,
234
- "loss": 0.0,
235
- "step": 30
236
- },
237
- {
238
- "epoch": 0.1173131504257332,
239
- "grad_norm": NaN,
240
- "learning_rate": 8.715724127386972e-05,
241
- "loss": 0.0,
242
- "step": 31
243
- },
244
- {
245
- "epoch": 0.12109744560075686,
246
- "grad_norm": NaN,
247
- "learning_rate": 8.596699001693255e-05,
248
- "loss": 0.0,
249
- "step": 32
250
- },
251
- {
252
- "epoch": 0.12488174077578051,
253
- "grad_norm": NaN,
254
- "learning_rate": 8.473291852294987e-05,
255
- "loss": 0.0,
256
- "step": 33
257
- },
258
- {
259
- "epoch": 0.12866603595080417,
260
- "grad_norm": NaN,
261
- "learning_rate": 8.345653031794292e-05,
262
- "loss": 0.0,
263
- "step": 34
264
- },
265
- {
266
- "epoch": 0.13245033112582782,
267
- "grad_norm": NaN,
268
- "learning_rate": 8.213938048432697e-05,
269
- "loss": 0.0,
270
- "step": 35
271
- },
272
- {
273
- "epoch": 0.13623462630085148,
274
- "grad_norm": NaN,
275
- "learning_rate": 8.07830737662829e-05,
276
- "loss": 0.0,
277
- "step": 36
278
- },
279
- {
280
- "epoch": 0.1400189214758751,
281
- "grad_norm": NaN,
282
- "learning_rate": 7.938926261462366e-05,
283
- "loss": 0.0,
284
- "step": 37
285
- },
286
- {
287
- "epoch": 0.14380321665089876,
288
- "grad_norm": NaN,
289
- "learning_rate": 7.795964517353735e-05,
290
- "loss": 0.0,
291
- "step": 38
292
- },
293
- {
294
- "epoch": 0.14758751182592242,
295
- "grad_norm": NaN,
296
- "learning_rate": 7.649596321166024e-05,
297
- "loss": 0.0,
298
- "step": 39
299
- },
300
- {
301
- "epoch": 0.15137180700094607,
302
- "grad_norm": NaN,
303
- "learning_rate": 7.500000000000001e-05,
304
- "loss": 0.0,
305
- "step": 40
306
- },
307
- {
308
- "epoch": 0.15515610217596973,
309
- "grad_norm": NaN,
310
- "learning_rate": 7.347357813929454e-05,
311
- "loss": 0.0,
312
- "step": 41
313
- },
314
- {
315
- "epoch": 0.15894039735099338,
316
- "grad_norm": NaN,
317
- "learning_rate": 7.191855733945387e-05,
318
- "loss": 0.0,
319
- "step": 42
320
- },
321
- {
322
- "epoch": 0.16272469252601704,
323
- "grad_norm": NaN,
324
- "learning_rate": 7.033683215379002e-05,
325
- "loss": 0.0,
326
- "step": 43
327
- },
328
- {
329
- "epoch": 0.1665089877010407,
330
- "grad_norm": NaN,
331
- "learning_rate": 6.873032967079561e-05,
332
- "loss": 0.0,
333
- "step": 44
334
- },
335
- {
336
- "epoch": 0.17029328287606432,
337
- "grad_norm": NaN,
338
- "learning_rate": 6.710100716628344e-05,
339
- "loss": 0.0,
340
- "step": 45
341
- },
342
- {
343
- "epoch": 0.17407757805108798,
344
- "grad_norm": NaN,
345
- "learning_rate": 6.545084971874738e-05,
346
- "loss": 0.0,
347
- "step": 46
348
- },
349
- {
350
- "epoch": 0.17786187322611163,
351
- "grad_norm": NaN,
352
- "learning_rate": 6.378186779084995e-05,
353
- "loss": 0.0,
354
- "step": 47
355
- },
356
- {
357
- "epoch": 0.1816461684011353,
358
- "grad_norm": NaN,
359
- "learning_rate": 6.209609477998338e-05,
360
- "loss": 0.0,
361
- "step": 48
362
- },
363
- {
364
- "epoch": 0.18543046357615894,
365
- "grad_norm": NaN,
366
- "learning_rate": 6.0395584540887963e-05,
367
- "loss": 0.0,
368
- "step": 49
369
  },
370
  {
371
- "epoch": 0.1892147587511826,
372
  "grad_norm": NaN,
373
- "learning_rate": 5.868240888334653e-05,
374
  "loss": 0.0,
375
- "step": 50
376
  },
377
  {
378
- "epoch": 0.1892147587511826,
379
  "eval_loss": NaN,
380
- "eval_runtime": 1.6547,
381
- "eval_samples_per_second": 268.932,
382
- "eval_steps_per_second": 134.768,
383
- "step": 50
384
- },
385
- {
386
- "epoch": 0.19299905392620625,
387
- "grad_norm": NaN,
388
- "learning_rate": 5.695865504800327e-05,
389
- "loss": 0.0,
390
- "step": 51
391
- },
392
- {
393
- "epoch": 0.1967833491012299,
394
- "grad_norm": NaN,
395
- "learning_rate": 5.522642316338268e-05,
396
- "loss": 0.0,
397
- "step": 52
398
- },
399
- {
400
- "epoch": 0.20056764427625354,
401
- "grad_norm": NaN,
402
- "learning_rate": 5.348782368720626e-05,
403
- "loss": 0.0,
404
- "step": 53
405
- },
406
- {
407
- "epoch": 0.2043519394512772,
408
- "grad_norm": NaN,
409
- "learning_rate": 5.174497483512506e-05,
410
- "loss": 0.0,
411
- "step": 54
412
- },
413
- {
414
- "epoch": 0.20813623462630085,
415
- "grad_norm": NaN,
416
- "learning_rate": 5e-05,
417
- "loss": 0.0,
418
- "step": 55
419
- },
420
- {
421
- "epoch": 0.2119205298013245,
422
- "grad_norm": NaN,
423
- "learning_rate": 4.825502516487497e-05,
424
- "loss": 0.0,
425
- "step": 56
426
- },
427
- {
428
- "epoch": 0.21570482497634816,
429
- "grad_norm": NaN,
430
- "learning_rate": 4.6512176312793736e-05,
431
- "loss": 0.0,
432
- "step": 57
433
- },
434
- {
435
- "epoch": 0.2194891201513718,
436
- "grad_norm": NaN,
437
- "learning_rate": 4.477357683661734e-05,
438
- "loss": 0.0,
439
- "step": 58
440
- },
441
- {
442
- "epoch": 0.22327341532639547,
443
- "grad_norm": NaN,
444
- "learning_rate": 4.3041344951996746e-05,
445
- "loss": 0.0,
446
- "step": 59
447
- },
448
- {
449
- "epoch": 0.22705771050141912,
450
- "grad_norm": NaN,
451
- "learning_rate": 4.131759111665349e-05,
452
- "loss": 0.0,
453
- "step": 60
454
- },
455
- {
456
- "epoch": 0.23084200567644275,
457
- "grad_norm": NaN,
458
- "learning_rate": 3.960441545911204e-05,
459
- "loss": 0.0,
460
- "step": 61
461
- },
462
- {
463
- "epoch": 0.2346263008514664,
464
- "grad_norm": NaN,
465
- "learning_rate": 3.790390522001662e-05,
466
- "loss": 0.0,
467
- "step": 62
468
- },
469
- {
470
- "epoch": 0.23841059602649006,
471
- "grad_norm": NaN,
472
- "learning_rate": 3.6218132209150045e-05,
473
- "loss": 0.0,
474
- "step": 63
475
- },
476
- {
477
- "epoch": 0.24219489120151372,
478
- "grad_norm": NaN,
479
- "learning_rate": 3.4549150281252636e-05,
480
- "loss": 0.0,
481
- "step": 64
482
- },
483
- {
484
- "epoch": 0.24597918637653737,
485
- "grad_norm": NaN,
486
- "learning_rate": 3.289899283371657e-05,
487
- "loss": 0.0,
488
- "step": 65
489
- },
490
- {
491
- "epoch": 0.24976348155156103,
492
- "grad_norm": NaN,
493
- "learning_rate": 3.12696703292044e-05,
494
- "loss": 0.0,
495
- "step": 66
496
- },
497
- {
498
- "epoch": 0.2535477767265847,
499
- "grad_norm": NaN,
500
- "learning_rate": 2.9663167846209998e-05,
501
- "loss": 0.0,
502
- "step": 67
503
- },
504
- {
505
- "epoch": 0.25733207190160834,
506
- "grad_norm": NaN,
507
- "learning_rate": 2.8081442660546125e-05,
508
- "loss": 0.0,
509
- "step": 68
510
- },
511
- {
512
- "epoch": 0.261116367076632,
513
- "grad_norm": NaN,
514
- "learning_rate": 2.6526421860705473e-05,
515
- "loss": 0.0,
516
- "step": 69
517
- },
518
- {
519
- "epoch": 0.26490066225165565,
520
- "grad_norm": NaN,
521
- "learning_rate": 2.500000000000001e-05,
522
- "loss": 0.0,
523
- "step": 70
524
- },
525
- {
526
- "epoch": 0.2686849574266793,
527
- "grad_norm": NaN,
528
- "learning_rate": 2.350403678833976e-05,
529
- "loss": 0.0,
530
- "step": 71
531
- },
532
- {
533
- "epoch": 0.27246925260170296,
534
- "grad_norm": NaN,
535
- "learning_rate": 2.2040354826462668e-05,
536
- "loss": 0.0,
537
- "step": 72
538
  },
539
  {
540
- "epoch": 0.27625354777672656,
541
  "grad_norm": NaN,
542
- "learning_rate": 2.061073738537635e-05,
543
  "loss": 0.0,
544
- "step": 73
545
  },
546
  {
547
- "epoch": 0.2800378429517502,
548
  "grad_norm": NaN,
549
- "learning_rate": 1.9216926233717085e-05,
550
  "loss": 0.0,
551
- "step": 74
552
  },
553
  {
554
- "epoch": 0.28382213812677387,
555
  "grad_norm": NaN,
556
- "learning_rate": 1.7860619515673033e-05,
557
  "loss": 0.0,
558
- "step": 75
559
  },
560
  {
561
- "epoch": 0.28382213812677387,
562
  "eval_loss": NaN,
563
- "eval_runtime": 1.6574,
564
- "eval_samples_per_second": 268.495,
565
- "eval_steps_per_second": 134.549,
566
- "step": 75
567
- },
568
- {
569
- "epoch": 0.2876064333017975,
570
- "grad_norm": NaN,
571
- "learning_rate": 1.6543469682057106e-05,
572
- "loss": 0.0,
573
- "step": 76
574
- },
575
- {
576
- "epoch": 0.2913907284768212,
577
- "grad_norm": NaN,
578
- "learning_rate": 1.526708147705013e-05,
579
- "loss": 0.0,
580
- "step": 77
581
- },
582
- {
583
- "epoch": 0.29517502365184484,
584
- "grad_norm": NaN,
585
- "learning_rate": 1.4033009983067452e-05,
586
- "loss": 0.0,
587
- "step": 78
588
- },
589
- {
590
- "epoch": 0.2989593188268685,
591
- "grad_norm": NaN,
592
- "learning_rate": 1.2842758726130283e-05,
593
- "loss": 0.0,
594
- "step": 79
595
- },
596
- {
597
- "epoch": 0.30274361400189215,
598
- "grad_norm": NaN,
599
- "learning_rate": 1.1697777844051105e-05,
600
- "loss": 0.0,
601
- "step": 80
602
- },
603
- {
604
- "epoch": 0.3065279091769158,
605
- "grad_norm": NaN,
606
- "learning_rate": 1.0599462319663905e-05,
607
- "loss": 0.0,
608
- "step": 81
609
- },
610
- {
611
- "epoch": 0.31031220435193946,
612
- "grad_norm": NaN,
613
- "learning_rate": 9.549150281252633e-06,
614
- "loss": 0.0,
615
- "step": 82
616
- },
617
- {
618
- "epoch": 0.3140964995269631,
619
- "grad_norm": NaN,
620
- "learning_rate": 8.548121372247918e-06,
621
- "loss": 0.0,
622
- "step": 83
623
- },
624
- {
625
- "epoch": 0.31788079470198677,
626
- "grad_norm": NaN,
627
- "learning_rate": 7.597595192178702e-06,
628
- "loss": 0.0,
629
- "step": 84
630
- },
631
- {
632
- "epoch": 0.3216650898770104,
633
- "grad_norm": NaN,
634
- "learning_rate": 6.698729810778065e-06,
635
- "loss": 0.0,
636
- "step": 85
637
- },
638
- {
639
- "epoch": 0.3254493850520341,
640
- "grad_norm": NaN,
641
- "learning_rate": 5.852620357053651e-06,
642
- "loss": 0.0,
643
- "step": 86
644
- },
645
- {
646
- "epoch": 0.32923368022705773,
647
- "grad_norm": NaN,
648
- "learning_rate": 5.060297685041659e-06,
649
- "loss": 0.0,
650
- "step": 87
651
- },
652
- {
653
- "epoch": 0.3330179754020814,
654
- "grad_norm": NaN,
655
- "learning_rate": 4.322727117869951e-06,
656
- "loss": 0.0,
657
- "step": 88
658
- },
659
- {
660
- "epoch": 0.336802270577105,
661
- "grad_norm": NaN,
662
- "learning_rate": 3.6408072716606346e-06,
663
- "loss": 0.0,
664
- "step": 89
665
- },
666
- {
667
- "epoch": 0.34058656575212864,
668
- "grad_norm": NaN,
669
- "learning_rate": 3.0153689607045845e-06,
670
- "loss": 0.0,
671
- "step": 90
672
- },
673
- {
674
- "epoch": 0.3443708609271523,
675
- "grad_norm": NaN,
676
- "learning_rate": 2.4471741852423237e-06,
677
- "loss": 0.0,
678
- "step": 91
679
- },
680
- {
681
- "epoch": 0.34815515610217596,
682
- "grad_norm": NaN,
683
- "learning_rate": 1.9369152030840556e-06,
684
- "loss": 0.0,
685
- "step": 92
686
- },
687
- {
688
- "epoch": 0.3519394512771996,
689
- "grad_norm": NaN,
690
- "learning_rate": 1.4852136862001764e-06,
691
- "loss": 0.0,
692
- "step": 93
693
- },
694
- {
695
- "epoch": 0.35572374645222327,
696
- "grad_norm": NaN,
697
- "learning_rate": 1.0926199633097157e-06,
698
- "loss": 0.0,
699
- "step": 94
700
- },
701
- {
702
- "epoch": 0.3595080416272469,
703
- "grad_norm": NaN,
704
- "learning_rate": 7.596123493895991e-07,
705
- "loss": 0.0,
706
- "step": 95
707
- },
708
- {
709
- "epoch": 0.3632923368022706,
710
- "grad_norm": NaN,
711
- "learning_rate": 4.865965629214819e-07,
712
- "loss": 0.0,
713
- "step": 96
714
- },
715
- {
716
- "epoch": 0.36707663197729423,
717
- "grad_norm": NaN,
718
- "learning_rate": 2.7390523158633554e-07,
719
- "loss": 0.0,
720
- "step": 97
721
- },
722
- {
723
- "epoch": 0.3708609271523179,
724
- "grad_norm": NaN,
725
- "learning_rate": 1.2179748700879012e-07,
726
- "loss": 0.0,
727
- "step": 98
728
- },
729
- {
730
- "epoch": 0.37464522232734154,
731
- "grad_norm": NaN,
732
- "learning_rate": 3.04586490452119e-08,
733
- "loss": 0.0,
734
- "step": 99
735
  },
736
  {
737
- "epoch": 0.3784295175023652,
738
  "grad_norm": NaN,
739
- "learning_rate": 0.0,
740
  "loss": 0.0,
741
- "step": 100
742
- },
743
- {
744
- "epoch": 0.3784295175023652,
745
- "eval_loss": NaN,
746
- "eval_runtime": 1.6516,
747
- "eval_samples_per_second": 269.436,
748
- "eval_steps_per_second": 135.021,
749
- "step": 100
750
  }
751
  ],
752
  "logging_steps": 1,
753
- "max_steps": 100,
754
  "num_input_tokens_seen": 0,
755
  "num_train_epochs": 1,
756
- "save_steps": 25,
757
  "stateful_callbacks": {
758
  "TrainerControl": {
759
  "args": {
@@ -766,7 +128,7 @@
766
  "attributes": {}
767
  }
768
  },
769
- "total_flos": 10265926041600.0,
770
  "train_batch_size": 2,
771
  "trial_name": null,
772
  "trial_params": null
 
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
+ "epoch": 0.00946073793755913,
5
+ "eval_steps": 3,
6
+ "global_step": 10,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
+ "epoch": 0.000946073793755913,
13
  "grad_norm": NaN,
14
+ "learning_rate": 2e-05,
15
  "loss": 0.0,
16
  "step": 1
17
  },
18
  {
19
+ "epoch": 0.000946073793755913,
20
  "eval_loss": NaN,
21
+ "eval_runtime": 2.1382,
22
+ "eval_samples_per_second": 208.123,
23
+ "eval_steps_per_second": 104.296,
24
  "step": 1
25
  },
26
  {
27
+ "epoch": 0.001892147587511826,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
  "grad_norm": NaN,
29
  "learning_rate": 4e-05,
30
  "loss": 0.0,
31
+ "step": 2
 
 
 
 
 
 
 
32
  },
33
  {
34
+ "epoch": 0.002838221381267739,
35
  "grad_norm": NaN,
36
  "learning_rate": 6e-05,
37
  "loss": 0.0,
38
+ "step": 3
39
  },
40
  {
41
+ "epoch": 0.002838221381267739,
42
+ "eval_loss": NaN,
43
+ "eval_runtime": 1.2299,
44
+ "eval_samples_per_second": 361.829,
45
+ "eval_steps_per_second": 181.321,
46
+ "step": 3
47
  },
48
  {
49
+ "epoch": 0.003784295175023652,
50
  "grad_norm": NaN,
51
  "learning_rate": 8e-05,
52
  "loss": 0.0,
53
+ "step": 4
 
 
 
 
 
 
 
54
  },
55
  {
56
+ "epoch": 0.004730368968779565,
57
  "grad_norm": NaN,
58
  "learning_rate": 0.0001,
59
  "loss": 0.0,
60
+ "step": 5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
61
  },
62
  {
63
+ "epoch": 0.005676442762535478,
64
  "grad_norm": NaN,
65
+ "learning_rate": 0.00012,
66
  "loss": 0.0,
67
+ "step": 6
68
  },
69
  {
70
+ "epoch": 0.005676442762535478,
71
  "eval_loss": NaN,
72
+ "eval_runtime": 1.2783,
73
+ "eval_samples_per_second": 348.122,
74
+ "eval_steps_per_second": 174.452,
75
+ "step": 6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
76
  },
77
  {
78
+ "epoch": 0.006622516556291391,
79
  "grad_norm": NaN,
80
+ "learning_rate": 0.00014,
81
  "loss": 0.0,
82
+ "step": 7
83
  },
84
  {
85
+ "epoch": 0.007568590350047304,
86
  "grad_norm": NaN,
87
+ "learning_rate": 0.00016,
88
  "loss": 0.0,
89
+ "step": 8
90
  },
91
  {
92
+ "epoch": 0.008514664143803218,
93
  "grad_norm": NaN,
94
+ "learning_rate": 0.00018,
95
  "loss": 0.0,
96
+ "step": 9
97
  },
98
  {
99
+ "epoch": 0.008514664143803218,
100
  "eval_loss": NaN,
101
+ "eval_runtime": 1.2302,
102
+ "eval_samples_per_second": 361.741,
103
+ "eval_steps_per_second": 181.277,
104
+ "step": 9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
105
  },
106
  {
107
+ "epoch": 0.00946073793755913,
108
  "grad_norm": NaN,
109
+ "learning_rate": 0.0002,
110
  "loss": 0.0,
111
+ "step": 10
 
 
 
 
 
 
 
 
112
  }
113
  ],
114
  "logging_steps": 1,
115
+ "max_steps": 10,
116
  "num_input_tokens_seen": 0,
117
  "num_train_epochs": 1,
118
+ "save_steps": 3,
119
  "stateful_callbacks": {
120
  "TrainerControl": {
121
  "args": {
 
128
  "attributes": {}
129
  }
130
  },
131
+ "total_flos": 256648151040.0,
132
  "train_batch_size": 2,
133
  "trial_name": null,
134
  "trial_params": null
last-checkpoint/training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:dc00a92cfc7c8c98b0bc5d7150d56efdb7b38c866732e449bb7d7de115fa289e
3
  size 6776
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5008c806fe2aaf7df7590c5412697e0c6dd5a1a55326923e59d716f7c97f2264
3
  size 6776
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:dc00a92cfc7c8c98b0bc5d7150d56efdb7b38c866732e449bb7d7de115fa289e
3
  size 6776
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5008c806fe2aaf7df7590c5412697e0c6dd5a1a55326923e59d716f7c97f2264
3
  size 6776