Hugging Face's logo

eeeebbb2
/
4686d268-dae0-47a7-9278-3db4f7d4d27d

PEFT
Safetensors
bloom
axolotl
Generated from Trainer
Model card Files Community
eeeebbb2 commited on
Commit
2ed194d
·
verified ·
1 Parent(s): 672d4fa

Training in progress, step 10, checkpoint

Browse files
Files changed (7) hide show
  1. last-checkpoint/adapter_config.json +3 -3
  2. last-checkpoint/adapter_model.safetensors +2 -2
  3. last-checkpoint/optimizer.pt +2 -2
  4. last-checkpoint/rng_state.pth +1 -1
  5. last-checkpoint/scheduler.pt +1 -1
  6. last-checkpoint/trainer_state.json +71 -359
  7. last-checkpoint/training_args.bin +1 -1
last-checkpoint/adapter_config.json CHANGED
@@ -13,19 +13,19 @@
13
  "layers_pattern": null,
14
  "layers_to_transform": null,
15
  "loftq_config": {},
16
- "lora_alpha": 32,
17
  "lora_bias": false,
18
  "lora_dropout": 0.05,
19
  "megatron_config": null,
20
  "megatron_core": "megatron.core",
21
  "modules_to_save": null,
22
  "peft_type": "LORA",
23
- "r": 32,
24
  "rank_pattern": {},
25
  "revision": null,
26
  "target_modules": [
27
- "query_key_value",
28
  "dense_h_to_4h",
 
29
  "dense_4h_to_h",
30
  "dense"
31
  ],
 
13
  "layers_pattern": null,
14
  "layers_to_transform": null,
15
  "loftq_config": {},
16
+ "lora_alpha": 16,
17
  "lora_bias": false,
18
  "lora_dropout": 0.05,
19
  "megatron_config": null,
20
  "megatron_core": "megatron.core",
21
  "modules_to_save": null,
22
  "peft_type": "LORA",
23
+ "r": 8,
24
  "rank_pattern": {},
25
  "revision": null,
26
  "target_modules": [
 
27
  "dense_h_to_4h",
28
+ "query_key_value",
29
  "dense_4h_to_h",
30
  "dense"
31
  ],
last-checkpoint/adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:ec1e7f7080357ae1b40f9ee56ae2a99fec050e5eb03ae8aee9deb9d164ef8442
3
- size 50358592
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9bf3ee78ff7954c537473b5b92b51c5d0e5c4b638f9301d05c2bb6b011a0be49
3
+ size 12609416
last-checkpoint/optimizer.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:bad34bb5faf88755b582ec4a8a26413c67e8fbcd43f4ebaa62ca620faac55ea5
3
- size 25785082
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1aaa15706c7258c82b2a464928fd8c1631b846558518add41e3af4d58b870f04
3
+ size 6615034
last-checkpoint/rng_state.pth CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:faddcc1a1e6acbdf8ed8c7b668d60bf53ba1ed4967a43f5b9bd18b87a815b476
3
  size 14244
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4b306ff6af7ac66da2121236640857f4fe7bee27bc05133c9f102fda11571991
3
  size 14244
last-checkpoint/scheduler.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:ca7f86851fd43735ee13397eb65c40cf428084518569d17f272896226a512377
3
  size 1064
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:122701fae37d44bf3b93bc496977ee4c27719a4f2a6e6f874dbd449a810b7e2b
3
  size 1064
last-checkpoint/trainer_state.json CHANGED
@@ -2,393 +2,105 @@
2
  "best_global_step": null,
3
  "best_metric": null,
4
  "best_model_checkpoint": null,
5
- "epoch": 0.6357279084551812,
6
- "eval_steps": 250,
7
- "global_step": 500,
8
  "is_hyper_param_search": false,
9
  "is_local_process_zero": true,
10
  "is_world_process_zero": true,
11
  "log_history": [
12
  {
13
- "epoch": 0.0012714558169103624,
14
- "eval_loss": 4.115750789642334,
15
- "eval_runtime": 10.9188,
16
- "eval_samples_per_second": 60.721,
17
- "eval_steps_per_second": 15.203,
18
  "step": 1
19
  },
20
  {
21
- "epoch": 0.012714558169103624,
22
- "grad_norm": 4.603362560272217,
23
- "learning_rate": 9e-06,
24
- "loss": 3.5205,
25
- "step": 10
26
- },
27
- {
28
- "epoch": 0.02542911633820725,
29
- "grad_norm": 1.3208400011062622,
30
- "learning_rate": 1.9e-05,
31
- "loss": 3.4195,
32
- "step": 20
33
- },
34
- {
35
- "epoch": 0.03814367450731087,
36
- "grad_norm": 4.439665794372559,
37
- "learning_rate": 2.9e-05,
38
- "loss": 3.4599,
39
- "step": 30
40
- },
41
- {
42
- "epoch": 0.0508582326764145,
43
- "grad_norm": 10.967772483825684,
44
- "learning_rate": 3.9000000000000006e-05,
45
- "loss": 3.0734,
46
- "step": 40
47
- },
48
- {
49
- "epoch": 0.06357279084551812,
50
- "grad_norm": 0.7781662344932556,
51
- "learning_rate": 4.9e-05,
52
- "loss": 2.8237,
53
- "step": 50
54
- },
55
- {
56
- "epoch": 0.07628734901462174,
57
- "grad_norm": 0.8800532221794128,
58
- "learning_rate": 5.9e-05,
59
- "loss": 2.8408,
60
- "step": 60
61
- },
62
- {
63
- "epoch": 0.08900190718372536,
64
- "grad_norm": 0.7101799249649048,
65
- "learning_rate": 6.9e-05,
66
- "loss": 2.9109,
67
- "step": 70
68
- },
69
- {
70
- "epoch": 0.101716465352829,
71
- "grad_norm": 2.2402427196502686,
72
- "learning_rate": 7.900000000000001e-05,
73
- "loss": 2.7659,
74
- "step": 80
75
- },
76
- {
77
- "epoch": 0.11443102352193261,
78
- "grad_norm": 0.7569252252578735,
79
- "learning_rate": 8.900000000000001e-05,
80
- "loss": 2.9271,
81
- "step": 90
82
- },
83
- {
84
- "epoch": 0.12714558169103624,
85
- "grad_norm": 0.8399560451507568,
86
- "learning_rate": 9.900000000000001e-05,
87
- "loss": 2.778,
88
- "step": 100
89
- },
90
- {
91
- "epoch": 0.13986013986013987,
92
- "grad_norm": 11.03789234161377,
93
- "learning_rate": 9.99907764989806e-05,
94
- "loss": 2.6077,
95
- "step": 110
96
- },
97
- {
98
- "epoch": 0.15257469802924348,
99
- "grad_norm": 2.1647400856018066,
100
- "learning_rate": 9.995889716056251e-05,
101
- "loss": 2.6074,
102
- "step": 120
103
- },
104
- {
105
- "epoch": 0.1652892561983471,
106
- "grad_norm": 0.8638129830360413,
107
- "learning_rate": 9.990426263125004e-05,
108
- "loss": 2.5554,
109
- "step": 130
110
- },
111
- {
112
- "epoch": 0.17800381436745072,
113
- "grad_norm": 1.3435686826705933,
114
- "learning_rate": 9.98268977958828e-05,
115
- "loss": 2.5672,
116
- "step": 140
117
- },
118
- {
119
- "epoch": 0.19071837253655435,
120
- "grad_norm": 0.6855206489562988,
121
- "learning_rate": 9.972683789246043e-05,
122
- "loss": 2.6077,
123
- "step": 150
124
- },
125
- {
126
- "epoch": 0.203432930705658,
127
- "grad_norm": 1.058072566986084,
128
- "learning_rate": 9.960412849609227e-05,
129
- "loss": 2.5583,
130
- "step": 160
131
- },
132
- {
133
- "epoch": 0.2161474888747616,
134
- "grad_norm": 1.611879587173462,
135
- "learning_rate": 9.945882549823906e-05,
136
- "loss": 2.6053,
137
- "step": 170
138
- },
139
- {
140
- "epoch": 0.22886204704386523,
141
- "grad_norm": 0.9940386414527893,
142
- "learning_rate": 9.929099508125553e-05,
143
- "loss": 2.4841,
144
- "step": 180
145
- },
146
- {
147
- "epoch": 0.24157660521296884,
148
- "grad_norm": 0.8796589374542236,
149
- "learning_rate": 9.91007136882458e-05,
150
- "loss": 2.5184,
151
- "step": 190
152
- },
153
- {
154
- "epoch": 0.25429116338207247,
155
- "grad_norm": 1.0905311107635498,
156
- "learning_rate": 9.888806798824515e-05,
157
- "loss": 2.23,
158
- "step": 200
159
- },
160
- {
161
- "epoch": 0.2670057215511761,
162
- "grad_norm": 0.8602861762046814,
163
- "learning_rate": 9.865315483674425e-05,
164
- "loss": 2.4859,
165
- "step": 210
166
- },
167
- {
168
- "epoch": 0.27972027972027974,
169
- "grad_norm": 0.6362634301185608,
170
- "learning_rate": 9.839608123157347e-05,
171
- "loss": 2.4449,
172
- "step": 220
173
- },
174
- {
175
- "epoch": 0.29243483788938335,
176
- "grad_norm": 0.618522047996521,
177
- "learning_rate": 9.811696426416783e-05,
178
- "loss": 2.468,
179
- "step": 230
180
- },
181
- {
182
- "epoch": 0.30514939605848695,
183
- "grad_norm": 0.7691351771354675,
184
- "learning_rate": 9.781593106623432e-05,
185
- "loss": 2.5984,
186
- "step": 240
187
- },
188
- {
189
- "epoch": 0.3178639542275906,
190
- "grad_norm": 0.729053795337677,
191
- "learning_rate": 9.749311875184619e-05,
192
- "loss": 2.3813,
193
- "step": 250
194
- },
195
- {
196
- "epoch": 0.3178639542275906,
197
- "eval_loss": 2.4610917568206787,
198
- "eval_runtime": 10.9425,
199
- "eval_samples_per_second": 60.59,
200
- "eval_steps_per_second": 15.17,
201
- "step": 250
202
- },
203
- {
204
- "epoch": 0.3305785123966942,
205
- "grad_norm": 1.6031802892684937,
206
- "learning_rate": 9.714867435499044e-05,
207
- "loss": 2.6991,
208
- "step": 260
209
- },
210
- {
211
- "epoch": 0.34329307056579783,
212
- "grad_norm": 0.7818431854248047,
213
- "learning_rate": 9.678275476259714e-05,
214
- "loss": 2.34,
215
- "step": 270
216
- },
217
- {
218
- "epoch": 0.35600762873490144,
219
- "grad_norm": 0.8531989455223083,
220
- "learning_rate": 9.63955266430807e-05,
221
- "loss": 2.5596,
222
- "step": 280
223
- },
224
- {
225
- "epoch": 0.3687221869040051,
226
- "grad_norm": 0.8265013098716736,
227
- "learning_rate": 9.598716637042613e-05,
228
- "loss": 2.5515,
229
- "step": 290
230
- },
231
- {
232
- "epoch": 0.3814367450731087,
233
- "grad_norm": 1.119960904121399,
234
- "learning_rate": 9.555785994385451e-05,
235
- "loss": 2.4984,
236
- "step": 300
237
- },
238
- {
239
- "epoch": 0.3941513032422123,
240
- "grad_norm": 0.7961328625679016,
241
- "learning_rate": 9.510780290310442e-05,
242
- "loss": 2.5512,
243
- "step": 310
244
- },
245
- {
246
- "epoch": 0.406865861411316,
247
- "grad_norm": 1.6780835390090942,
248
- "learning_rate": 9.463720023936775e-05,
249
- "loss": 2.3739,
250
- "step": 320
251
- },
252
- {
253
- "epoch": 0.4195804195804196,
254
- "grad_norm": 1.1733721494674683,
255
- "learning_rate": 9.414626630192074e-05,
256
- "loss": 2.4161,
257
- "step": 330
258
- },
259
- {
260
- "epoch": 0.4322949777495232,
261
- "grad_norm": 0.8168957829475403,
262
- "learning_rate": 9.363522470049257e-05,
263
- "loss": 2.4176,
264
- "step": 340
265
- },
266
- {
267
- "epoch": 0.44500953591862685,
268
- "grad_norm": 1.0270198583602905,
269
- "learning_rate": 9.310430820341594e-05,
270
- "loss": 2.449,
271
- "step": 350
272
- },
273
- {
274
- "epoch": 0.45772409408773046,
275
- "grad_norm": 1.7757889032363892,
276
- "learning_rate": 9.255375863160616e-05,
277
- "loss": 2.4652,
278
- "step": 360
279
- },
280
- {
281
- "epoch": 0.47043865225683407,
282
- "grad_norm": 12.661322593688965,
283
- "learning_rate": 9.198382674841702e-05,
284
- "loss": 2.3101,
285
- "step": 370
286
- },
287
- {
288
- "epoch": 0.4831532104259377,
289
- "grad_norm": 18.186603546142578,
290
- "learning_rate": 9.139477214542349e-05,
291
- "loss": 2.4303,
292
- "step": 380
293
- },
294
- {
295
- "epoch": 0.49586776859504134,
296
- "grad_norm": 52.01583480834961,
297
- "learning_rate": 9.07868631241835e-05,
298
- "loss": 2.2535,
299
- "step": 390
300
- },
301
- {
302
- "epoch": 0.5085823267641449,
303
- "grad_norm": 64.61749267578125,
304
- "learning_rate": 9.016037657403224e-05,
305
- "loss": 2.5411,
306
- "step": 400
307
- },
308
- {
309
- "epoch": 0.5212968849332485,
310
- "grad_norm": 14.11105728149414,
311
- "learning_rate": 8.951559784596533e-05,
312
- "loss": 2.7646,
313
- "step": 410
314
  },
315
  {
316
- "epoch": 0.5340114431023522,
317
- "grad_norm": 401.874267578125,
318
- "learning_rate": 8.88528206226674e-05,
319
- "loss": 2.6372,
320
- "step": 420
321
  },
322
  {
323
- "epoch": 0.5467260012714558,
324
- "grad_norm": 229.18875122070312,
325
- "learning_rate": 8.817234678474622e-05,
326
- "loss": 2.6614,
327
- "step": 430
328
  },
329
  {
330
- "epoch": 0.5594405594405595,
331
- "grad_norm": 165.98165893554688,
332
- "learning_rate": 8.747448627323252e-05,
333
- "loss": 2.7541,
334
- "step": 440
335
  },
336
  {
337
- "epoch": 0.5721551176096631,
338
- "grad_norm": 5714.4814453125,
339
- "learning_rate": 8.675955694840863e-05,
340
- "loss": 2.782,
341
- "step": 450
342
  },
343
  {
344
- "epoch": 0.5848696757787667,
345
- "grad_norm": 298284.5,
346
- "learning_rate": 8.602788444503005e-05,
347
- "loss": 3.3277,
348
- "step": 460
349
  },
350
  {
351
- "epoch": 0.5975842339478703,
352
- "grad_norm": 8342007.5,
353
- "learning_rate": 8.527980202400595e-05,
354
- "loss": 6.4328,
355
- "step": 470
356
  },
357
  {
358
- "epoch": 0.6102987921169739,
359
- "grad_norm": 736.1781616210938,
360
- "learning_rate": 8.451565042060592e-05,
361
- "loss": 12.635,
362
- "step": 480
363
  },
364
  {
365
- "epoch": 0.6230133502860775,
366
- "grad_norm": 576991.125,
367
- "learning_rate": 8.373577768926265e-05,
368
- "loss": 16.0493,
369
- "step": 490
370
  },
371
  {
372
- "epoch": 0.6357279084551812,
373
- "grad_norm": 11404688.0,
374
- "learning_rate": 8.29405390450406e-05,
375
- "loss": 15.8506,
376
- "step": 500
377
  },
378
  {
379
- "epoch": 0.6357279084551812,
380
- "eval_loss": 15.568441390991211,
381
- "eval_runtime": 10.9486,
382
- "eval_samples_per_second": 60.556,
383
- "eval_steps_per_second": 15.162,
384
- "step": 500
385
  }
386
  ],
387
- "logging_steps": 10,
388
- "max_steps": 1572,
389
  "num_input_tokens_seen": 0,
390
- "num_train_epochs": 2,
391
- "save_steps": 250,
392
  "stateful_callbacks": {
393
  "TrainerControl": {
394
  "args": {
@@ -396,13 +108,13 @@
396
  "should_evaluate": false,
397
  "should_log": false,
398
  "should_save": true,
399
- "should_training_stop": false
400
  },
401
  "attributes": {}
402
  }
403
  },
404
- "total_flos": 1.5477787066368e+16,
405
- "train_batch_size": 4,
406
  "trial_name": null,
407
  "trial_params": null
408
  }
 
2
  "best_global_step": null,
3
  "best_metric": null,
4
  "best_model_checkpoint": null,
5
+ "epoch": 0.006074411541381929,
6
+ "eval_steps": 10,
7
+ "global_step": 10,
8
  "is_hyper_param_search": false,
9
  "is_local_process_zero": true,
10
  "is_world_process_zero": true,
11
  "log_history": [
12
  {
13
+ "epoch": 0.0006074411541381929,
14
+ "grad_norm": 1.7081327438354492,
15
+ "learning_rate": 2e-05,
16
+ "loss": 2.8032,
 
17
  "step": 1
18
  },
19
  {
20
+ "epoch": 0.0006074411541381929,
21
+ "eval_loss": 4.683979511260986,
22
+ "eval_runtime": 9.5765,
23
+ "eval_samples_per_second": 72.469,
24
+ "eval_steps_per_second": 36.234,
25
+ "step": 1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
26
  },
27
  {
28
+ "epoch": 0.0012148823082763858,
29
+ "grad_norm": 1.5416028499603271,
30
+ "learning_rate": 2e-05,
31
+ "loss": 4.1634,
32
+ "step": 2
33
  },
34
  {
35
+ "epoch": 0.0018223234624145787,
36
+ "grad_norm": 3.041093111038208,
37
+ "learning_rate": 2e-05,
38
+ "loss": 2.9374,
39
+ "step": 3
40
  },
41
  {
42
+ "epoch": 0.0024297646165527716,
43
+ "grad_norm": 3.702852249145508,
44
+ "learning_rate": 2e-05,
45
+ "loss": 4.6189,
46
+ "step": 4
47
  },
48
  {
49
+ "epoch": 0.0030372057706909645,
50
+ "grad_norm": 44.08717727661133,
51
+ "learning_rate": 2e-05,
52
+ "loss": 3.2511,
53
+ "step": 5
54
  },
55
  {
56
+ "epoch": 0.0036446469248291574,
57
+ "grad_norm": 25.48660659790039,
58
+ "learning_rate": 2e-05,
59
+ "loss": 2.8151,
60
+ "step": 6
61
  },
62
  {
63
+ "epoch": 0.00425208807896735,
64
+ "grad_norm": 1.7461680173873901,
65
+ "learning_rate": 2e-05,
66
+ "loss": 3.5287,
67
+ "step": 7
68
  },
69
  {
70
+ "epoch": 0.004859529233105543,
71
+ "grad_norm": 2.2297744750976562,
72
+ "learning_rate": 2e-05,
73
+ "loss": 4.4866,
74
+ "step": 8
75
  },
76
  {
77
+ "epoch": 0.005466970387243736,
78
+ "grad_norm": 1.853412389755249,
79
+ "learning_rate": 2e-05,
80
+ "loss": 3.1538,
81
+ "step": 9
82
  },
83
  {
84
+ "epoch": 0.006074411541381929,
85
+ "grad_norm": 14.022143363952637,
86
+ "learning_rate": 2e-05,
87
+ "loss": 3.1281,
88
+ "step": 10
89
  },
90
  {
91
+ "epoch": 0.006074411541381929,
92
+ "eval_loss": 4.62240743637085,
93
+ "eval_runtime": 9.5783,
94
+ "eval_samples_per_second": 72.455,
95
+ "eval_steps_per_second": 36.228,
96
+ "step": 10
97
  }
98
  ],
99
+ "logging_steps": 1,
100
+ "max_steps": 10,
101
  "num_input_tokens_seen": 0,
102
+ "num_train_epochs": 1,
103
+ "save_steps": 10,
104
  "stateful_callbacks": {
105
  "TrainerControl": {
106
  "args": {
 
108
  "should_evaluate": false,
109
  "should_log": false,
110
  "should_save": true,
111
+ "should_training_stop": true
112
  },
113
  "attributes": {}
114
  }
115
  },
116
+ "total_flos": 75069652992000.0,
117
+ "train_batch_size": 2,
118
  "trial_name": null,
119
  "trial_params": null
120
  }
last-checkpoint/training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:b04e6b1dcc680af57b758040dc34e96b57d7e39783a5b767c696bb2cdf09f64a
3
  size 7224
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e6b02eb71541c854f7fab36869439a9871519892a5f750b5819c16ebeb5d14b4
3
  size 7224