error577 commited on
Commit
1b243b6
·
verified ·
1 Parent(s): 5a3b76c

Training in progress, step 350, checkpoint

Browse files
last-checkpoint/adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0dd5f507f711922c7b8320658d6ece974548a3d5ebc8be0f42fd8e364e5813c7
3
  size 281587880
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:40f2cf472b8c7e6ccdc86217290eb8d7dc33e3ab80f71ee367c67e02d39e1fe3
3
  size 281587880
last-checkpoint/optimizer.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:06848f8e12e285844f5edc96b75755387463f518c2e0bc9bc5ecad0826299239
3
  size 143366164
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f6e64630ea125e1b76094f0464fb8369e95526b154858cc41fd58814081e1a20
3
  size 143366164
last-checkpoint/rng_state.pth CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:fc006f1a0ce7c69becbe0ac1a74f9775813e2049cb4aa6448f1cab721a6aa7a1
3
  size 14244
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8bbaea94c24565bdc57de5411d2276eefbc226e76b091640d4f584c6a6fdc009
3
  size 14244
last-checkpoint/scheduler.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:83f30fc7a303c581db8790c0d4f8638955c788c0b8516a8046c52d38b258639f
3
  size 1064
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cc4359ecb90a50bbea116b9a12b6a99effac889cc2d40f6093f2c443d61fb593
3
  size 1064
last-checkpoint/trainer_state.json CHANGED
@@ -1,9 +1,9 @@
1
  {
2
  "best_metric": 0.0175247173756361,
3
  "best_model_checkpoint": "miner_id_24/checkpoint-200",
4
- "epoch": 0.02218391388944092,
5
  "eval_steps": 50,
6
- "global_step": 300,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
@@ -2163,6 +2163,364 @@
2163
  "eval_samples_per_second": 15.434,
2164
  "eval_steps_per_second": 15.434,
2165
  "step": 300
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2166
  }
2167
  ],
2168
  "logging_steps": 1,
@@ -2177,7 +2535,7 @@
2177
  "early_stopping_threshold": 0.0
2178
  },
2179
  "attributes": {
2180
- "early_stopping_patience_counter": 2
2181
  }
2182
  },
2183
  "TrainerControl": {
@@ -2186,12 +2544,12 @@
2186
  "should_evaluate": false,
2187
  "should_log": false,
2188
  "should_save": true,
2189
- "should_training_stop": false
2190
  },
2191
  "attributes": {}
2192
  }
2193
  },
2194
- "total_flos": 8703274531553280.0,
2195
  "train_batch_size": 1,
2196
  "trial_name": null,
2197
  "trial_params": null
 
1
  {
2
  "best_metric": 0.0175247173756361,
3
  "best_model_checkpoint": "miner_id_24/checkpoint-200",
4
+ "epoch": 0.025881232871014407,
5
  "eval_steps": 50,
6
+ "global_step": 350,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
 
2163
  "eval_samples_per_second": 15.434,
2164
  "eval_steps_per_second": 15.434,
2165
  "step": 300
2166
+ },
2167
+ {
2168
+ "epoch": 0.02225786026907239,
2169
+ "grad_norm": 0.3705998659133911,
2170
+ "learning_rate": 0.0002404622145698234,
2171
+ "loss": 0.0342,
2172
+ "step": 301
2173
+ },
2174
+ {
2175
+ "epoch": 0.022331806648703857,
2176
+ "grad_norm": 0.3507665693759918,
2177
+ "learning_rate": 0.00024008206511550044,
2178
+ "loss": 0.021,
2179
+ "step": 302
2180
+ },
2181
+ {
2182
+ "epoch": 0.022405753028335328,
2183
+ "grad_norm": 0.2580109238624573,
2184
+ "learning_rate": 0.00023970100853578185,
2185
+ "loss": 0.0366,
2186
+ "step": 303
2187
+ },
2188
+ {
2189
+ "epoch": 0.022479699407966798,
2190
+ "grad_norm": 0.2751363217830658,
2191
+ "learning_rate": 0.00023931904866790317,
2192
+ "loss": 0.0383,
2193
+ "step": 304
2194
+ },
2195
+ {
2196
+ "epoch": 0.02255364578759827,
2197
+ "grad_norm": 0.1519552618265152,
2198
+ "learning_rate": 0.00023893618935819607,
2199
+ "loss": 0.0114,
2200
+ "step": 305
2201
+ },
2202
+ {
2203
+ "epoch": 0.02262759216722974,
2204
+ "grad_norm": 0.4399930536746979,
2205
+ "learning_rate": 0.00023855243446204946,
2206
+ "loss": 0.1788,
2207
+ "step": 306
2208
+ },
2209
+ {
2210
+ "epoch": 0.022701538546861206,
2211
+ "grad_norm": 0.3781653046607971,
2212
+ "learning_rate": 0.00023816778784387094,
2213
+ "loss": 0.0413,
2214
+ "step": 307
2215
+ },
2216
+ {
2217
+ "epoch": 0.022775484926492676,
2218
+ "grad_norm": 0.2434411644935608,
2219
+ "learning_rate": 0.00023778225337704772,
2220
+ "loss": 0.002,
2221
+ "step": 308
2222
+ },
2223
+ {
2224
+ "epoch": 0.022849431306124147,
2225
+ "grad_norm": 0.06450709700584412,
2226
+ "learning_rate": 0.00023739583494390752,
2227
+ "loss": 0.0019,
2228
+ "step": 309
2229
+ },
2230
+ {
2231
+ "epoch": 0.022923377685755617,
2232
+ "grad_norm": 0.3262978494167328,
2233
+ "learning_rate": 0.0002370085364356797,
2234
+ "loss": 0.0118,
2235
+ "step": 310
2236
+ },
2237
+ {
2238
+ "epoch": 0.022997324065387088,
2239
+ "grad_norm": 0.22701017558574677,
2240
+ "learning_rate": 0.00023662036175245595,
2241
+ "loss": 0.0297,
2242
+ "step": 311
2243
+ },
2244
+ {
2245
+ "epoch": 0.023071270445018555,
2246
+ "grad_norm": 0.14085568487644196,
2247
+ "learning_rate": 0.00023623131480315107,
2248
+ "loss": 0.0104,
2249
+ "step": 312
2250
+ },
2251
+ {
2252
+ "epoch": 0.023145216824650025,
2253
+ "grad_norm": 0.2725294232368469,
2254
+ "learning_rate": 0.00023584139950546344,
2255
+ "loss": 0.0214,
2256
+ "step": 313
2257
+ },
2258
+ {
2259
+ "epoch": 0.023219163204281495,
2260
+ "grad_norm": 0.27837783098220825,
2261
+ "learning_rate": 0.0002354506197858358,
2262
+ "loss": 0.0054,
2263
+ "step": 314
2264
+ },
2265
+ {
2266
+ "epoch": 0.023293109583912966,
2267
+ "grad_norm": 0.4964681565761566,
2268
+ "learning_rate": 0.00023505897957941556,
2269
+ "loss": 0.0053,
2270
+ "step": 315
2271
+ },
2272
+ {
2273
+ "epoch": 0.023367055963544436,
2274
+ "grad_norm": 0.19499018788337708,
2275
+ "learning_rate": 0.00023466648283001538,
2276
+ "loss": 0.0111,
2277
+ "step": 316
2278
+ },
2279
+ {
2280
+ "epoch": 0.023441002343175903,
2281
+ "grad_norm": 0.11361473798751831,
2282
+ "learning_rate": 0.000234273133490073,
2283
+ "loss": 0.0189,
2284
+ "step": 317
2285
+ },
2286
+ {
2287
+ "epoch": 0.023514948722807374,
2288
+ "grad_norm": 0.1209384948015213,
2289
+ "learning_rate": 0.00023387893552061199,
2290
+ "loss": 0.0048,
2291
+ "step": 318
2292
+ },
2293
+ {
2294
+ "epoch": 0.023588895102438844,
2295
+ "grad_norm": 0.3064891993999481,
2296
+ "learning_rate": 0.00023348389289120158,
2297
+ "loss": 0.0043,
2298
+ "step": 319
2299
+ },
2300
+ {
2301
+ "epoch": 0.023662841482070315,
2302
+ "grad_norm": 0.04545588046312332,
2303
+ "learning_rate": 0.00023308800957991653,
2304
+ "loss": 0.0012,
2305
+ "step": 320
2306
+ },
2307
+ {
2308
+ "epoch": 0.023736787861701785,
2309
+ "grad_norm": 0.16587400436401367,
2310
+ "learning_rate": 0.00023269128957329748,
2311
+ "loss": 0.0077,
2312
+ "step": 321
2313
+ },
2314
+ {
2315
+ "epoch": 0.023810734241333252,
2316
+ "grad_norm": 0.03504616394639015,
2317
+ "learning_rate": 0.0002322937368663105,
2318
+ "loss": 0.0009,
2319
+ "step": 322
2320
+ },
2321
+ {
2322
+ "epoch": 0.023884680620964722,
2323
+ "grad_norm": 0.3943489193916321,
2324
+ "learning_rate": 0.00023189535546230683,
2325
+ "loss": 0.0193,
2326
+ "step": 323
2327
+ },
2328
+ {
2329
+ "epoch": 0.023958627000596193,
2330
+ "grad_norm": 4.359734058380127,
2331
+ "learning_rate": 0.00023149614937298296,
2332
+ "loss": 0.3536,
2333
+ "step": 324
2334
+ },
2335
+ {
2336
+ "epoch": 0.024032573380227663,
2337
+ "grad_norm": 0.07727823406457901,
2338
+ "learning_rate": 0.00023109612261833963,
2339
+ "loss": 0.0029,
2340
+ "step": 325
2341
+ },
2342
+ {
2343
+ "epoch": 0.024106519759859134,
2344
+ "grad_norm": 0.23159946501255035,
2345
+ "learning_rate": 0.00023069527922664186,
2346
+ "loss": 0.0352,
2347
+ "step": 326
2348
+ },
2349
+ {
2350
+ "epoch": 0.0241804661394906,
2351
+ "grad_norm": 0.07464925199747086,
2352
+ "learning_rate": 0.00023029362323437818,
2353
+ "loss": 0.0012,
2354
+ "step": 327
2355
+ },
2356
+ {
2357
+ "epoch": 0.02425441251912207,
2358
+ "grad_norm": 0.4283057749271393,
2359
+ "learning_rate": 0.00022989115868621995,
2360
+ "loss": 0.0076,
2361
+ "step": 328
2362
+ },
2363
+ {
2364
+ "epoch": 0.02432835889875354,
2365
+ "grad_norm": 0.3561228811740875,
2366
+ "learning_rate": 0.0002294878896349807,
2367
+ "loss": 0.0053,
2368
+ "step": 329
2369
+ },
2370
+ {
2371
+ "epoch": 0.024402305278385012,
2372
+ "grad_norm": 0.5599501132965088,
2373
+ "learning_rate": 0.00022908382014157533,
2374
+ "loss": 0.0079,
2375
+ "step": 330
2376
+ },
2377
+ {
2378
+ "epoch": 0.024476251658016483,
2379
+ "grad_norm": 0.8999078869819641,
2380
+ "learning_rate": 0.00022867895427497914,
2381
+ "loss": 0.1989,
2382
+ "step": 331
2383
+ },
2384
+ {
2385
+ "epoch": 0.02455019803764795,
2386
+ "grad_norm": 0.6236395239830017,
2387
+ "learning_rate": 0.00022827329611218688,
2388
+ "loss": 0.0226,
2389
+ "step": 332
2390
+ },
2391
+ {
2392
+ "epoch": 0.02462414441727942,
2393
+ "grad_norm": 0.292870432138443,
2394
+ "learning_rate": 0.0002278668497381718,
2395
+ "loss": 0.06,
2396
+ "step": 333
2397
+ },
2398
+ {
2399
+ "epoch": 0.02469809079691089,
2400
+ "grad_norm": 0.14794032275676727,
2401
+ "learning_rate": 0.00022745961924584428,
2402
+ "loss": 0.0028,
2403
+ "step": 334
2404
+ },
2405
+ {
2406
+ "epoch": 0.02477203717654236,
2407
+ "grad_norm": 0.013083276338875294,
2408
+ "learning_rate": 0.00022705160873601096,
2409
+ "loss": 0.0005,
2410
+ "step": 335
2411
+ },
2412
+ {
2413
+ "epoch": 0.024845983556173828,
2414
+ "grad_norm": 0.21283191442489624,
2415
+ "learning_rate": 0.00022664282231733309,
2416
+ "loss": 0.0248,
2417
+ "step": 336
2418
+ },
2419
+ {
2420
+ "epoch": 0.024919929935805298,
2421
+ "grad_norm": 0.444803923368454,
2422
+ "learning_rate": 0.00022623326410628534,
2423
+ "loss": 0.0026,
2424
+ "step": 337
2425
+ },
2426
+ {
2427
+ "epoch": 0.02499387631543677,
2428
+ "grad_norm": 1.0099908113479614,
2429
+ "learning_rate": 0.00022582293822711444,
2430
+ "loss": 0.0088,
2431
+ "step": 338
2432
+ },
2433
+ {
2434
+ "epoch": 0.02506782269506824,
2435
+ "grad_norm": 0.45318788290023804,
2436
+ "learning_rate": 0.00022541184881179737,
2437
+ "loss": 0.1016,
2438
+ "step": 339
2439
+ },
2440
+ {
2441
+ "epoch": 0.02514176907469971,
2442
+ "grad_norm": 0.5700947046279907,
2443
+ "learning_rate": 0.000225,
2444
+ "loss": 0.0194,
2445
+ "step": 340
2446
+ },
2447
+ {
2448
+ "epoch": 0.025215715454331177,
2449
+ "grad_norm": 0.12225595116615295,
2450
+ "learning_rate": 0.0002245873959390353,
2451
+ "loss": 0.0016,
2452
+ "step": 341
2453
+ },
2454
+ {
2455
+ "epoch": 0.025289661833962647,
2456
+ "grad_norm": 0.21263572573661804,
2457
+ "learning_rate": 0.00022417404078382152,
2458
+ "loss": 0.0088,
2459
+ "step": 342
2460
+ },
2461
+ {
2462
+ "epoch": 0.025363608213594117,
2463
+ "grad_norm": 0.7097033262252808,
2464
+ "learning_rate": 0.00022375993869684058,
2465
+ "loss": 0.0065,
2466
+ "step": 343
2467
+ },
2468
+ {
2469
+ "epoch": 0.025437554593225588,
2470
+ "grad_norm": 0.3727460205554962,
2471
+ "learning_rate": 0.00022334509384809584,
2472
+ "loss": 0.0088,
2473
+ "step": 344
2474
+ },
2475
+ {
2476
+ "epoch": 0.025511500972857058,
2477
+ "grad_norm": 3.5641732215881348,
2478
+ "learning_rate": 0.00022292951041507028,
2479
+ "loss": 0.0075,
2480
+ "step": 345
2481
+ },
2482
+ {
2483
+ "epoch": 0.025585447352488525,
2484
+ "grad_norm": 0.12729483842849731,
2485
+ "learning_rate": 0.00022251319258268453,
2486
+ "loss": 0.0027,
2487
+ "step": 346
2488
+ },
2489
+ {
2490
+ "epoch": 0.025659393732119996,
2491
+ "grad_norm": 0.2512650787830353,
2492
+ "learning_rate": 0.00022209614454325459,
2493
+ "loss": 0.0099,
2494
+ "step": 347
2495
+ },
2496
+ {
2497
+ "epoch": 0.025733340111751466,
2498
+ "grad_norm": 0.754295825958252,
2499
+ "learning_rate": 0.00022167837049644947,
2500
+ "loss": 0.0754,
2501
+ "step": 348
2502
+ },
2503
+ {
2504
+ "epoch": 0.025807286491382937,
2505
+ "grad_norm": 0.0925987958908081,
2506
+ "learning_rate": 0.00022125987464924926,
2507
+ "loss": 0.0022,
2508
+ "step": 349
2509
+ },
2510
+ {
2511
+ "epoch": 0.025881232871014407,
2512
+ "grad_norm": 1.8068702220916748,
2513
+ "learning_rate": 0.0002208406612159024,
2514
+ "loss": 0.0575,
2515
+ "step": 350
2516
+ },
2517
+ {
2518
+ "epoch": 0.025881232871014407,
2519
+ "eval_loss": 0.019101565703749657,
2520
+ "eval_runtime": 28.2195,
2521
+ "eval_samples_per_second": 15.379,
2522
+ "eval_steps_per_second": 15.379,
2523
+ "step": 350
2524
  }
2525
  ],
2526
  "logging_steps": 1,
 
2535
  "early_stopping_threshold": 0.0
2536
  },
2537
  "attributes": {
2538
+ "early_stopping_patience_counter": 3
2539
  }
2540
  },
2541
  "TrainerControl": {
 
2544
  "should_evaluate": false,
2545
  "should_log": false,
2546
  "should_save": true,
2547
+ "should_training_stop": true
2548
  },
2549
  "attributes": {}
2550
  }
2551
  },
2552
+ "total_flos": 1.0126848687734784e+16,
2553
  "train_batch_size": 1,
2554
  "trial_name": null,
2555
  "trial_params": null