{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9997759689343589,
  "eval_steps": 500,
  "global_step": 3347,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio": 0.0,
      "completion_length": 2035.6161193847656,
      "epoch": 0.00029870808752146963,
      "grad_norm": 0.07377588003873825,
      "kl": 0.0,
      "learning_rate": 1.4925373134328358e-09,
      "loss": 0.0095,
      "reward": 0.2991071566939354,
      "reward_std": 0.035552696557715535,
      "rewards/accuracy_reward": 0.042410716181620955,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2566964477300644,
      "step": 1
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2006.6161193847656,
      "epoch": 0.0005974161750429393,
      "grad_norm": 0.07657723128795624,
      "kl": 0.0,
      "learning_rate": 2.9850746268656717e-09,
      "loss": 0.0203,
      "reward": 0.3392857313156128,
      "reward_std": 0.023831423372030258,
      "rewards/accuracy_reward": 0.0736607164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2656250149011612,
      "step": 2
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2046.8616333007812,
      "epoch": 0.0008961242625644089,
      "grad_norm": 0.059486813843250275,
      "kl": 0.00015664100646972656,
      "learning_rate": 4.477611940298507e-09,
      "loss": 0.0006,
      "reward": 0.2611607238650322,
      "reward_std": 0.026519399601966143,
      "rewards/accuracy_reward": 0.004464285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2566964328289032,
      "step": 3
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2044.7544860839844,
      "epoch": 0.0011948323500858785,
      "grad_norm": 0.039279546588659286,
      "kl": 0.0001628398895263672,
      "learning_rate": 5.970149253731343e-09,
      "loss": 0.0061,
      "reward": 0.3231026902794838,
      "reward_std": 0.006696428870782256,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2516741156578064,
      "step": 4
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2008.3393859863281,
      "epoch": 0.0014935404376073482,
      "grad_norm": 0.08565092086791992,
      "kl": 0.00017762184143066406,
      "learning_rate": 7.462686567164179e-09,
      "loss": 0.0173,
      "reward": 0.3404018059372902,
      "reward_std": 0.023454802576452494,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2689732238650322,
      "step": 5
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2036.5625610351562,
      "epoch": 0.0017922485251288178,
      "grad_norm": 0.06848834455013275,
      "kl": 0.00017714500427246094,
      "learning_rate": 8.955223880597015e-09,
      "loss": 0.005,
      "reward": 0.3108259066939354,
      "reward_std": 0.05675417836755514,
      "rewards/accuracy_reward": 0.049107146449387074,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2617187649011612,
      "step": 6
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2044.9017944335938,
      "epoch": 0.0020909566126502874,
      "grad_norm": 0.04619584605097771,
      "kl": 0.0001800060272216797,
      "learning_rate": 1.044776119402985e-08,
      "loss": 0.0023,
      "reward": 0.2539062649011612,
      "reward_std": 0.008738514501601458,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2539062649011612,
      "step": 7
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2034.5111999511719,
      "epoch": 0.002389664700171757,
      "grad_norm": 0.05582153797149658,
      "kl": 0.00017952919006347656,
      "learning_rate": 1.1940298507462687e-08,
      "loss": 0.0136,
      "reward": 0.3314732238650322,
      "reward_std": 0.0254965724889189,
      "rewards/accuracy_reward": 0.07589286053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2555803656578064,
      "step": 8
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2038.3259582519531,
      "epoch": 0.0026883727876932267,
      "grad_norm": 0.0764269232749939,
      "kl": 0.00017571449279785156,
      "learning_rate": 1.3432835820895521e-08,
      "loss": 0.0089,
      "reward": 0.2996651902794838,
      "reward_std": 0.04299068497493863,
      "rewards/accuracy_reward": 0.04017857206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.259486623108387,
      "step": 9
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2032.02685546875,
      "epoch": 0.0029870808752146963,
      "grad_norm": 0.06721504032611847,
      "kl": 0.00016880035400390625,
      "learning_rate": 1.4925373134328357e-08,
      "loss": 0.013,
      "reward": 0.3320312649011612,
      "reward_std": 0.027131953742355108,
      "rewards/accuracy_reward": 0.07366071757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2583705484867096,
      "step": 10
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1999.3639221191406,
      "epoch": 0.003285788962736166,
      "grad_norm": 0.09062264859676361,
      "kl": 0.00017023086547851562,
      "learning_rate": 1.6417910447761193e-08,
      "loss": 0.0221,
      "reward": 0.3119419813156128,
      "reward_std": 0.039642661809921265,
      "rewards/accuracy_reward": 0.04017857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2717634066939354,
      "step": 11
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2043.2790832519531,
      "epoch": 0.0035844970502576356,
      "grad_norm": 0.07396755367517471,
      "kl": 0.00017070770263671875,
      "learning_rate": 1.791044776119403e-08,
      "loss": 0.0033,
      "reward": 0.3376116156578064,
      "reward_std": 0.04667802760377526,
      "rewards/accuracy_reward": 0.07812500349245965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.259486623108387,
      "step": 12
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2009.6763916015625,
      "epoch": 0.003883205137779105,
      "grad_norm": 0.06218832731246948,
      "kl": 0.00017213821411132812,
      "learning_rate": 1.9402985074626865e-08,
      "loss": 0.0124,
      "reward": 0.301897332072258,
      "reward_std": 0.016571818618103862,
      "rewards/accuracy_reward": 0.03794643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2639508992433548,
      "step": 13
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2045.700927734375,
      "epoch": 0.004181913225300575,
      "grad_norm": 0.05105062201619148,
      "kl": 0.0001728534698486328,
      "learning_rate": 2.08955223880597e-08,
      "loss": 0.0035,
      "reward": 0.3231026902794838,
      "reward_std": 0.00974611658602953,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2516741082072258,
      "step": 14
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2007.4286499023438,
      "epoch": 0.004480621312822045,
      "grad_norm": 0.07748989015817642,
      "kl": 0.0001804828643798828,
      "learning_rate": 2.2388059701492534e-08,
      "loss": 0.0197,
      "reward": 0.3297991156578064,
      "reward_std": 0.0723750137258321,
      "rewards/accuracy_reward": 0.06026785937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2695312649011612,
      "step": 15
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2044.7009582519531,
      "epoch": 0.004779329400343514,
      "grad_norm": 0.06081807240843773,
      "kl": 0.00017905235290527344,
      "learning_rate": 2.3880597014925373e-08,
      "loss": 0.001,
      "reward": 0.2991071566939354,
      "reward_std": 0.0357142873108387,
      "rewards/accuracy_reward": 0.0446428582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.254464291036129,
      "step": 16
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2036.9978332519531,
      "epoch": 0.005078037487864984,
      "grad_norm": 0.07298179715871811,
      "kl": 0.00016069412231445312,
      "learning_rate": 2.5373134328358206e-08,
      "loss": 0.0098,
      "reward": 0.3046875149011612,
      "reward_std": 0.04351334855891764,
      "rewards/accuracy_reward": 0.042410716181620955,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2622768059372902,
      "step": 17
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2031.7232971191406,
      "epoch": 0.005376745575386453,
      "grad_norm": 0.05950786545872688,
      "kl": 0.00017952919006347656,
      "learning_rate": 2.6865671641791042e-08,
      "loss": 0.0092,
      "reward": 0.3370535895228386,
      "reward_std": 0.03425386408343911,
      "rewards/accuracy_reward": 0.0781250037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2589285895228386,
      "step": 18
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2000.6094360351562,
      "epoch": 0.0056754536629079234,
      "grad_norm": 0.0792248547077179,
      "kl": 0.00018167495727539062,
      "learning_rate": 2.835820895522388e-08,
      "loss": 0.0219,
      "reward": 0.3571428656578064,
      "reward_std": 0.07195467129349709,
      "rewards/accuracy_reward": 0.09151786053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2656250074505806,
      "step": 19
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2033.5491638183594,
      "epoch": 0.005974161750429393,
      "grad_norm": 0.06336899846792221,
      "kl": 0.0001785755157470703,
      "learning_rate": 2.9850746268656714e-08,
      "loss": 0.009,
      "reward": 0.364955373108387,
      "reward_std": 0.0678480037022382,
      "rewards/accuracy_reward": 0.10491071920841932,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2600446566939354,
      "step": 20
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2025.8326416015625,
      "epoch": 0.006272869837950863,
      "grad_norm": 0.04469405114650726,
      "kl": 0.00017404556274414062,
      "learning_rate": 3.134328358208955e-08,
      "loss": 0.0088,
      "reward": 0.3309151902794838,
      "reward_std": 0.008267207071185112,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2594866156578064,
      "step": 21
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2033.9129943847656,
      "epoch": 0.006571577925472332,
      "grad_norm": 0.058748260140419006,
      "kl": 0.00017070770263671875,
      "learning_rate": 3.2835820895522386e-08,
      "loss": 0.0099,
      "reward": 0.263392873108387,
      "reward_std": 0.028387735132128,
      "rewards/accuracy_reward": 0.0066964291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2566964402794838,
      "step": 22
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2042.3192138671875,
      "epoch": 0.006870286012993802,
      "grad_norm": 0.05660548433661461,
      "kl": 0.00017571449279785156,
      "learning_rate": 3.432835820895522e-08,
      "loss": 0.0034,
      "reward": 0.3030134066939354,
      "reward_std": 0.036540572764351964,
      "rewards/accuracy_reward": 0.04687500116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2561384066939354,
      "step": 23
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2004.4509887695312,
      "epoch": 0.007168994100515271,
      "grad_norm": 0.07296764850616455,
      "kl": 0.0001685619354248047,
      "learning_rate": 3.582089552238806e-08,
      "loss": 0.0175,
      "reward": 0.424107164144516,
      "reward_std": 0.05331416008993983,
      "rewards/accuracy_reward": 0.1540178656578064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2700892984867096,
      "step": 24
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2046.154052734375,
      "epoch": 0.007467702188036741,
      "grad_norm": 0.050552818924188614,
      "kl": 0.00017547607421875,
      "learning_rate": 3.731343283582089e-08,
      "loss": 0.0016,
      "reward": 0.3046875149011612,
      "reward_std": 0.03753993893042207,
      "rewards/accuracy_reward": 0.04910714481957257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.255580373108387,
      "step": 25
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2035.3482666015625,
      "epoch": 0.00776641027555821,
      "grad_norm": 0.06812873482704163,
      "kl": 0.00017070770263671875,
      "learning_rate": 3.880597014925373e-08,
      "loss": 0.0089,
      "reward": 0.3113839328289032,
      "reward_std": 0.04659886518493295,
      "rewards/accuracy_reward": 0.051339288242161274,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2600446566939354,
      "step": 26
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2041.4754943847656,
      "epoch": 0.00806511836307968,
      "grad_norm": 0.060088448226451874,
      "kl": 0.0001895427703857422,
      "learning_rate": 4.029850746268657e-08,
      "loss": 0.0076,
      "reward": 0.4140625223517418,
      "reward_std": 0.0314870816655457,
      "rewards/accuracy_reward": 0.1584821492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2555803656578064,
      "step": 27
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2033.1295471191406,
      "epoch": 0.00836382645060115,
      "grad_norm": 0.07130387425422668,
      "kl": 0.00017189979553222656,
      "learning_rate": 4.17910447761194e-08,
      "loss": 0.0102,
      "reward": 0.309151791036129,
      "reward_std": 0.03787641413509846,
      "rewards/accuracy_reward": 0.04687500186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.262276791036129,
      "step": 28
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1981.497802734375,
      "epoch": 0.008662534538122619,
      "grad_norm": 0.09127658605575562,
      "kl": 0.000179290771484375,
      "learning_rate": 4.328358208955224e-08,
      "loss": 0.0169,
      "reward": 0.3242187649011612,
      "reward_std": 0.0687947750557214,
      "rewards/accuracy_reward": 0.05133928847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2728794813156128,
      "step": 29
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2005.3996276855469,
      "epoch": 0.00896124262564409,
      "grad_norm": 0.08891815692186356,
      "kl": 0.00018072128295898438,
      "learning_rate": 4.477611940298507e-08,
      "loss": 0.0183,
      "reward": 0.3867187574505806,
      "reward_std": 0.04750661668367684,
      "rewards/accuracy_reward": 0.11607143376022577,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2706473395228386,
      "step": 30
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2021.1786193847656,
      "epoch": 0.009259950713165559,
      "grad_norm": 0.07822487503290176,
      "kl": 0.0001800060272216797,
      "learning_rate": 4.6268656716417904e-08,
      "loss": 0.0172,
      "reward": 0.3147321566939354,
      "reward_std": 0.04173394991084933,
      "rewards/accuracy_reward": 0.05357143096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2611607313156128,
      "step": 31
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2030.0335693359375,
      "epoch": 0.009558658800687028,
      "grad_norm": 0.06654183566570282,
      "kl": 0.0001800060272216797,
      "learning_rate": 4.776119402985075e-08,
      "loss": 0.01,
      "reward": 0.3504464402794838,
      "reward_std": 0.03687205188907683,
      "rewards/accuracy_reward": 0.0892857201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2611607313156128,
      "step": 32
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2036.5781555175781,
      "epoch": 0.009857366888208497,
      "grad_norm": 0.05982429161667824,
      "kl": 0.0001819133758544922,
      "learning_rate": 4.925373134328358e-08,
      "loss": 0.0073,
      "reward": 0.3091517984867096,
      "reward_std": 0.029121503233909607,
      "rewards/accuracy_reward": 0.0513392873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2578125074505806,
      "step": 33
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2041.2656860351562,
      "epoch": 0.010156074975729968,
      "grad_norm": 0.0702994167804718,
      "kl": 0.00018072128295898438,
      "learning_rate": 5.074626865671641e-08,
      "loss": 0.0051,
      "reward": 0.3203125149011612,
      "reward_std": 0.048810507170856,
      "rewards/accuracy_reward": 0.060267860535532236,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2600446492433548,
      "step": 34
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2017.1295166015625,
      "epoch": 0.010454783063251438,
      "grad_norm": 0.08044405281543732,
      "kl": 0.00017762184143066406,
      "learning_rate": 5.223880597014925e-08,
      "loss": 0.0197,
      "reward": 0.325892873108387,
      "reward_std": 0.07425616635009646,
      "rewards/accuracy_reward": 0.06250000209547579,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2633928656578064,
      "step": 35
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2024.4531860351562,
      "epoch": 0.010753491150772907,
      "grad_norm": 0.08431462198495865,
      "kl": 0.00018262863159179688,
      "learning_rate": 5.3731343283582085e-08,
      "loss": 0.0135,
      "reward": 0.2924107238650322,
      "reward_std": 0.0366232821252197,
      "rewards/accuracy_reward": 0.0290178582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2633928656578064,
      "step": 36
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2035.3326721191406,
      "epoch": 0.011052199238294378,
      "grad_norm": 0.09053537994623184,
      "kl": 0.0001785755157470703,
      "learning_rate": 5.522388059701493e-08,
      "loss": 0.0089,
      "reward": 0.3113839402794838,
      "reward_std": 0.06095702131278813,
      "rewards/accuracy_reward": 0.04910714412108064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2622767984867096,
      "step": 37
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2046.5714721679688,
      "epoch": 0.011350907325815847,
      "grad_norm": 0.039195768535137177,
      "kl": 0.00018405914306640625,
      "learning_rate": 5.671641791044776e-08,
      "loss": 0.0017,
      "reward": 0.2885044738650322,
      "reward_std": 0.0066489099990576506,
      "rewards/accuracy_reward": 0.0357142873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2527901828289032,
      "step": 38
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2040.6607666015625,
      "epoch": 0.011649615413337316,
      "grad_norm": 0.061163000762462616,
      "kl": 0.0001842975616455078,
      "learning_rate": 5.820895522388059e-08,
      "loss": 0.0079,
      "reward": 0.2762276828289032,
      "reward_std": 0.05739667871966958,
      "rewards/accuracy_reward": 0.0200892873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2561383992433548,
      "step": 39
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2029.3884582519531,
      "epoch": 0.011948323500858785,
      "grad_norm": 0.08882297575473785,
      "kl": 0.00017404556274414062,
      "learning_rate": 5.970149253731343e-08,
      "loss": 0.018,
      "reward": 0.3593750149011612,
      "reward_std": 0.06567940767854452,
      "rewards/accuracy_reward": 0.0937500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2656250074505806,
      "step": 40
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2037.27685546875,
      "epoch": 0.012247031588380256,
      "grad_norm": 0.07564129680395126,
      "kl": 0.00018024444580078125,
      "learning_rate": 6.119402985074627e-08,
      "loss": 0.0138,
      "reward": 0.3800223395228386,
      "reward_std": 0.0511107025668025,
      "rewards/accuracy_reward": 0.11830357555299997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2617187649011612,
      "step": 41
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2047.5491333007812,
      "epoch": 0.012545739675901725,
      "grad_norm": 0.05243632569909096,
      "kl": 0.00017881393432617188,
      "learning_rate": 6.26865671641791e-08,
      "loss": 0.0006,
      "reward": 0.2885044813156128,
      "reward_std": 0.00974611658602953,
      "rewards/accuracy_reward": 0.0357142873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2527901977300644,
      "step": 42
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2043.0000915527344,
      "epoch": 0.012844447763423195,
      "grad_norm": 0.07800869643688202,
      "kl": 0.00017881393432617188,
      "learning_rate": 6.417910447761193e-08,
      "loss": 0.0083,
      "reward": 0.2700892984867096,
      "reward_std": 0.06405318272300065,
      "rewards/accuracy_reward": 0.013392857508733869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2566964402794838,
      "step": 43
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2031.6116638183594,
      "epoch": 0.013143155850944664,
      "grad_norm": 0.07709817588329315,
      "kl": 0.00018334388732910156,
      "learning_rate": 6.567164179104477e-08,
      "loss": 0.0088,
      "reward": 0.3225446566939354,
      "reward_std": 0.05965192103758454,
      "rewards/accuracy_reward": 0.060267861699685454,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2622767984867096,
      "step": 44
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2036.4643859863281,
      "epoch": 0.013441863938466135,
      "grad_norm": 0.06595548987388611,
      "kl": 0.0001723766326904297,
      "learning_rate": 6.71641791044776e-08,
      "loss": 0.0057,
      "reward": 0.2996651902794838,
      "reward_std": 0.021251919912174344,
      "rewards/accuracy_reward": 0.0357142873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2639509066939354,
      "step": 45
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.013740572025987604,
      "grad_norm": 0.03848166763782501,
      "kl": 0.00019097328186035156,
      "learning_rate": 6.865671641791045e-08,
      "loss": 0.0,
      "reward": 0.3270089402794838,
      "reward_std": 0.0223214291036129,
      "rewards/accuracy_reward": 0.07589285937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.251116082072258,
      "step": 46
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2033.2969360351562,
      "epoch": 0.014039280113509073,
      "grad_norm": 0.06175874546170235,
      "kl": 0.00018286705017089844,
      "learning_rate": 7.014925373134329e-08,
      "loss": 0.008,
      "reward": 0.3337053656578064,
      "reward_std": 0.018003552686423063,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.262276791036129,
      "step": 47
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2019.2522888183594,
      "epoch": 0.014337988201030542,
      "grad_norm": 0.06305798888206482,
      "kl": 0.00019073486328125,
      "learning_rate": 7.164179104477612e-08,
      "loss": 0.0071,
      "reward": 0.2924107238650322,
      "reward_std": 0.017975562950596213,
      "rewards/accuracy_reward": 0.0334821455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.258928582072258,
      "step": 48
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2011.7879638671875,
      "epoch": 0.014636696288552013,
      "grad_norm": 0.07480230927467346,
      "kl": 0.0001747608184814453,
      "learning_rate": 7.313432835820896e-08,
      "loss": 0.0087,
      "reward": 0.3002232238650322,
      "reward_std": 0.0174260251224041,
      "rewards/accuracy_reward": 0.04017857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2600446492433548,
      "step": 49
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2022.6875610351562,
      "epoch": 0.014935404376073482,
      "grad_norm": 0.05982408672571182,
      "kl": 0.00017499923706054688,
      "learning_rate": 7.462686567164178e-08,
      "loss": 0.0099,
      "reward": 0.2873883992433548,
      "reward_std": 0.030299714766442776,
      "rewards/accuracy_reward": 0.02455357275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.262834832072258,
      "step": 50
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2037.3437805175781,
      "epoch": 0.015234112463594952,
      "grad_norm": 0.054498765617609024,
      "kl": 0.0001773834228515625,
      "learning_rate": 7.611940298507462e-08,
      "loss": 0.0053,
      "reward": 0.3688616305589676,
      "reward_std": 0.024287256645038724,
      "rewards/accuracy_reward": 0.1116071455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2572544738650322,
      "step": 51
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2023.9777221679688,
      "epoch": 0.01553282055111642,
      "grad_norm": 0.05936834216117859,
      "kl": 0.0001766681671142578,
      "learning_rate": 7.761194029850746e-08,
      "loss": 0.0108,
      "reward": 0.310825914144516,
      "reward_std": 0.04481076914817095,
      "rewards/accuracy_reward": 0.04687500232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2639509066939354,
      "step": 52
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2044.587158203125,
      "epoch": 0.01583152863863789,
      "grad_norm": 0.08270423114299774,
      "kl": 0.00019097328186035156,
      "learning_rate": 7.910447761194029e-08,
      "loss": 0.0052,
      "reward": 0.3459821566939354,
      "reward_std": 0.07618198730051517,
      "rewards/accuracy_reward": 0.08928571850992739,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2566964477300644,
      "step": 53
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2016.7879943847656,
      "epoch": 0.01613023672615936,
      "grad_norm": 0.06661954522132874,
      "kl": 0.00017571449279785156,
      "learning_rate": 8.059701492537313e-08,
      "loss": 0.0178,
      "reward": 0.3565848395228386,
      "reward_std": 0.03894307743757963,
      "rewards/accuracy_reward": 0.0892857201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2672991156578064,
      "step": 54
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2034.5335083007812,
      "epoch": 0.016428944813680832,
      "grad_norm": 0.07243689894676208,
      "kl": 0.00017905235290527344,
      "learning_rate": 8.208955223880596e-08,
      "loss": 0.0078,
      "reward": 0.2723214477300644,
      "reward_std": 0.041094141313806176,
      "rewards/accuracy_reward": 0.011160714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2611607238650322,
      "step": 55
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1993.5469360351562,
      "epoch": 0.0167276529012023,
      "grad_norm": 0.08347208052873611,
      "kl": 0.00016999244689941406,
      "learning_rate": 8.35820895522388e-08,
      "loss": 0.0186,
      "reward": 0.2739955484867096,
      "reward_std": 0.03362244344316423,
      "rewards/accuracy_reward": 0.004464285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2695312649011612,
      "step": 56
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2014.4041137695312,
      "epoch": 0.01702636098872377,
      "grad_norm": 0.08523304760456085,
      "kl": 0.00016808509826660156,
      "learning_rate": 8.507462686567165e-08,
      "loss": 0.0184,
      "reward": 0.3080357313156128,
      "reward_std": 0.039523518877103925,
      "rewards/accuracy_reward": 0.0401785746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2678571566939354,
      "step": 57
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2045.7567138671875,
      "epoch": 0.017325069076245238,
      "grad_norm": 0.05177473649382591,
      "kl": 0.00017523765563964844,
      "learning_rate": 8.656716417910448e-08,
      "loss": 0.0014,
      "reward": 0.306361623108387,
      "reward_std": 0.03653183137066662,
      "rewards/accuracy_reward": 0.0513392873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.255022332072258,
      "step": 58
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2035.2054443359375,
      "epoch": 0.01762377716376671,
      "grad_norm": 0.08070658892393112,
      "kl": 0.00017690658569335938,
      "learning_rate": 8.805970149253731e-08,
      "loss": 0.0116,
      "reward": 0.333705373108387,
      "reward_std": 0.025266710203140974,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.262276791036129,
      "step": 59
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2038.529052734375,
      "epoch": 0.01792248525128818,
      "grad_norm": 0.05810768902301788,
      "kl": 0.0001773834228515625,
      "learning_rate": 8.955223880597014e-08,
      "loss": 0.0045,
      "reward": 0.2924107313156128,
      "reward_std": 0.011270960327237844,
      "rewards/accuracy_reward": 0.0357142873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2566964402794838,
      "step": 60
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2013.5000915527344,
      "epoch": 0.018221193338809647,
      "grad_norm": 0.0838748887181282,
      "kl": 0.0001804828643798828,
      "learning_rate": 9.104477611940298e-08,
      "loss": 0.0184,
      "reward": 0.309709832072258,
      "reward_std": 0.04307688772678375,
      "rewards/accuracy_reward": 0.044642859138548374,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2650669664144516,
      "step": 61
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2027.3326721191406,
      "epoch": 0.018519901426331118,
      "grad_norm": 0.07275748252868652,
      "kl": 0.00019049644470214844,
      "learning_rate": 9.253731343283581e-08,
      "loss": 0.0139,
      "reward": 0.271205373108387,
      "reward_std": 0.03510888130404055,
      "rewards/accuracy_reward": 0.008928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2622767984867096,
      "step": 62
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2017.0089721679688,
      "epoch": 0.01881860951385259,
      "grad_norm": 0.06982765346765518,
      "kl": 0.0001742839813232422,
      "learning_rate": 9.402985074626865e-08,
      "loss": 0.0141,
      "reward": 0.2617187574505806,
      "reward_std": 0.015106174862012267,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2617187574505806,
      "step": 63
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2036.0312805175781,
      "epoch": 0.019117317601374056,
      "grad_norm": 0.06350357085466385,
      "kl": 0.00017786026000976562,
      "learning_rate": 9.55223880597015e-08,
      "loss": 0.0088,
      "reward": 0.3816964477300644,
      "reward_std": 0.04492915794253349,
      "rewards/accuracy_reward": 0.11830357555299997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2633928656578064,
      "step": 64
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2032.7590026855469,
      "epoch": 0.019416025688895527,
      "grad_norm": 0.06322678178548813,
      "kl": 0.0001800060272216797,
      "learning_rate": 9.701492537313432e-08,
      "loss": 0.0101,
      "reward": 0.259486623108387,
      "reward_std": 0.015845492016524076,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.259486623108387,
      "step": 65
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2037.4486999511719,
      "epoch": 0.019714733776416995,
      "grad_norm": 0.04374164342880249,
      "kl": 0.00018858909606933594,
      "learning_rate": 9.850746268656717e-08,
      "loss": 0.0051,
      "reward": 0.2985491156578064,
      "reward_std": 0.026388566941022873,
      "rewards/accuracy_reward": 0.04017857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.258370541036129,
      "step": 66
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2041.3214721679688,
      "epoch": 0.020013441863938466,
      "grad_norm": 0.05143103748559952,
      "kl": 0.0001704692840576172,
      "learning_rate": 1e-07,
      "loss": 0.0072,
      "reward": 0.290736623108387,
      "reward_std": 0.011788201984018087,
      "rewards/accuracy_reward": 0.0357142873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.255022332072258,
      "step": 67
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2038.4375610351562,
      "epoch": 0.020312149951459937,
      "grad_norm": 0.09448003768920898,
      "kl": 0.00017786026000976562,
      "learning_rate": 1.0149253731343282e-07,
      "loss": 0.0083,
      "reward": 0.3554687649011612,
      "reward_std": 0.06969616492278874,
      "rewards/accuracy_reward": 0.0937500074505806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2617187649011612,
      "step": 68
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2023.22998046875,
      "epoch": 0.020610858038981404,
      "grad_norm": 0.0896100401878357,
      "kl": 0.000171661376953125,
      "learning_rate": 1.0298507462686567e-07,
      "loss": 0.0196,
      "reward": 0.318080373108387,
      "reward_std": 0.06904237624257803,
      "rewards/accuracy_reward": 0.053571431431919336,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2645089402794838,
      "step": 69
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2045.3013916015625,
      "epoch": 0.020909566126502875,
      "grad_norm": 0.05585716664791107,
      "kl": 0.00017952919006347656,
      "learning_rate": 1.044776119402985e-07,
      "loss": 0.0015,
      "reward": 0.2695312649011612,
      "reward_std": 0.03684081370010972,
      "rewards/accuracy_reward": 0.013392857508733869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2561384066939354,
      "step": 70
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2014.7322082519531,
      "epoch": 0.021208274214024346,
      "grad_norm": 0.0923839658498764,
      "kl": 0.0001842975616455078,
      "learning_rate": 1.0597014925373134e-07,
      "loss": 0.0187,
      "reward": 0.2924107313156128,
      "reward_std": 0.08336303988471627,
      "rewards/accuracy_reward": 0.020089286379516125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2723214477300644,
      "step": 71
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2022.5179748535156,
      "epoch": 0.021506982301545814,
      "grad_norm": 0.07934773713350296,
      "kl": 0.00018072128295898438,
      "learning_rate": 1.0746268656716417e-07,
      "loss": 0.0192,
      "reward": 0.345982164144516,
      "reward_std": 0.0572027824819088,
      "rewards/accuracy_reward": 0.08258928824216127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2633928656578064,
      "step": 72
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1991.4375305175781,
      "epoch": 0.021805690389067284,
      "grad_norm": 0.09225533157587051,
      "kl": 0.0001690387725830078,
      "learning_rate": 1.0895522388059701e-07,
      "loss": 0.0219,
      "reward": 0.4408482387661934,
      "reward_std": 0.05567918159067631,
      "rewards/accuracy_reward": 0.17187500465661287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2689732238650322,
      "step": 73
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2034.4286193847656,
      "epoch": 0.022104398476588755,
      "grad_norm": 0.07546935975551605,
      "kl": 0.00016951560974121094,
      "learning_rate": 1.1044776119402985e-07,
      "loss": 0.0101,
      "reward": 0.2645089477300644,
      "reward_std": 0.03580968314781785,
      "rewards/accuracy_reward": 0.004464285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2600446566939354,
      "step": 74
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1989.0290832519531,
      "epoch": 0.022403106564110223,
      "grad_norm": 0.0822647288441658,
      "kl": 0.00017499923706054688,
      "learning_rate": 1.1194029850746268e-07,
      "loss": 0.0132,
      "reward": 0.3398437649011612,
      "reward_std": 0.0308816097676754,
      "rewards/accuracy_reward": 0.0669642873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2728794738650322,
      "step": 75
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2013.0848693847656,
      "epoch": 0.022701814651631694,
      "grad_norm": 0.08197572827339172,
      "kl": 0.00018215179443359375,
      "learning_rate": 1.1343283582089553e-07,
      "loss": 0.0141,
      "reward": 0.3046875223517418,
      "reward_std": 0.032418082701042295,
      "rewards/accuracy_reward": 0.03794643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2667410895228386,
      "step": 76
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2012.8304138183594,
      "epoch": 0.02300052273915316,
      "grad_norm": 0.07649620622396469,
      "kl": 0.00017404556274414062,
      "learning_rate": 1.1492537313432834e-07,
      "loss": 0.017,
      "reward": 0.325892873108387,
      "reward_std": 0.03620903054252267,
      "rewards/accuracy_reward": 0.06026786006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2656250149011612,
      "step": 77
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2043.5357360839844,
      "epoch": 0.023299230826674632,
      "grad_norm": 0.04983355477452278,
      "kl": 0.00018095970153808594,
      "learning_rate": 1.1641791044776119e-07,
      "loss": 0.0017,
      "reward": 0.3671875149011612,
      "reward_std": 0.02090683183632791,
      "rewards/accuracy_reward": 0.1093750037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2578125074505806,
      "step": 78
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2036.232177734375,
      "epoch": 0.023597938914196103,
      "grad_norm": 0.0644640326499939,
      "kl": 0.00017499923706054688,
      "learning_rate": 1.1791044776119401e-07,
      "loss": 0.0046,
      "reward": 0.294084832072258,
      "reward_std": 0.03216286585666239,
      "rewards/accuracy_reward": 0.0424107164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2516741156578064,
      "step": 79
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2040.5491638183594,
      "epoch": 0.02389664700171757,
      "grad_norm": 0.05917118489742279,
      "kl": 0.00016927719116210938,
      "learning_rate": 1.1940298507462686e-07,
      "loss": 0.0101,
      "reward": 0.2912946566939354,
      "reward_std": 0.016127016162499785,
      "rewards/accuracy_reward": 0.0357142873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2555803656578064,
      "step": 80
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2025.88623046875,
      "epoch": 0.02419535508923904,
      "grad_norm": 0.07038428634405136,
      "kl": 0.0001671314239501953,
      "learning_rate": 1.208955223880597e-07,
      "loss": 0.0085,
      "reward": 0.364955373108387,
      "reward_std": 0.02935676625929773,
      "rewards/accuracy_reward": 0.1026785783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2622767984867096,
      "step": 81
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2041.138427734375,
      "epoch": 0.024494063176760512,
      "grad_norm": 0.07874462753534317,
      "kl": 0.00017833709716796875,
      "learning_rate": 1.2238805970149254e-07,
      "loss": 0.0071,
      "reward": 0.2639509066939354,
      "reward_std": 0.042566894786432385,
      "rewards/accuracy_reward": 0.004464285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2594866156578064,
      "step": 82
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2001.4063110351562,
      "epoch": 0.02479277126428198,
      "grad_norm": 0.08482035994529724,
      "kl": 0.00017404556274414062,
      "learning_rate": 1.2388059701492537e-07,
      "loss": 0.022,
      "reward": 0.2829241156578064,
      "reward_std": 0.03838158119469881,
      "rewards/accuracy_reward": 0.004464285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.278459832072258,
      "step": 83
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2018.4666137695312,
      "epoch": 0.02509147935180345,
      "grad_norm": 0.06894195824861526,
      "kl": 0.0001761913299560547,
      "learning_rate": 1.253731343283582e-07,
      "loss": 0.0142,
      "reward": 0.3119419738650322,
      "reward_std": 0.05138242733664811,
      "rewards/accuracy_reward": 0.04687500186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2650669738650322,
      "step": 84
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2014.7991333007812,
      "epoch": 0.02539018743932492,
      "grad_norm": 0.10145162791013718,
      "kl": 0.0001747608184814453,
      "learning_rate": 1.2686567164179106e-07,
      "loss": 0.0171,
      "reward": 0.3989955484867096,
      "reward_std": 0.12134538032114506,
      "rewards/accuracy_reward": 0.1272321492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2717633992433548,
      "step": 85
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2041.6652221679688,
      "epoch": 0.02568889552684639,
      "grad_norm": 0.05340084061026573,
      "kl": 0.0001823902130126953,
      "learning_rate": 1.2835820895522386e-07,
      "loss": 0.0067,
      "reward": 0.2963169738650322,
      "reward_std": 0.03224052069708705,
      "rewards/accuracy_reward": 0.042410716181620955,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2539062574505806,
      "step": 86
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2021.9666137695312,
      "epoch": 0.02598760361436786,
      "grad_norm": 0.09708087891340256,
      "kl": 0.00016617774963378906,
      "learning_rate": 1.2985074626865672e-07,
      "loss": 0.0223,
      "reward": 0.353794664144516,
      "reward_std": 0.07332677789963782,
      "rewards/accuracy_reward": 0.0870535746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.266741082072258,
      "step": 87
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2021.8527526855469,
      "epoch": 0.026286311701889328,
      "grad_norm": 0.06138848513364792,
      "kl": 0.000179290771484375,
      "learning_rate": 1.3134328358208955e-07,
      "loss": 0.0059,
      "reward": 0.2823660895228386,
      "reward_std": 0.03848878503777087,
      "rewards/accuracy_reward": 0.01785714295692742,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2645089477300644,
      "step": 88
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2027.1541137695312,
      "epoch": 0.0265850197894108,
      "grad_norm": 0.10084333270788193,
      "kl": 0.0001804828643798828,
      "learning_rate": 1.328358208955224e-07,
      "loss": 0.0158,
      "reward": 0.3476562649011612,
      "reward_std": 0.07806684356182814,
      "rewards/accuracy_reward": 0.08258929080329835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2650669813156128,
      "step": 89
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2025.8303833007812,
      "epoch": 0.02688372787693227,
      "grad_norm": 0.05966576561331749,
      "kl": 0.00017905235290527344,
      "learning_rate": 1.343283582089552e-07,
      "loss": 0.0094,
      "reward": 0.293526791036129,
      "reward_std": 0.008457264630123973,
      "rewards/accuracy_reward": 0.0357142873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2578125074505806,
      "step": 90
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2012.3058776855469,
      "epoch": 0.027182435964453737,
      "grad_norm": 0.08248227089643478,
      "kl": 0.00018858909606933594,
      "learning_rate": 1.3582089552238803e-07,
      "loss": 0.012,
      "reward": 0.2929687574505806,
      "reward_std": 0.07058397959917784,
      "rewards/accuracy_reward": 0.02455357206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2684151977300644,
      "step": 91
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2025.9219360351562,
      "epoch": 0.027481144051975208,
      "grad_norm": 0.07628428936004639,
      "kl": 0.0001621246337890625,
      "learning_rate": 1.373134328358209e-07,
      "loss": 0.0196,
      "reward": 0.2672991156578064,
      "reward_std": 0.037932771956548095,
      "rewards/accuracy_reward": 0.004464285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.262834832072258,
      "step": 92
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2007.9419860839844,
      "epoch": 0.027779852139496675,
      "grad_norm": 0.08513079583644867,
      "kl": 0.00016760826110839844,
      "learning_rate": 1.3880597014925372e-07,
      "loss": 0.02,
      "reward": 0.3376116156578064,
      "reward_std": 0.0704271961003542,
      "rewards/accuracy_reward": 0.07142857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.266183041036129,
      "step": 93
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2043.3304443359375,
      "epoch": 0.028078560227018146,
      "grad_norm": 0.07018104195594788,
      "kl": 0.00017690658569335938,
      "learning_rate": 1.4029850746268658e-07,
      "loss": 0.0082,
      "reward": 0.2544642984867096,
      "reward_std": 0.016442545456811786,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2544642984867096,
      "step": 94
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2038.9465026855469,
      "epoch": 0.028377268314539617,
      "grad_norm": 0.051355332136154175,
      "kl": 0.00018906593322753906,
      "learning_rate": 1.4179104477611938e-07,
      "loss": 0.0034,
      "reward": 0.2935268059372902,
      "reward_std": 0.014339675661176443,
      "rewards/accuracy_reward": 0.03794643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2555803656578064,
      "step": 95
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2035.7902221679688,
      "epoch": 0.028675976402061085,
      "grad_norm": 0.06005255505442619,
      "kl": 0.0001811981201171875,
      "learning_rate": 1.4328358208955223e-07,
      "loss": 0.0061,
      "reward": 0.3074776977300644,
      "reward_std": 0.04148227581754327,
      "rewards/accuracy_reward": 0.04910714412108064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2583705559372902,
      "step": 96
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2009.4754943847656,
      "epoch": 0.028974684489582556,
      "grad_norm": 0.05179405212402344,
      "kl": 0.00016689300537109375,
      "learning_rate": 1.4477611940298506e-07,
      "loss": 0.0022,
      "reward": 0.2806919738650322,
      "reward_std": 0.03668789169751108,
      "rewards/accuracy_reward": 0.01116071455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2695312574505806,
      "step": 97
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2039.5647888183594,
      "epoch": 0.029273392577104027,
      "grad_norm": 0.06932333111763,
      "kl": 0.00017881393432617188,
      "learning_rate": 1.4626865671641792e-07,
      "loss": 0.0054,
      "reward": 0.3816964402794838,
      "reward_std": 0.08063330501317978,
      "rewards/accuracy_reward": 0.1205357201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2611607238650322,
      "step": 98
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2024.915283203125,
      "epoch": 0.029572100664625494,
      "grad_norm": 0.0861809253692627,
      "kl": 0.0001742839813232422,
      "learning_rate": 1.4776119402985075e-07,
      "loss": 0.0137,
      "reward": 0.2639509066939354,
      "reward_std": 0.027508222497999668,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2639509066939354,
      "step": 99
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1998.0201721191406,
      "epoch": 0.029870808752146965,
      "grad_norm": 0.07582151889801025,
      "kl": 0.00018906593322753906,
      "learning_rate": 1.4925373134328355e-07,
      "loss": 0.0295,
      "reward": 0.3164062649011612,
      "reward_std": 0.04657869855873287,
      "rewards/accuracy_reward": 0.053571430034935474,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2628348395228386,
      "step": 100
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2043.654052734375,
      "epoch": 0.030169516839668432,
      "grad_norm": 0.07266262918710709,
      "kl": 0.00016546249389648438,
      "learning_rate": 1.507462686567164e-07,
      "loss": 0.0059,
      "reward": 0.2963169738650322,
      "reward_std": 0.02936145500279963,
      "rewards/accuracy_reward": 0.04017857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2561384066939354,
      "step": 101
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2044.7969360351562,
      "epoch": 0.030468224927189903,
      "grad_norm": 0.06327736377716064,
      "kl": 0.0001804828643798828,
      "learning_rate": 1.5223880597014924e-07,
      "loss": 0.0031,
      "reward": 0.3264509066939354,
      "reward_std": 0.013739095069468021,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.255022332072258,
      "step": 102
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2034.2098693847656,
      "epoch": 0.030766933014711374,
      "grad_norm": 0.060658276081085205,
      "kl": 0.00017571449279785156,
      "learning_rate": 1.537313432835821e-07,
      "loss": 0.0087,
      "reward": 0.2935267984867096,
      "reward_std": 0.01807763520628214,
      "rewards/accuracy_reward": 0.03794643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.255580373108387,
      "step": 103
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2018.9576416015625,
      "epoch": 0.03106564110223284,
      "grad_norm": 0.0794973373413086,
      "kl": 0.00017499923706054688,
      "learning_rate": 1.5522388059701492e-07,
      "loss": 0.0126,
      "reward": 0.368861623108387,
      "reward_std": 0.051864683628082275,
      "rewards/accuracy_reward": 0.10714286309666932,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2617187574505806,
      "step": 104
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2044.6964416503906,
      "epoch": 0.03136434918975431,
      "grad_norm": 0.05705650523304939,
      "kl": 0.000179290771484375,
      "learning_rate": 1.5671641791044775e-07,
      "loss": 0.0062,
      "reward": 0.325892873108387,
      "reward_std": 0.017857143888249993,
      "rewards/accuracy_reward": 0.0736607164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2522321492433548,
      "step": 105
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2047.9464416503906,
      "epoch": 0.03166305727727578,
      "grad_norm": 0.03761906176805496,
      "kl": 0.00017333030700683594,
      "learning_rate": 1.5820895522388058e-07,
      "loss": 0.0001,
      "reward": 0.2578125074505806,
      "reward_std": 0.024980786256492138,
      "rewards/accuracy_reward": 0.006696428870782256,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.251116082072258,
      "step": 106
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2037.4933776855469,
      "epoch": 0.03196176536479725,
      "grad_norm": 0.08437875658273697,
      "kl": 0.0001685619354248047,
      "learning_rate": 1.5970149253731344e-07,
      "loss": 0.0108,
      "reward": 0.3035714402794838,
      "reward_std": 0.040224216878414154,
      "rewards/accuracy_reward": 0.0424107164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2611607238650322,
      "step": 107
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2042.9509582519531,
      "epoch": 0.03226047345231872,
      "grad_norm": 0.06780313700437546,
      "kl": 0.00017261505126953125,
      "learning_rate": 1.6119402985074627e-07,
      "loss": 0.006,
      "reward": 0.2712053656578064,
      "reward_std": 0.043657024623826146,
      "rewards/accuracy_reward": 0.013392857741564512,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2578125149011612,
      "step": 108
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2022.8505249023438,
      "epoch": 0.03255918153984019,
      "grad_norm": 0.08141493797302246,
      "kl": 0.00018525123596191406,
      "learning_rate": 1.626865671641791e-07,
      "loss": 0.0163,
      "reward": 0.3727678805589676,
      "reward_std": 0.035714288242161274,
      "rewards/accuracy_reward": 0.1116071455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2611607313156128,
      "step": 109
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1993.1295471191406,
      "epoch": 0.032857889627361664,
      "grad_norm": 0.08384981006383896,
      "kl": 0.0001888275146484375,
      "learning_rate": 1.6417910447761193e-07,
      "loss": 0.0195,
      "reward": 0.2840401902794838,
      "reward_std": 0.0435154908336699,
      "rewards/accuracy_reward": 0.008928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2751116156578064,
      "step": 110
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2039.0670166015625,
      "epoch": 0.03315659771488313,
      "grad_norm": 0.07148782163858414,
      "kl": 0.00017333030700683594,
      "learning_rate": 1.6567164179104476e-07,
      "loss": 0.0052,
      "reward": 0.2661830559372902,
      "reward_std": 0.03985769907012582,
      "rewards/accuracy_reward": 0.006696428870782256,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2594866305589676,
      "step": 111
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2002.8214721679688,
      "epoch": 0.0334553058024046,
      "grad_norm": 0.07812437415122986,
      "kl": 0.000186920166015625,
      "learning_rate": 1.671641791044776e-07,
      "loss": 0.0165,
      "reward": 0.3002232238650322,
      "reward_std": 0.021530525060370564,
      "rewards/accuracy_reward": 0.0379464291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.262276791036129,
      "step": 112
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2044.3326416015625,
      "epoch": 0.03375401388992607,
      "grad_norm": 0.05895126983523369,
      "kl": 0.00018262863159179688,
      "learning_rate": 1.6865671641791044e-07,
      "loss": 0.0049,
      "reward": 0.361049123108387,
      "reward_std": 0.011113195912912488,
      "rewards/accuracy_reward": 0.1071428619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2539062649011612,
      "step": 113
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2018.3460693359375,
      "epoch": 0.03405272197744754,
      "grad_norm": 0.08534523099660873,
      "kl": 0.000171661376953125,
      "learning_rate": 1.701492537313433e-07,
      "loss": 0.0188,
      "reward": 0.3588169813156128,
      "reward_std": 0.06383948028087616,
      "rewards/accuracy_reward": 0.08928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2695312574505806,
      "step": 114
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2042.1027221679688,
      "epoch": 0.03435143006496901,
      "grad_norm": 0.0591108538210392,
      "kl": 0.0001659393310546875,
      "learning_rate": 1.716417910447761e-07,
      "loss": 0.0048,
      "reward": 0.258928582072258,
      "reward_std": 0.022477608872577548,
      "rewards/accuracy_reward": 0.0022321429569274187,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2566964402794838,
      "step": 115
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2036.5558776855469,
      "epoch": 0.034650138152490476,
      "grad_norm": 0.0832672119140625,
      "kl": 0.0001685619354248047,
      "learning_rate": 1.7313432835820896e-07,
      "loss": 0.0105,
      "reward": 0.2594866156578064,
      "reward_std": 0.0251810597255826,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2594866156578064,
      "step": 116
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.034948846240011947,
      "grad_norm": 0.0015277797356247902,
      "kl": 0.0001728534698486328,
      "learning_rate": 1.7462686567164178e-07,
      "loss": 0.0,
      "reward": 0.321428582072258,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 117
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2019.4844360351562,
      "epoch": 0.03524755432753342,
      "grad_norm": 0.08572108298540115,
      "kl": 0.0001685619354248047,
      "learning_rate": 1.7611940298507461e-07,
      "loss": 0.0139,
      "reward": 0.4123884066939354,
      "reward_std": 0.02840944961644709,
      "rewards/accuracy_reward": 0.1428571492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2695312723517418,
      "step": 118
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2033.4978332519531,
      "epoch": 0.03554626241505489,
      "grad_norm": 0.07758825272321701,
      "kl": 0.00016450881958007812,
      "learning_rate": 1.7761194029850744e-07,
      "loss": 0.0108,
      "reward": 0.3225446566939354,
      "reward_std": 0.05096272611990571,
      "rewards/accuracy_reward": 0.06250000232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2600446492433548,
      "step": 119
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2034.2254943847656,
      "epoch": 0.03584497050257636,
      "grad_norm": 0.049877848476171494,
      "kl": 0.0001666545867919922,
      "learning_rate": 1.7910447761194027e-07,
      "loss": 0.0058,
      "reward": 0.309151791036129,
      "reward_std": 0.029003681614995003,
      "rewards/accuracy_reward": 0.0491071455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2600446492433548,
      "step": 120
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1999.0000610351562,
      "epoch": 0.03614367859009783,
      "grad_norm": 0.08883936703205109,
      "kl": 0.0001862049102783203,
      "learning_rate": 1.8059701492537313e-07,
      "loss": 0.0261,
      "reward": 0.3191964402794838,
      "reward_std": 0.0875182505697012,
      "rewards/accuracy_reward": 0.04464285937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.274553582072258,
      "step": 121
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2005.6429138183594,
      "epoch": 0.036442386677619294,
      "grad_norm": 0.09178085625171661,
      "kl": 0.00017452239990234375,
      "learning_rate": 1.8208955223880596e-07,
      "loss": 0.0188,
      "reward": 0.2695312574505806,
      "reward_std": 0.026280531659722328,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2695312574505806,
      "step": 122
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2016.9197082519531,
      "epoch": 0.036741094765140765,
      "grad_norm": 0.06954555213451385,
      "kl": 0.00016760826110839844,
      "learning_rate": 1.8358208955223881e-07,
      "loss": 0.0176,
      "reward": 0.3052455559372902,
      "reward_std": 0.038904729299247265,
      "rewards/accuracy_reward": 0.04017857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2650669813156128,
      "step": 123
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2041.5916137695312,
      "epoch": 0.037039802852662236,
      "grad_norm": 0.06397812813520432,
      "kl": 0.00016689300537109375,
      "learning_rate": 1.8507462686567162e-07,
      "loss": 0.0067,
      "reward": 0.3018973246216774,
      "reward_std": 0.030726489843800664,
      "rewards/accuracy_reward": 0.044642859138548374,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2572544738650322,
      "step": 124
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2006.9866638183594,
      "epoch": 0.03733851094018371,
      "grad_norm": 0.09185361117124557,
      "kl": 0.00017261505126953125,
      "learning_rate": 1.8656716417910447e-07,
      "loss": 0.0185,
      "reward": 0.3487723395228386,
      "reward_std": 0.042637437116354704,
      "rewards/accuracy_reward": 0.0781250037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2706473246216774,
      "step": 125
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2046.5848999023438,
      "epoch": 0.03763721902770518,
      "grad_norm": 0.05983275547623634,
      "kl": 0.00016832351684570312,
      "learning_rate": 1.880597014925373e-07,
      "loss": 0.0026,
      "reward": 0.3621651977300644,
      "reward_std": 0.020089286379516125,
      "rewards/accuracy_reward": 0.10937500488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2527901977300644,
      "step": 126
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2016.3952026367188,
      "epoch": 0.03793592711522664,
      "grad_norm": 0.09380698204040527,
      "kl": 0.00017547607421875,
      "learning_rate": 1.8955223880597013e-07,
      "loss": 0.0224,
      "reward": 0.380580373108387,
      "reward_std": 0.045238421531394124,
      "rewards/accuracy_reward": 0.11160714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2689732238650322,
      "step": 127
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2045.8505249023438,
      "epoch": 0.03823463520274811,
      "grad_norm": 0.08567311614751816,
      "kl": 0.00015878677368164062,
      "learning_rate": 1.91044776119403e-07,
      "loss": 0.0034,
      "reward": 0.298549123108387,
      "reward_std": 0.0419728672131896,
      "rewards/accuracy_reward": 0.04241071757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2561384066939354,
      "step": 128
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1984.0290832519531,
      "epoch": 0.038533343290269584,
      "grad_norm": 0.11230553686618805,
      "kl": 0.00020241737365722656,
      "learning_rate": 1.925373134328358e-07,
      "loss": 0.0317,
      "reward": 0.4575893059372902,
      "reward_std": 0.10603193193674088,
      "rewards/accuracy_reward": 0.1785714365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2790178656578064,
      "step": 129
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2000.2120971679688,
      "epoch": 0.038832051377791055,
      "grad_norm": 0.08793243020772934,
      "kl": 0.00019097328186035156,
      "learning_rate": 1.9402985074626865e-07,
      "loss": 0.0255,
      "reward": 0.3510044738650322,
      "reward_std": 0.0863843634724617,
      "rewards/accuracy_reward": 0.07142857206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2795759066939354,
      "step": 130
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2026.97998046875,
      "epoch": 0.039130759465312526,
      "grad_norm": 0.06578430533409119,
      "kl": 0.00017571449279785156,
      "learning_rate": 1.9552238805970148e-07,
      "loss": 0.0128,
      "reward": 0.274553582072258,
      "reward_std": 0.048977102153003216,
      "rewards/accuracy_reward": 0.013392857508733869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2611607238650322,
      "step": 131
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2028.12060546875,
      "epoch": 0.03942946755283399,
      "grad_norm": 0.07726174592971802,
      "kl": 0.00016307830810546875,
      "learning_rate": 1.9701492537313433e-07,
      "loss": 0.0166,
      "reward": 0.2779017984867096,
      "reward_std": 0.05110340751707554,
      "rewards/accuracy_reward": 0.01785714295692742,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2600446566939354,
      "step": 132
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2040.2098693847656,
      "epoch": 0.03972817564035546,
      "grad_norm": 0.0705326646566391,
      "kl": 0.0001780986785888672,
      "learning_rate": 1.9850746268656716e-07,
      "loss": 0.0041,
      "reward": 0.368861623108387,
      "reward_std": 0.019334469456225634,
      "rewards/accuracy_reward": 0.1071428619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2617187574505806,
      "step": 133
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2035.1786193847656,
      "epoch": 0.04002688372787693,
      "grad_norm": 0.08128512650728226,
      "kl": 0.00018453598022460938,
      "learning_rate": 2e-07,
      "loss": 0.0095,
      "reward": 0.334263414144516,
      "reward_std": 0.02304393658414483,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.262834832072258,
      "step": 134
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2009.9956359863281,
      "epoch": 0.0403255918153984,
      "grad_norm": 0.09043879806995392,
      "kl": 0.0001990795135498047,
      "learning_rate": 2.0149253731343282e-07,
      "loss": 0.0151,
      "reward": 0.3342634066939354,
      "reward_std": 0.05122541659511626,
      "rewards/accuracy_reward": 0.06473214668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2695312649011612,
      "step": 135
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2045.732177734375,
      "epoch": 0.04062429990291987,
      "grad_norm": 0.05822138860821724,
      "kl": 0.00017023086547851562,
      "learning_rate": 2.0298507462686565e-07,
      "loss": 0.003,
      "reward": 0.2539062574505806,
      "reward_std": 0.01252779457718134,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2539062574505806,
      "step": 136
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2042.2790832519531,
      "epoch": 0.040923007990441344,
      "grad_norm": 0.06518001854419708,
      "kl": 0.00017118453979492188,
      "learning_rate": 2.044776119402985e-07,
      "loss": 0.0047,
      "reward": 0.2566964402794838,
      "reward_std": 0.017070032423362136,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2566964402794838,
      "step": 137
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2039.3973999023438,
      "epoch": 0.04122171607796281,
      "grad_norm": 0.06714539229869843,
      "kl": 0.00017905235290527344,
      "learning_rate": 2.0597014925373134e-07,
      "loss": 0.0059,
      "reward": 0.3074776977300644,
      "reward_std": 0.04616858554072678,
      "rewards/accuracy_reward": 0.0491071455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2583705484867096,
      "step": 138
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2017.4219360351562,
      "epoch": 0.04152042416548428,
      "grad_norm": 0.07822741568088531,
      "kl": 0.00020813941955566406,
      "learning_rate": 2.0746268656716416e-07,
      "loss": 0.017,
      "reward": 0.4458705633878708,
      "reward_std": 0.06650421163067222,
      "rewards/accuracy_reward": 0.17857143841683865,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2672991156578064,
      "step": 139
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1998.2366943359375,
      "epoch": 0.04181913225300575,
      "grad_norm": 0.07849594205617905,
      "kl": 0.0001964569091796875,
      "learning_rate": 2.08955223880597e-07,
      "loss": 0.0147,
      "reward": 0.2963169738650322,
      "reward_std": 0.07019457221031189,
      "rewards/accuracy_reward": 0.022321429569274187,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.273995541036129,
      "step": 140
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2026.83935546875,
      "epoch": 0.04211784034052722,
      "grad_norm": 0.07670233398675919,
      "kl": 0.00019168853759765625,
      "learning_rate": 2.1044776119402985e-07,
      "loss": 0.0124,
      "reward": 0.2829241156578064,
      "reward_std": 0.052819991018623114,
      "rewards/accuracy_reward": 0.0200892873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.262834832072258,
      "step": 141
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2015.1161193847656,
      "epoch": 0.04241654842804869,
      "grad_norm": 0.08329468965530396,
      "kl": 0.00020742416381835938,
      "learning_rate": 2.1194029850746268e-07,
      "loss": 0.0171,
      "reward": 0.2756696492433548,
      "reward_std": 0.0507113803178072,
      "rewards/accuracy_reward": 0.008928572060540318,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2667410746216774,
      "step": 142
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2008.1317749023438,
      "epoch": 0.042715256515570156,
      "grad_norm": 0.07765533030033112,
      "kl": 0.00021910667419433594,
      "learning_rate": 2.1343283582089554e-07,
      "loss": 0.0167,
      "reward": 0.3582589328289032,
      "reward_std": 0.037035543005913496,
      "rewards/accuracy_reward": 0.0915178582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.266741082072258,
      "step": 143
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2003.9531860351562,
      "epoch": 0.04301396460309163,
      "grad_norm": 0.07212961465120316,
      "kl": 0.0002162456512451172,
      "learning_rate": 2.1492537313432834e-07,
      "loss": 0.0095,
      "reward": 0.2818080484867096,
      "reward_std": 0.04215896176174283,
      "rewards/accuracy_reward": 0.01562500116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.266183041036129,
      "step": 144
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2016.5804138183594,
      "epoch": 0.0433126726906131,
      "grad_norm": 0.09445866197347641,
      "kl": 0.00021696090698242188,
      "learning_rate": 2.1641791044776117e-07,
      "loss": 0.0177,
      "reward": 0.3085937574505806,
      "reward_std": 0.05049149505794048,
      "rewards/accuracy_reward": 0.04017857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2684151828289032,
      "step": 145
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2025.1273498535156,
      "epoch": 0.04361138077813457,
      "grad_norm": 0.07148821651935577,
      "kl": 0.0002124309539794922,
      "learning_rate": 2.1791044776119402e-07,
      "loss": 0.0092,
      "reward": 0.310267873108387,
      "reward_std": 0.03736744727939367,
      "rewards/accuracy_reward": 0.04241071757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2678571566939354,
      "step": 146
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2030.2411499023438,
      "epoch": 0.04391008886565604,
      "grad_norm": 0.0774233415722847,
      "kl": 0.000213623046875,
      "learning_rate": 2.1940298507462685e-07,
      "loss": 0.0142,
      "reward": 0.3219866156578064,
      "reward_std": 0.07287575164809823,
      "rewards/accuracy_reward": 0.058035715483129025,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2639509066939354,
      "step": 147
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1998.4263916015625,
      "epoch": 0.04420879695317751,
      "grad_norm": 0.0685528814792633,
      "kl": 0.00023698806762695312,
      "learning_rate": 2.208955223880597e-07,
      "loss": 0.0138,
      "reward": 0.3002232238650322,
      "reward_std": 0.008928572060540318,
      "rewards/accuracy_reward": 0.0357142873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2645089402794838,
      "step": 148
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2000.587158203125,
      "epoch": 0.044507505040698975,
      "grad_norm": 0.09035354852676392,
      "kl": 0.00025200843811035156,
      "learning_rate": 2.223880597014925e-07,
      "loss": 0.0241,
      "reward": 0.314174123108387,
      "reward_std": 0.04314139811322093,
      "rewards/accuracy_reward": 0.0401785746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2739955484867096,
      "step": 149
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2043.4085693359375,
      "epoch": 0.044806213128220446,
      "grad_norm": 0.059354864060878754,
      "kl": 0.0001952648162841797,
      "learning_rate": 2.2388059701492537e-07,
      "loss": 0.0062,
      "reward": 0.2594866156578064,
      "reward_std": 0.02636163542047143,
      "rewards/accuracy_reward": 0.004464285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.255022332072258,
      "step": 150
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2017.7724304199219,
      "epoch": 0.04510492121574192,
      "grad_norm": 0.08813710510730743,
      "kl": 0.0002467632293701172,
      "learning_rate": 2.253731343283582e-07,
      "loss": 0.0144,
      "reward": 0.2896205484867096,
      "reward_std": 0.052120857406407595,
      "rewards/accuracy_reward": 0.022321430267766118,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2672991156578064,
      "step": 151
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2014.7522888183594,
      "epoch": 0.04540362930326339,
      "grad_norm": 0.07390749454498291,
      "kl": 0.00024819374084472656,
      "learning_rate": 2.2686567164179105e-07,
      "loss": 0.015,
      "reward": 0.345424123108387,
      "reward_std": 0.04857433959841728,
      "rewards/accuracy_reward": 0.082589291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.262834832072258,
      "step": 152
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2034.9889221191406,
      "epoch": 0.04570233739078486,
      "grad_norm": 0.07716519385576248,
      "kl": 0.0002162456512451172,
      "learning_rate": 2.2835820895522386e-07,
      "loss": 0.0137,
      "reward": 0.3738839402794838,
      "reward_std": 0.046810401137918234,
      "rewards/accuracy_reward": 0.113839291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2600446566939354,
      "step": 153
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2038.6629943847656,
      "epoch": 0.04600104547830632,
      "grad_norm": 0.06641247123479843,
      "kl": 0.00021338462829589844,
      "learning_rate": 2.2985074626865669e-07,
      "loss": 0.0085,
      "reward": 0.2606026902794838,
      "reward_std": 0.02775944024324417,
      "rewards/accuracy_reward": 0.0022321429569274187,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2583705484867096,
      "step": 154
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2012.1495971679688,
      "epoch": 0.04629975356582779,
      "grad_norm": 0.07931293547153473,
      "kl": 0.00025773048400878906,
      "learning_rate": 2.3134328358208954e-07,
      "loss": 0.0142,
      "reward": 0.3995535895228386,
      "reward_std": 0.05525131174363196,
      "rewards/accuracy_reward": 0.13169643469154835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.267857164144516,
      "step": 155
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2048.0,
      "epoch": 0.046598461653349264,
      "grad_norm": 0.004591322038322687,
      "kl": 0.00019311904907226562,
      "learning_rate": 2.3283582089552237e-07,
      "loss": 0.0,
      "reward": 0.357142873108387,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.1071428619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.25,
      "step": 156
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2019.1652526855469,
      "epoch": 0.046897169740870735,
      "grad_norm": 0.0849694237112999,
      "kl": 0.0002593994140625,
      "learning_rate": 2.3432835820895523e-07,
      "loss": 0.015,
      "reward": 0.314174123108387,
      "reward_std": 0.05707283806987107,
      "rewards/accuracy_reward": 0.044642860535532236,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2695312649011612,
      "step": 157
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2024.0000305175781,
      "epoch": 0.047195877828392206,
      "grad_norm": 0.066276416182518,
      "kl": 0.0002455711364746094,
      "learning_rate": 2.3582089552238803e-07,
      "loss": 0.0057,
      "reward": 0.2806919664144516,
      "reward_std": 0.05360085237771273,
      "rewards/accuracy_reward": 0.011160715017467737,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2695312574505806,
      "step": 158
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2023.3482666015625,
      "epoch": 0.04749458591591367,
      "grad_norm": 0.07106047123670578,
      "kl": 0.00024819374084472656,
      "learning_rate": 2.3731343283582089e-07,
      "loss": 0.0123,
      "reward": 0.3454241082072258,
      "reward_std": 0.03872924018651247,
      "rewards/accuracy_reward": 0.08035714644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2650669738650322,
      "step": 159
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2034.930908203125,
      "epoch": 0.04779329400343514,
      "grad_norm": 0.08272335678339005,
      "kl": 0.00025153160095214844,
      "learning_rate": 2.388059701492537e-07,
      "loss": 0.0087,
      "reward": 0.439174123108387,
      "reward_std": 0.07193068787455559,
      "rewards/accuracy_reward": 0.1741071455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2650669738650322,
      "step": 160
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2000.8148193359375,
      "epoch": 0.04809200209095661,
      "grad_norm": 0.09384261071681976,
      "kl": 0.0003211498260498047,
      "learning_rate": 2.4029850746268654e-07,
      "loss": 0.0226,
      "reward": 0.3387276902794838,
      "reward_std": 0.07356675760820508,
      "rewards/accuracy_reward": 0.06473214644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2739955559372902,
      "step": 161
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2002.7879943847656,
      "epoch": 0.04839071017847808,
      "grad_norm": 0.08281215280294418,
      "kl": 0.00030422210693359375,
      "learning_rate": 2.417910447761194e-07,
      "loss": 0.0133,
      "reward": 0.3069196566939354,
      "reward_std": 0.030919374199584126,
      "rewards/accuracy_reward": 0.04017857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.266741082072258,
      "step": 162
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2042.8572082519531,
      "epoch": 0.048689418265999554,
      "grad_norm": 0.06405437737703323,
      "kl": 0.000217437744140625,
      "learning_rate": 2.432835820895522e-07,
      "loss": 0.0066,
      "reward": 0.2963169738650322,
      "reward_std": 0.0340583180077374,
      "rewards/accuracy_reward": 0.04017857206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2561383992433548,
      "step": 163
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2036.7857971191406,
      "epoch": 0.048988126353521025,
      "grad_norm": 0.0808204635977745,
      "kl": 0.0002453327178955078,
      "learning_rate": 2.447761194029851e-07,
      "loss": 0.0067,
      "reward": 0.3125000149011612,
      "reward_std": 0.056629103841260076,
      "rewards/accuracy_reward": 0.049107146449387074,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2633928656578064,
      "step": 164
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2017.0023193359375,
      "epoch": 0.04928683444104249,
      "grad_norm": 0.0806288942694664,
      "kl": 0.0002999305725097656,
      "learning_rate": 2.4626865671641786e-07,
      "loss": 0.0133,
      "reward": 0.305803582072258,
      "reward_std": 0.0739367906935513,
      "rewards/accuracy_reward": 0.0334821455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2723214402794838,
      "step": 165
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1993.0089721679688,
      "epoch": 0.04958554252856396,
      "grad_norm": 0.10269467532634735,
      "kl": 0.00039839744567871094,
      "learning_rate": 2.4776119402985074e-07,
      "loss": 0.0286,
      "reward": 0.4095982313156128,
      "reward_std": 0.11181990662589669,
      "rewards/accuracy_reward": 0.129464291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2801339477300644,
      "step": 166
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2015.4710693359375,
      "epoch": 0.04988425061608543,
      "grad_norm": 0.09286423772573471,
      "kl": 0.00032258033752441406,
      "learning_rate": 2.492537313432836e-07,
      "loss": 0.0183,
      "reward": 0.3833705559372902,
      "reward_std": 0.049257969949394464,
      "rewards/accuracy_reward": 0.113839291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2695312574505806,
      "step": 167
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2012.0982666015625,
      "epoch": 0.0501829587036069,
      "grad_norm": 0.09242697060108185,
      "kl": 0.0003287792205810547,
      "learning_rate": 2.507462686567164e-07,
      "loss": 0.0272,
      "reward": 0.275111623108387,
      "reward_std": 0.059367209672927856,
      "rewards/accuracy_reward": 0.006696428870782256,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2684151902794838,
      "step": 168
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1978.33935546875,
      "epoch": 0.05048166679112837,
      "grad_norm": 0.08841448277235031,
      "kl": 0.0004553794860839844,
      "learning_rate": 2.5223880597014923e-07,
      "loss": 0.0081,
      "reward": 0.2756696566939354,
      "reward_std": 0.033615279011428356,
      "rewards/accuracy_reward": 0.004464285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.271205373108387,
      "step": 169
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2044.5781555175781,
      "epoch": 0.05078037487864984,
      "grad_norm": 0.06271202862262726,
      "kl": 0.0002613067626953125,
      "learning_rate": 2.537313432835821e-07,
      "loss": 0.0011,
      "reward": 0.368303582072258,
      "reward_std": 0.07509586028754711,
      "rewards/accuracy_reward": 0.10491071827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2633928656578064,
      "step": 170
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1972.055908203125,
      "epoch": 0.05107908296617131,
      "grad_norm": 0.09658797830343246,
      "kl": 0.0004730224609375,
      "learning_rate": 2.5522388059701494e-07,
      "loss": 0.0326,
      "reward": 0.3498884066939354,
      "reward_std": 0.03977184183895588,
      "rewards/accuracy_reward": 0.0691964328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2806919738650322,
      "step": 171
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2000.3907165527344,
      "epoch": 0.05137779105369278,
      "grad_norm": 0.08945363014936447,
      "kl": 0.0004181861877441406,
      "learning_rate": 2.567164179104477e-07,
      "loss": 0.0162,
      "reward": 0.340401791036129,
      "reward_std": 0.08885344956070185,
      "rewards/accuracy_reward": 0.064732147147879,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2756696566939354,
      "step": 172
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2023.0536499023438,
      "epoch": 0.05167649914121425,
      "grad_norm": 0.06620240211486816,
      "kl": 0.0003552436828613281,
      "learning_rate": 2.582089552238806e-07,
      "loss": 0.0092,
      "reward": 0.3158482238650322,
      "reward_std": 0.041362314485013485,
      "rewards/accuracy_reward": 0.05133928777649999,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2645089402794838,
      "step": 173
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2024.8192443847656,
      "epoch": 0.05197520722873572,
      "grad_norm": 0.07562818378210068,
      "kl": 0.0003762245178222656,
      "learning_rate": 2.5970149253731343e-07,
      "loss": 0.0108,
      "reward": 0.2689732238650322,
      "reward_std": 0.029865912161767483,
      "rewards/accuracy_reward": 0.0022321429569274187,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.266741082072258,
      "step": 174
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2031.05810546875,
      "epoch": 0.052273915316257184,
      "grad_norm": 0.06956419348716736,
      "kl": 0.0003361701965332031,
      "learning_rate": 2.611940298507462e-07,
      "loss": 0.0113,
      "reward": 0.299107164144516,
      "reward_std": 0.020119994645938277,
      "rewards/accuracy_reward": 0.0357142873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.263392873108387,
      "step": 175
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2034.6072082519531,
      "epoch": 0.052572623403778655,
      "grad_norm": 0.08091429620981216,
      "kl": 0.00033473968505859375,
      "learning_rate": 2.626865671641791e-07,
      "loss": 0.0096,
      "reward": 0.3286830484867096,
      "reward_std": 0.07504337094724178,
      "rewards/accuracy_reward": 0.06696428847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2617187649011612,
      "step": 176
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2033.3750610351562,
      "epoch": 0.052871331491300126,
      "grad_norm": 0.07236970961093903,
      "kl": 0.0003561973571777344,
      "learning_rate": 2.641791044776119e-07,
      "loss": 0.0104,
      "reward": 0.388392873108387,
      "reward_std": 0.05889256298542023,
      "rewards/accuracy_reward": 0.12723214784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2611607313156128,
      "step": 177
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2021.0201721191406,
      "epoch": 0.0531700395788216,
      "grad_norm": 0.08121785521507263,
      "kl": 0.0004134178161621094,
      "learning_rate": 2.656716417910448e-07,
      "loss": 0.0165,
      "reward": 0.3588169887661934,
      "reward_std": 0.07267192704603076,
      "rewards/accuracy_reward": 0.09375000488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2650669813156128,
      "step": 178
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2010.5983276367188,
      "epoch": 0.05346874766634307,
      "grad_norm": 0.09636127948760986,
      "kl": 0.00045299530029296875,
      "learning_rate": 2.671641791044776e-07,
      "loss": 0.0223,
      "reward": 0.3839285895228386,
      "reward_std": 0.06966402754187584,
      "rewards/accuracy_reward": 0.1093750074505806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.274553582072258,
      "step": 179
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2035.7701721191406,
      "epoch": 0.05376745575386454,
      "grad_norm": 0.09264959394931793,
      "kl": 0.0003719329833984375,
      "learning_rate": 2.686567164179104e-07,
      "loss": 0.0126,
      "reward": 0.314174123108387,
      "reward_std": 0.06786144804209471,
      "rewards/accuracy_reward": 0.051339288242161274,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2628348395228386,
      "step": 180
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1970.44873046875,
      "epoch": 0.054066163841386,
      "grad_norm": 0.09745245426893234,
      "kl": 0.0006742477416992188,
      "learning_rate": 2.701492537313433e-07,
      "loss": 0.0225,
      "reward": 0.318080373108387,
      "reward_std": 0.040587071562185884,
      "rewards/accuracy_reward": 0.04017857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2779018059372902,
      "step": 181
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2022.649658203125,
      "epoch": 0.054364871928907474,
      "grad_norm": 0.09323485195636749,
      "kl": 0.0004668235778808594,
      "learning_rate": 2.7164179104477607e-07,
      "loss": 0.0139,
      "reward": 0.3777901977300644,
      "reward_std": 0.047922884579747915,
      "rewards/accuracy_reward": 0.11160714668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2661830484867096,
      "step": 182
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2027.26123046875,
      "epoch": 0.054663580016428945,
      "grad_norm": 0.10046170651912689,
      "kl": 0.00045013427734375,
      "learning_rate": 2.7313432835820895e-07,
      "loss": 0.0174,
      "reward": 0.3270089402794838,
      "reward_std": 0.1327594155445695,
      "rewards/accuracy_reward": 0.049107145983725786,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2779017984867096,
      "step": 183
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1997.7746276855469,
      "epoch": 0.054962288103950416,
      "grad_norm": 0.08770574629306793,
      "kl": 0.0006394386291503906,
      "learning_rate": 2.746268656716418e-07,
      "loss": 0.0186,
      "reward": 0.3794643059372902,
      "reward_std": 0.04777905996888876,
      "rewards/accuracy_reward": 0.1026785746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2767857238650322,
      "step": 184
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2012.6808776855469,
      "epoch": 0.05526099619147189,
      "grad_norm": 0.08510146290063858,
      "kl": 0.000522613525390625,
      "learning_rate": 2.761194029850746e-07,
      "loss": 0.0111,
      "reward": 0.3231026902794838,
      "reward_std": 0.064674180932343,
      "rewards/accuracy_reward": 0.05133928800933063,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2717634066939354,
      "step": 185
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1981.0090026855469,
      "epoch": 0.05555970427899335,
      "grad_norm": 0.09995196014642715,
      "kl": 0.0007429122924804688,
      "learning_rate": 2.7761194029850744e-07,
      "loss": 0.0247,
      "reward": 0.3482143059372902,
      "reward_std": 0.04264641599729657,
      "rewards/accuracy_reward": 0.0758928619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2723214402794838,
      "step": 186
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2036.5826721191406,
      "epoch": 0.05585841236651482,
      "grad_norm": 0.06772337108850479,
      "kl": 0.0004420280456542969,
      "learning_rate": 2.7910447761194027e-07,
      "loss": 0.0054,
      "reward": 0.3554687649011612,
      "reward_std": 0.04325869586318731,
      "rewards/accuracy_reward": 0.0937500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2617187574505806,
      "step": 187
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2017.743408203125,
      "epoch": 0.05615712045403629,
      "grad_norm": 0.08378896862268448,
      "kl": 0.0005846023559570312,
      "learning_rate": 2.8059701492537315e-07,
      "loss": 0.012,
      "reward": 0.3738839477300644,
      "reward_std": 0.060776281636208296,
      "rewards/accuracy_reward": 0.0982142873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2756696566939354,
      "step": 188
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2031.747802734375,
      "epoch": 0.056455828541557763,
      "grad_norm": 0.060328032821416855,
      "kl": 0.0004730224609375,
      "learning_rate": 2.82089552238806e-07,
      "loss": 0.0074,
      "reward": 0.336495541036129,
      "reward_std": 0.0339016814250499,
      "rewards/accuracy_reward": 0.0803571455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2561384066939354,
      "step": 189
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2010.9331359863281,
      "epoch": 0.056754536629079234,
      "grad_norm": 0.1147855892777443,
      "kl": 0.000637054443359375,
      "learning_rate": 2.8358208955223876e-07,
      "loss": 0.0203,
      "reward": 0.3415178805589676,
      "reward_std": 0.07420498505234718,
      "rewards/accuracy_reward": 0.0669642873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2745535895228386,
      "step": 190
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2015.8929138183594,
      "epoch": 0.057053244716600705,
      "grad_norm": 0.0833522379398346,
      "kl": 0.000637054443359375,
      "learning_rate": 2.8507462686567164e-07,
      "loss": 0.0149,
      "reward": 0.3074776977300644,
      "reward_std": 0.0650724545121193,
      "rewards/accuracy_reward": 0.04017857206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.267299123108387,
      "step": 191
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2033.3504943847656,
      "epoch": 0.05735195280412217,
      "grad_norm": 0.07649856805801392,
      "kl": 0.000545501708984375,
      "learning_rate": 2.8656716417910447e-07,
      "loss": 0.0104,
      "reward": 0.3470982238650322,
      "reward_std": 0.05379192181862891,
      "rewards/accuracy_reward": 0.08928571757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2578125074505806,
      "step": 192
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1989.2947387695312,
      "epoch": 0.05765066089164364,
      "grad_norm": 0.11090464890003204,
      "kl": 0.0008726119995117188,
      "learning_rate": 2.880597014925373e-07,
      "loss": 0.0297,
      "reward": 0.3828125149011612,
      "reward_std": 0.0587000735104084,
      "rewards/accuracy_reward": 0.1071428582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2756696566939354,
      "step": 193
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2020.8683776855469,
      "epoch": 0.05794936897916511,
      "grad_norm": 0.09388872981071472,
      "kl": 0.0006723403930664062,
      "learning_rate": 2.8955223880597013e-07,
      "loss": 0.0177,
      "reward": 0.3046875074505806,
      "reward_std": 0.03817676939070225,
      "rewards/accuracy_reward": 0.03794643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.266741082072258,
      "step": 194
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1999.3527221679688,
      "epoch": 0.05824807706668658,
      "grad_norm": 0.07774338871240616,
      "kl": 0.0007452964782714844,
      "learning_rate": 2.9104477611940296e-07,
      "loss": 0.0094,
      "reward": 0.3264508992433548,
      "reward_std": 0.07910588569939137,
      "rewards/accuracy_reward": 0.058035717345774174,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2684151865541935,
      "step": 195
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2013.7143859863281,
      "epoch": 0.05854678515420805,
      "grad_norm": 0.10577115416526794,
      "kl": 0.0007781982421875,
      "learning_rate": 2.9253731343283584e-07,
      "loss": 0.0289,
      "reward": 0.345424123108387,
      "reward_std": 0.04334849026054144,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2739955559372902,
      "step": 196
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1994.1607666015625,
      "epoch": 0.05884549324172952,
      "grad_norm": 0.08896368741989136,
      "kl": 0.000904083251953125,
      "learning_rate": 2.940298507462686e-07,
      "loss": 0.0191,
      "reward": 0.2801339402794838,
      "reward_std": 0.045548498164862394,
      "rewards/accuracy_reward": 0.008928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2712053656578064,
      "step": 197
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2018.2813110351562,
      "epoch": 0.05914420132925099,
      "grad_norm": 0.09556843340396881,
      "kl": 0.0007686614990234375,
      "learning_rate": 2.955223880597015e-07,
      "loss": 0.0131,
      "reward": 0.3565848395228386,
      "reward_std": 0.10666803759522736,
      "rewards/accuracy_reward": 0.0848214328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2717634066939354,
      "step": 198
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2009.1942749023438,
      "epoch": 0.05944290941677246,
      "grad_norm": 0.0993853360414505,
      "kl": 0.0008955001831054688,
      "learning_rate": 2.9701492537313433e-07,
      "loss": 0.0222,
      "reward": 0.2812500149011612,
      "reward_std": 0.0642466563731432,
      "rewards/accuracy_reward": 0.011160714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2700892984867096,
      "step": 199
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2008.6295166015625,
      "epoch": 0.05974161750429393,
      "grad_norm": 0.09655937552452087,
      "kl": 0.0009717941284179688,
      "learning_rate": 2.985074626865671e-07,
      "loss": 0.0233,
      "reward": 0.275111623108387,
      "reward_std": 0.042582636466249824,
      "rewards/accuracy_reward": 0.0066964291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2684151902794838,
      "step": 200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2045.0759582519531,
      "epoch": 0.0600403255918154,
      "grad_norm": 0.0667165145277977,
      "kl": 0.0006380081176757812,
      "learning_rate": 3e-07,
      "loss": 0.0034,
      "reward": 0.2929687649011612,
      "reward_std": 0.017887577414512634,
      "rewards/accuracy_reward": 0.0357142873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2572544738650322,
      "step": 201
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1981.1585083007812,
      "epoch": 0.060339033679336865,
      "grad_norm": 0.08809357136487961,
      "kl": 0.0011606216430664062,
      "learning_rate": 3.014925373134328e-07,
      "loss": 0.0175,
      "reward": 0.369419664144516,
      "reward_std": 0.026709757279604673,
      "rewards/accuracy_reward": 0.1026785783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.266741082072258,
      "step": 202
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2015.9219360351562,
      "epoch": 0.060637741766858336,
      "grad_norm": 0.09069853276014328,
      "kl": 0.0008955001831054688,
      "learning_rate": 3.029850746268657e-07,
      "loss": 0.0076,
      "reward": 0.2885044738650322,
      "reward_std": 0.0576875985134393,
      "rewards/accuracy_reward": 0.0133928582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.275111623108387,
      "step": 203
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2000.76123046875,
      "epoch": 0.06093644985437981,
      "grad_norm": 0.09514117985963821,
      "kl": 0.0011749267578125,
      "learning_rate": 3.044776119402985e-07,
      "loss": 0.0227,
      "reward": 0.3738839477300644,
      "reward_std": 0.0642862762324512,
      "rewards/accuracy_reward": 0.1004464328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2734375149011612,
      "step": 204
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1987.99560546875,
      "epoch": 0.06123515794190128,
      "grad_norm": 0.13551844656467438,
      "kl": 0.001445770263671875,
      "learning_rate": 3.059701492537313e-07,
      "loss": 0.0227,
      "reward": 0.365513414144516,
      "reward_std": 0.09118174319155514,
      "rewards/accuracy_reward": 0.07366071874275804,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2918526902794838,
      "step": 205
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2022.5960693359375,
      "epoch": 0.06153386602942275,
      "grad_norm": 0.07157505303621292,
      "kl": 0.0010194778442382812,
      "learning_rate": 3.074626865671642e-07,
      "loss": 0.0168,
      "reward": 0.2689732238650322,
      "reward_std": 0.03119064774364233,
      "rewards/accuracy_reward": 0.004464285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2645089402794838,
      "step": 206
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1997.1228637695312,
      "epoch": 0.06183257411694422,
      "grad_norm": 0.13191767036914825,
      "kl": 0.00151824951171875,
      "learning_rate": 3.08955223880597e-07,
      "loss": 0.0221,
      "reward": 0.4386160969734192,
      "reward_std": 0.11744646169245243,
      "rewards/accuracy_reward": 0.1540178619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2845982238650322,
      "step": 207
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2015.1875915527344,
      "epoch": 0.06213128220446568,
      "grad_norm": 0.10034525394439697,
      "kl": 0.001201629638671875,
      "learning_rate": 3.1044776119402985e-07,
      "loss": 0.0177,
      "reward": 0.2745535895228386,
      "reward_std": 0.0458978358656168,
      "rewards/accuracy_reward": 0.004464285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2700893059372902,
      "step": 208
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1959.9041137695312,
      "epoch": 0.062429990291987154,
      "grad_norm": 0.09279676526784897,
      "kl": 0.001972198486328125,
      "learning_rate": 3.119402985074627e-07,
      "loss": 0.0086,
      "reward": 0.3130580484867096,
      "reward_std": 0.029862043214961886,
      "rewards/accuracy_reward": 0.03794643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.275111623108387,
      "step": 209
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2003.9018249511719,
      "epoch": 0.06272869837950862,
      "grad_norm": 0.11280187964439392,
      "kl": 0.0014495849609375,
      "learning_rate": 3.134328358208955e-07,
      "loss": 0.0172,
      "reward": 0.3231026902794838,
      "reward_std": 0.0484922849573195,
      "rewards/accuracy_reward": 0.0424107164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2806919813156128,
      "step": 210
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1956.9063415527344,
      "epoch": 0.06302740646703009,
      "grad_norm": 0.10100255906581879,
      "kl": 0.002162933349609375,
      "learning_rate": 3.1492537313432833e-07,
      "loss": 0.005,
      "reward": 0.3973214328289032,
      "reward_std": 0.06160887097939849,
      "rewards/accuracy_reward": 0.11160715110599995,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.285714291036129,
      "step": 211
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1979.3326721191406,
      "epoch": 0.06332611455455156,
      "grad_norm": 0.12217862159013748,
      "kl": 0.00196075439453125,
      "learning_rate": 3.1641791044776116e-07,
      "loss": 0.0191,
      "reward": 0.368861623108387,
      "reward_std": 0.07665435736998916,
      "rewards/accuracy_reward": 0.0892857164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2795758992433548,
      "step": 212
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1962.6607666015625,
      "epoch": 0.06362482264207303,
      "grad_norm": 0.10036396980285645,
      "kl": 0.0022592544555664062,
      "learning_rate": 3.1791044776119405e-07,
      "loss": 0.0117,
      "reward": 0.3549107313156128,
      "reward_std": 0.07280908105894923,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2834821566939354,
      "step": 213
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1977.3594665527344,
      "epoch": 0.0639235307295945,
      "grad_norm": 0.1393975466489792,
      "kl": 0.00225067138671875,
      "learning_rate": 3.194029850746269e-07,
      "loss": 0.0257,
      "reward": 0.3978794813156128,
      "reward_std": 0.07919775880873203,
      "rewards/accuracy_reward": 0.1160714328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2818080484867096,
      "step": 214
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2023.7500915527344,
      "epoch": 0.06422223881711597,
      "grad_norm": 0.14747878909111023,
      "kl": 0.001598358154296875,
      "learning_rate": 3.2089552238805965e-07,
      "loss": 0.02,
      "reward": 0.3119419738650322,
      "reward_std": 0.0637703463435173,
      "rewards/accuracy_reward": 0.04017857206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2717634066939354,
      "step": 215
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2020.6675109863281,
      "epoch": 0.06452094690463744,
      "grad_norm": 0.13431069254875183,
      "kl": 0.0016956329345703125,
      "learning_rate": 3.2238805970149253e-07,
      "loss": 0.0227,
      "reward": 0.2935268059372902,
      "reward_std": 0.08633354329504073,
      "rewards/accuracy_reward": 0.020089285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2734375149011612,
      "step": 216
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1989.3192749023438,
      "epoch": 0.06481965499215891,
      "grad_norm": 0.10301192104816437,
      "kl": 0.0023717880249023438,
      "learning_rate": 3.2388059701492536e-07,
      "loss": 0.0061,
      "reward": 0.3889509066939354,
      "reward_std": 0.03769204462878406,
      "rewards/accuracy_reward": 0.1071428619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2818080484867096,
      "step": 217
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2009.4532470703125,
      "epoch": 0.06511836307968039,
      "grad_norm": 0.11261283606290817,
      "kl": 0.0020198822021484375,
      "learning_rate": 3.253731343283582e-07,
      "loss": 0.0203,
      "reward": 0.286272332072258,
      "reward_std": 0.059363205917179585,
      "rewards/accuracy_reward": 0.013392857508733869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2728794738650322,
      "step": 218
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2034.0290832519531,
      "epoch": 0.06541707116720186,
      "grad_norm": 0.1040838211774826,
      "kl": 0.0014801025390625,
      "learning_rate": 3.26865671641791e-07,
      "loss": 0.011,
      "reward": 0.3191964477300644,
      "reward_std": 0.06018436141312122,
      "rewards/accuracy_reward": 0.04687500186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2723214402794838,
      "step": 219
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2023.2723999023438,
      "epoch": 0.06571577925472333,
      "grad_norm": 0.10817437618970871,
      "kl": 0.0018329620361328125,
      "learning_rate": 3.2835820895522385e-07,
      "loss": 0.0171,
      "reward": 0.3465401977300644,
      "reward_std": 0.04764120141044259,
      "rewards/accuracy_reward": 0.07589286053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.270647332072258,
      "step": 220
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2003.1853332519531,
      "epoch": 0.06601448734224478,
      "grad_norm": 0.1078382208943367,
      "kl": 0.0023593902587890625,
      "learning_rate": 3.2985074626865673e-07,
      "loss": 0.0206,
      "reward": 0.3125000223517418,
      "reward_std": 0.03786096489056945,
      "rewards/accuracy_reward": 0.03794643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2745535895228386,
      "step": 221
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1965.6384582519531,
      "epoch": 0.06631319542976626,
      "grad_norm": 0.10188373178243637,
      "kl": 0.003177642822265625,
      "learning_rate": 3.313432835820895e-07,
      "loss": 0.0258,
      "reward": 0.3800223395228386,
      "reward_std": 0.06845005787909031,
      "rewards/accuracy_reward": 0.09598214784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2840401977300644,
      "step": 222
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2022.44873046875,
      "epoch": 0.06661190351728773,
      "grad_norm": 0.12264679372310638,
      "kl": 0.0021533966064453125,
      "learning_rate": 3.328358208955224e-07,
      "loss": 0.0181,
      "reward": 0.3683035895228386,
      "reward_std": 0.0722658357117325,
      "rewards/accuracy_reward": 0.09375000605359674,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2745535895228386,
      "step": 223
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2020.4911499023438,
      "epoch": 0.0669106116048092,
      "grad_norm": 0.12676876783370972,
      "kl": 0.0020999908447265625,
      "learning_rate": 3.343283582089552e-07,
      "loss": 0.0238,
      "reward": 0.3822544813156128,
      "reward_std": 0.06705653108656406,
      "rewards/accuracy_reward": 0.11160714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.270647332072258,
      "step": 224
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2028.0960693359375,
      "epoch": 0.06720931969233067,
      "grad_norm": 0.0972866490483284,
      "kl": 0.002017974853515625,
      "learning_rate": 3.3582089552238805e-07,
      "loss": 0.0126,
      "reward": 0.3130580484867096,
      "reward_std": 0.0542047219350934,
      "rewards/accuracy_reward": 0.04017857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2728794738650322,
      "step": 225
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2003.4643859863281,
      "epoch": 0.06750802777985214,
      "grad_norm": 0.11222253739833832,
      "kl": 0.0025501251220703125,
      "learning_rate": 3.373134328358209e-07,
      "loss": 0.0246,
      "reward": 0.3242187723517418,
      "reward_std": 0.07053999602794647,
      "rewards/accuracy_reward": 0.04464285937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.279575914144516,
      "step": 226
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2014.6853637695312,
      "epoch": 0.06780673586737361,
      "grad_norm": 0.11316769570112228,
      "kl": 0.0024852752685546875,
      "learning_rate": 3.388059701492537e-07,
      "loss": 0.0155,
      "reward": 0.3030134066939354,
      "reward_std": 0.09731542132794857,
      "rewards/accuracy_reward": 0.020089286845177412,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2829241156578064,
      "step": 227
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1990.4375610351562,
      "epoch": 0.06810544395489508,
      "grad_norm": 0.12635187804698944,
      "kl": 0.00305938720703125,
      "learning_rate": 3.402985074626866e-07,
      "loss": 0.026,
      "reward": 0.3331473395228386,
      "reward_std": 0.130776546895504,
      "rewards/accuracy_reward": 0.04910714388824999,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2840401902794838,
      "step": 228
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2024.805908203125,
      "epoch": 0.06840415204241655,
      "grad_norm": 0.10100309550762177,
      "kl": 0.002277374267578125,
      "learning_rate": 3.4179104477611937e-07,
      "loss": 0.021,
      "reward": 0.2929687649011612,
      "reward_std": 0.07502161804586649,
      "rewards/accuracy_reward": 0.0200892873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2728794813156128,
      "step": 229
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1949.1451721191406,
      "epoch": 0.06870286012993802,
      "grad_norm": 0.11945144832134247,
      "kl": 0.004306793212890625,
      "learning_rate": 3.432835820895522e-07,
      "loss": 0.0318,
      "reward": 0.3420759066939354,
      "reward_std": 0.0771923428401351,
      "rewards/accuracy_reward": 0.04464285937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.297433041036129,
      "step": 230
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1954.1429443359375,
      "epoch": 0.0690015682174595,
      "grad_norm": 0.1123351976275444,
      "kl": 0.004077911376953125,
      "learning_rate": 3.447761194029851e-07,
      "loss": 0.0268,
      "reward": 0.3789062574505806,
      "reward_std": 0.10282119456678629,
      "rewards/accuracy_reward": 0.08705357322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2918526902794838,
      "step": 231
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1985.8438110351562,
      "epoch": 0.06930027630498095,
      "grad_norm": 0.11126238852739334,
      "kl": 0.003376007080078125,
      "learning_rate": 3.462686567164179e-07,
      "loss": 0.0137,
      "reward": 0.3593750149011612,
      "reward_std": 0.060739551205188036,
      "rewards/accuracy_reward": 0.0781250037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2812500074505806,
      "step": 232
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1960.2098999023438,
      "epoch": 0.06959898439250242,
      "grad_norm": 0.11529677361249924,
      "kl": 0.004241943359375,
      "learning_rate": 3.477611940298507e-07,
      "loss": 0.036,
      "reward": 0.3599330484867096,
      "reward_std": 0.04292834363877773,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2885044813156128,
      "step": 233
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1950.6429443359375,
      "epoch": 0.06989769248002389,
      "grad_norm": 0.14292439818382263,
      "kl": 0.0045623779296875,
      "learning_rate": 3.4925373134328357e-07,
      "loss": 0.0525,
      "reward": 0.3883928805589676,
      "reward_std": 0.07674214523285627,
      "rewards/accuracy_reward": 0.0937500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.294642873108387,
      "step": 234
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2024.9487609863281,
      "epoch": 0.07019640056754536,
      "grad_norm": 0.1095770075917244,
      "kl": 0.0025806427001953125,
      "learning_rate": 3.507462686567164e-07,
      "loss": 0.0118,
      "reward": 0.318080373108387,
      "reward_std": 0.06389593239873648,
      "rewards/accuracy_reward": 0.04017857206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2779017984867096,
      "step": 235
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1993.9398498535156,
      "epoch": 0.07049510865506683,
      "grad_norm": 0.12697340548038483,
      "kl": 0.003467559814453125,
      "learning_rate": 3.5223880597014923e-07,
      "loss": 0.0335,
      "reward": 0.4174107313156128,
      "reward_std": 0.08734581060707569,
      "rewards/accuracy_reward": 0.129464291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2879464402794838,
      "step": 236
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1992.6808776855469,
      "epoch": 0.0707938167425883,
      "grad_norm": 0.11588187515735626,
      "kl": 0.003673553466796875,
      "learning_rate": 3.5373134328358206e-07,
      "loss": 0.0204,
      "reward": 0.349330373108387,
      "reward_std": 0.08447225950658321,
      "rewards/accuracy_reward": 0.05803571757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2912946566939354,
      "step": 237
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1927.74560546875,
      "epoch": 0.07109252483010978,
      "grad_norm": 0.15074247121810913,
      "kl": 0.005779266357421875,
      "learning_rate": 3.552238805970149e-07,
      "loss": 0.0528,
      "reward": 0.3822544738650322,
      "reward_std": 0.11215895228087902,
      "rewards/accuracy_reward": 0.07589286169968545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3063616156578064,
      "step": 238
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1994.2389221191406,
      "epoch": 0.07139123291763125,
      "grad_norm": 0.1309901624917984,
      "kl": 0.003681182861328125,
      "learning_rate": 3.5671641791044777e-07,
      "loss": 0.0373,
      "reward": 0.3560268133878708,
      "reward_std": 0.07334023341536522,
      "rewards/accuracy_reward": 0.07812500349245965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2779017984867096,
      "step": 239
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1966.6362609863281,
      "epoch": 0.07168994100515272,
      "grad_norm": 0.1274717003107071,
      "kl": 0.005023956298828125,
      "learning_rate": 3.5820895522388055e-07,
      "loss": 0.02,
      "reward": 0.357142873108387,
      "reward_std": 0.09366848599165678,
      "rewards/accuracy_reward": 0.053571432596072555,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3035714477300644,
      "step": 240
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1948.4732971191406,
      "epoch": 0.07198864909267419,
      "grad_norm": 0.11765734851360321,
      "kl": 0.00518798828125,
      "learning_rate": 3.5970149253731343e-07,
      "loss": 0.0277,
      "reward": 0.4274553805589676,
      "reward_std": 0.124588243663311,
      "rewards/accuracy_reward": 0.12500000558793545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3024553656578064,
      "step": 241
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1994.3416137695312,
      "epoch": 0.07228735718019566,
      "grad_norm": 0.13728979229927063,
      "kl": 0.004390716552734375,
      "learning_rate": 3.6119402985074626e-07,
      "loss": 0.0298,
      "reward": 0.3867187723517418,
      "reward_std": 0.0792832737788558,
      "rewards/accuracy_reward": 0.0959821455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2907366305589676,
      "step": 242
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2010.3750915527344,
      "epoch": 0.07258606526771712,
      "grad_norm": 0.12306191027164459,
      "kl": 0.003597259521484375,
      "learning_rate": 3.626865671641791e-07,
      "loss": 0.0291,
      "reward": 0.3353794887661934,
      "reward_std": 0.08254489116370678,
      "rewards/accuracy_reward": 0.05580357392318547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.279575914144516,
      "step": 243
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1989.7612609863281,
      "epoch": 0.07288477335523859,
      "grad_norm": 0.1261904090642929,
      "kl": 0.0045013427734375,
      "learning_rate": 3.641791044776119e-07,
      "loss": 0.0369,
      "reward": 0.4497768133878708,
      "reward_std": 0.09010925143957138,
      "rewards/accuracy_reward": 0.1629464365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2868303656578064,
      "step": 244
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1933.2344665527344,
      "epoch": 0.07318348144276006,
      "grad_norm": 0.10987003892660141,
      "kl": 0.0060577392578125,
      "learning_rate": 3.6567164179104475e-07,
      "loss": 0.025,
      "reward": 0.3018973395228386,
      "reward_std": 0.05150948045775294,
      "rewards/accuracy_reward": 0.0022321429569274187,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2996651977300644,
      "step": 245
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2002.0357971191406,
      "epoch": 0.07348218953028153,
      "grad_norm": 0.1187586858868599,
      "kl": 0.0045013427734375,
      "learning_rate": 3.6716417910447763e-07,
      "loss": 0.0237,
      "reward": 0.4743303880095482,
      "reward_std": 0.11379939876496792,
      "rewards/accuracy_reward": 0.18750000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2868303656578064,
      "step": 246
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2016.8058471679688,
      "epoch": 0.073780897617803,
      "grad_norm": 0.1318998634815216,
      "kl": 0.00414276123046875,
      "learning_rate": 3.686567164179104e-07,
      "loss": 0.0252,
      "reward": 0.3755580484867096,
      "reward_std": 0.11344264447689056,
      "rewards/accuracy_reward": 0.08705357415601611,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2885044813156128,
      "step": 247
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1981.5514221191406,
      "epoch": 0.07407960570532447,
      "grad_norm": 0.12940748035907745,
      "kl": 0.005161285400390625,
      "learning_rate": 3.7014925373134323e-07,
      "loss": 0.0354,
      "reward": 0.3069196566939354,
      "reward_std": 0.08730226382613182,
      "rewards/accuracy_reward": 0.01116071455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2957589402794838,
      "step": 248
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1978.3795776367188,
      "epoch": 0.07437831379284594,
      "grad_norm": 0.1312132328748703,
      "kl": 0.005615234375,
      "learning_rate": 3.716417910447761e-07,
      "loss": 0.0268,
      "reward": 0.3816964477300644,
      "reward_std": 0.08822421077638865,
      "rewards/accuracy_reward": 0.08705357555299997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.294642873108387,
      "step": 249
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2002.4532165527344,
      "epoch": 0.07467702188036741,
      "grad_norm": 0.13362780213356018,
      "kl": 0.00518035888671875,
      "learning_rate": 3.7313432835820895e-07,
      "loss": 0.032,
      "reward": 0.2996651902794838,
      "reward_std": 0.07552485540509224,
      "rewards/accuracy_reward": 0.0022321429569274187,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2974330484867096,
      "step": 250
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1987.8326721191406,
      "epoch": 0.07497572996788888,
      "grad_norm": 0.12010441720485687,
      "kl": 0.00563812255859375,
      "learning_rate": 3.746268656716418e-07,
      "loss": 0.0278,
      "reward": 0.3599330484867096,
      "reward_std": 0.12494660541415215,
      "rewards/accuracy_reward": 0.06026786006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2996651902794838,
      "step": 251
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1997.212158203125,
      "epoch": 0.07527443805541036,
      "grad_norm": 0.1480213850736618,
      "kl": 0.0054168701171875,
      "learning_rate": 3.761194029850746e-07,
      "loss": 0.0296,
      "reward": 0.3325892984867096,
      "reward_std": 0.07889928296208382,
      "rewards/accuracy_reward": 0.03794643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.294642873108387,
      "step": 252
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1922.7322082519531,
      "epoch": 0.07557314614293181,
      "grad_norm": 0.14510825276374817,
      "kl": 0.00768280029296875,
      "learning_rate": 3.7761194029850743e-07,
      "loss": 0.0486,
      "reward": 0.5000000149011612,
      "reward_std": 0.1937486194074154,
      "rewards/accuracy_reward": 0.1696428656578064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.330357164144516,
      "step": 253
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1985.1451721191406,
      "epoch": 0.07587185423045328,
      "grad_norm": 0.13130295276641846,
      "kl": 0.00585174560546875,
      "learning_rate": 3.7910447761194026e-07,
      "loss": 0.0307,
      "reward": 0.3342634066939354,
      "reward_std": 0.12067104503512383,
      "rewards/accuracy_reward": 0.042410717345774174,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2918526902794838,
      "step": 254
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1950.2768859863281,
      "epoch": 0.07617056231797475,
      "grad_norm": 0.14331762492656708,
      "kl": 0.006862640380859375,
      "learning_rate": 3.805970149253731e-07,
      "loss": 0.0403,
      "reward": 0.407924123108387,
      "reward_std": 0.1356019787490368,
      "rewards/accuracy_reward": 0.09375000488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.314174123108387,
      "step": 255
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1987.290283203125,
      "epoch": 0.07646927040549623,
      "grad_norm": 0.1288195699453354,
      "kl": 0.006378173828125,
      "learning_rate": 3.82089552238806e-07,
      "loss": 0.0266,
      "reward": 0.3526785895228386,
      "reward_std": 0.11634229216724634,
      "rewards/accuracy_reward": 0.055803575087338686,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2968750074505806,
      "step": 256
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1938.4063415527344,
      "epoch": 0.0767679784930177,
      "grad_norm": 0.13025054335594177,
      "kl": 0.00708770751953125,
      "learning_rate": 3.835820895522388e-07,
      "loss": 0.0269,
      "reward": 0.3811384066939354,
      "reward_std": 0.1112741008400917,
      "rewards/accuracy_reward": 0.07589285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.305245541036129,
      "step": 257
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1939.4778137207031,
      "epoch": 0.07706668658053917,
      "grad_norm": 0.1458735316991806,
      "kl": 0.0081024169921875,
      "learning_rate": 3.850746268656716e-07,
      "loss": 0.0466,
      "reward": 0.4693080559372902,
      "reward_std": 0.10791886784136295,
      "rewards/accuracy_reward": 0.1540178640279919,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3152901828289032,
      "step": 258
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1945.0000915527344,
      "epoch": 0.07736539466806064,
      "grad_norm": 0.1510995477437973,
      "kl": 0.0080413818359375,
      "learning_rate": 3.8656716417910446e-07,
      "loss": 0.0499,
      "reward": 0.388392873108387,
      "reward_std": 0.12511307187378407,
      "rewards/accuracy_reward": 0.08258929033763707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.305803582072258,
      "step": 259
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2011.9040832519531,
      "epoch": 0.07766410275558211,
      "grad_norm": 0.13501103222370148,
      "kl": 0.006011962890625,
      "learning_rate": 3.880597014925373e-07,
      "loss": 0.03,
      "reward": 0.3091518059372902,
      "reward_std": 0.12465482205152512,
      "rewards/accuracy_reward": 0.01562500069849193,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2935267984867096,
      "step": 260
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1950.7210693359375,
      "epoch": 0.07796281084310358,
      "grad_norm": 0.13436302542686462,
      "kl": 0.00799560546875,
      "learning_rate": 3.895522388059702e-07,
      "loss": 0.0391,
      "reward": 0.3325892984867096,
      "reward_std": 0.09527294337749481,
      "rewards/accuracy_reward": 0.01785714295692742,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3147321566939354,
      "step": 261
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1989.2880249023438,
      "epoch": 0.07826151893062505,
      "grad_norm": 0.1438877284526825,
      "kl": 0.00701141357421875,
      "learning_rate": 3.9104477611940295e-07,
      "loss": 0.0371,
      "reward": 0.366629496216774,
      "reward_std": 0.11085704527795315,
      "rewards/accuracy_reward": 0.0736607147846371,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2929687649011612,
      "step": 262
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1978.3773498535156,
      "epoch": 0.07856022701814652,
      "grad_norm": 0.11992578208446503,
      "kl": 0.00756072998046875,
      "learning_rate": 3.925373134328358e-07,
      "loss": 0.027,
      "reward": 0.3794642984867096,
      "reward_std": 0.0741656506434083,
      "rewards/accuracy_reward": 0.0870535746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2924107313156128,
      "step": 263
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1958.2032165527344,
      "epoch": 0.07885893510566798,
      "grad_norm": 0.1292649656534195,
      "kl": 0.00836944580078125,
      "learning_rate": 3.9402985074626866e-07,
      "loss": 0.0389,
      "reward": 0.3777901902794838,
      "reward_std": 0.1510040182620287,
      "rewards/accuracy_reward": 0.07366071571595967,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3041294813156128,
      "step": 264
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1912.3460693359375,
      "epoch": 0.07915764319318945,
      "grad_norm": 0.14024987816810608,
      "kl": 0.009674072265625,
      "learning_rate": 3.9552238805970144e-07,
      "loss": 0.0295,
      "reward": 0.3588169813156128,
      "reward_std": 0.09611312951892614,
      "rewards/accuracy_reward": 0.042410716181620955,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3164062649011612,
      "step": 265
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1977.9710998535156,
      "epoch": 0.07945635128071092,
      "grad_norm": 0.1375216543674469,
      "kl": 0.00830078125,
      "learning_rate": 3.970149253731343e-07,
      "loss": 0.0299,
      "reward": 0.326450914144516,
      "reward_std": 0.11287194304168224,
      "rewards/accuracy_reward": 0.020089286845177412,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3063616156578064,
      "step": 266
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1963.9532165527344,
      "epoch": 0.07975505936823239,
      "grad_norm": 0.14886994659900665,
      "kl": 0.0087890625,
      "learning_rate": 3.9850746268656715e-07,
      "loss": 0.0378,
      "reward": 0.3604910895228386,
      "reward_std": 0.12182177230715752,
      "rewards/accuracy_reward": 0.049107145285233855,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3113839402794838,
      "step": 267
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1931.149658203125,
      "epoch": 0.08005376745575386,
      "grad_norm": 0.14939945936203003,
      "kl": 0.009857177734375,
      "learning_rate": 4e-07,
      "loss": 0.0574,
      "reward": 0.389508955180645,
      "reward_std": 0.12413572147488594,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.318080373108387,
      "step": 268
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1889.2657165527344,
      "epoch": 0.08035247554327533,
      "grad_norm": 0.15972210466861725,
      "kl": 0.0113372802734375,
      "learning_rate": 4.014925373134328e-07,
      "loss": 0.0609,
      "reward": 0.4397321566939354,
      "reward_std": 0.12599130906164646,
      "rewards/accuracy_reward": 0.11160715157166123,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3281250149011612,
      "step": 269
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1956.0313110351562,
      "epoch": 0.0806511836307968,
      "grad_norm": 0.14033837616443634,
      "kl": 0.00965118408203125,
      "learning_rate": 4.0298507462686564e-07,
      "loss": 0.0267,
      "reward": 0.4776785969734192,
      "reward_std": 0.15621951408684254,
      "rewards/accuracy_reward": 0.17410714644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3035714402794838,
      "step": 270
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1930.5447082519531,
      "epoch": 0.08094989171831828,
      "grad_norm": 0.16332978010177612,
      "kl": 0.010589599609375,
      "learning_rate": 4.044776119402985e-07,
      "loss": 0.0448,
      "reward": 0.3995535895228386,
      "reward_std": 0.15651614032685757,
      "rewards/accuracy_reward": 0.07366071688011289,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3258928656578064,
      "step": 271
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1905.2076416015625,
      "epoch": 0.08124859980583975,
      "grad_norm": 0.1436133086681366,
      "kl": 0.0112457275390625,
      "learning_rate": 4.059701492537313e-07,
      "loss": 0.0511,
      "reward": 0.3710937649011612,
      "reward_std": 0.10712305642664433,
      "rewards/accuracy_reward": 0.05580357578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3152901902794838,
      "step": 272
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1935.8907165527344,
      "epoch": 0.08154730789336122,
      "grad_norm": 0.15039677917957306,
      "kl": 0.01129150390625,
      "learning_rate": 4.0746268656716413e-07,
      "loss": 0.0554,
      "reward": 0.3549107313156128,
      "reward_std": 0.11849102191627026,
      "rewards/accuracy_reward": 0.0334821455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3214285895228386,
      "step": 273
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1928.9375610351562,
      "epoch": 0.08184601598088269,
      "grad_norm": 0.17037099599838257,
      "kl": 0.0119171142578125,
      "learning_rate": 4.08955223880597e-07,
      "loss": 0.061,
      "reward": 0.541294664144516,
      "reward_std": 0.1505908127874136,
      "rewards/accuracy_reward": 0.20758929662406445,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.333705373108387,
      "step": 274
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1842.02685546875,
      "epoch": 0.08214472406840415,
      "grad_norm": 0.1686246395111084,
      "kl": 0.0147247314453125,
      "learning_rate": 4.1044776119402984e-07,
      "loss": 0.0457,
      "reward": 0.4207589477300644,
      "reward_std": 0.1368700247257948,
      "rewards/accuracy_reward": 0.08482143119908869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3359375074505806,
      "step": 275
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1908.7768859863281,
      "epoch": 0.08244343215592562,
      "grad_norm": 0.15488971769809723,
      "kl": 0.013427734375,
      "learning_rate": 4.1194029850746267e-07,
      "loss": 0.0419,
      "reward": 0.4185267984867096,
      "reward_std": 0.1520002130419016,
      "rewards/accuracy_reward": 0.0736607164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.344866082072258,
      "step": 276
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1918.1652526855469,
      "epoch": 0.08274214024344709,
      "grad_norm": 0.1761590540409088,
      "kl": 0.012664794921875,
      "learning_rate": 4.134328358208955e-07,
      "loss": 0.0539,
      "reward": 0.407366082072258,
      "reward_std": 0.14757522195577621,
      "rewards/accuracy_reward": 0.0758928582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3314732313156128,
      "step": 277
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1892.0916137695312,
      "epoch": 0.08304084833096856,
      "grad_norm": 0.17069895565509796,
      "kl": 0.0139617919921875,
      "learning_rate": 4.1492537313432833e-07,
      "loss": 0.0582,
      "reward": 0.3627232313156128,
      "reward_std": 0.13992940448224545,
      "rewards/accuracy_reward": 0.029017859371379018,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3337053656578064,
      "step": 278
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1831.7121276855469,
      "epoch": 0.08333955641849003,
      "grad_norm": 0.17750589549541473,
      "kl": 0.0163726806640625,
      "learning_rate": 4.164179104477612e-07,
      "loss": 0.0518,
      "reward": 0.4045759066939354,
      "reward_std": 0.11226295493543148,
      "rewards/accuracy_reward": 0.060267860535532236,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3443080484867096,
      "step": 279
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1964.7657165527344,
      "epoch": 0.0836382645060115,
      "grad_norm": 0.15312790870666504,
      "kl": 0.0133056640625,
      "learning_rate": 4.17910447761194e-07,
      "loss": 0.0483,
      "reward": 0.368303582072258,
      "reward_std": 0.11700172908604145,
      "rewards/accuracy_reward": 0.0513392873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3169642984867096,
      "step": 280
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1957.99560546875,
      "epoch": 0.08393697259353297,
      "grad_norm": 0.16639547049999237,
      "kl": 0.0134735107421875,
      "learning_rate": 4.1940298507462687e-07,
      "loss": 0.045,
      "reward": 0.3649553805589676,
      "reward_std": 0.11487031169235706,
      "rewards/accuracy_reward": 0.04241071501746774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3225446566939354,
      "step": 281
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1979.2255249023438,
      "epoch": 0.08423568068105444,
      "grad_norm": 0.13930590450763702,
      "kl": 0.0135345458984375,
      "learning_rate": 4.208955223880597e-07,
      "loss": 0.039,
      "reward": 0.4335937723517418,
      "reward_std": 0.1112994160503149,
      "rewards/accuracy_reward": 0.11830357951112092,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3152901902794838,
      "step": 282
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1907.8884582519531,
      "epoch": 0.08453438876857591,
      "grad_norm": 0.15998511016368866,
      "kl": 0.0160064697265625,
      "learning_rate": 4.223880597014925e-07,
      "loss": 0.0495,
      "reward": 0.380580373108387,
      "reward_std": 0.11969679035246372,
      "rewards/accuracy_reward": 0.05580357555299997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3247768059372902,
      "step": 283
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1902.8326721191406,
      "epoch": 0.08483309685609738,
      "grad_norm": 0.16308756172657013,
      "kl": 0.017120361328125,
      "learning_rate": 4.2388059701492536e-07,
      "loss": 0.0597,
      "reward": 0.4787946715950966,
      "reward_std": 0.17327484115958214,
      "rewards/accuracy_reward": 0.1339285783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3448660895228386,
      "step": 284
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1934.0134887695312,
      "epoch": 0.08513180494361886,
      "grad_norm": 0.15444514155387878,
      "kl": 0.015869140625,
      "learning_rate": 4.253731343283582e-07,
      "loss": 0.0489,
      "reward": 0.3677455559372902,
      "reward_std": 0.11584950052201748,
      "rewards/accuracy_reward": 0.049107145285233855,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3186383992433548,
      "step": 285
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1907.7813110351562,
      "epoch": 0.08543051303114031,
      "grad_norm": 0.165074422955513,
      "kl": 0.017547607421875,
      "learning_rate": 4.2686567164179107e-07,
      "loss": 0.0528,
      "reward": 0.4810268133878708,
      "reward_std": 0.12199302576482296,
      "rewards/accuracy_reward": 0.14062500977888703,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3404018059372902,
      "step": 286
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1947.7277526855469,
      "epoch": 0.08572922111866178,
      "grad_norm": 0.16399633884429932,
      "kl": 0.01678466796875,
      "learning_rate": 4.2835820895522385e-07,
      "loss": 0.0547,
      "reward": 0.4386160895228386,
      "reward_std": 0.11736038699746132,
      "rewards/accuracy_reward": 0.11160714668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3270089477300644,
      "step": 287
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1851.12060546875,
      "epoch": 0.08602792920618325,
      "grad_norm": 0.16809551417827606,
      "kl": 0.020721435546875,
      "learning_rate": 4.298507462686567e-07,
      "loss": 0.0541,
      "reward": 0.368303582072258,
      "reward_std": 0.1017408948391676,
      "rewards/accuracy_reward": 0.006696428870782256,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3616071566939354,
      "step": 288
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1920.4241943359375,
      "epoch": 0.08632663729370472,
      "grad_norm": 0.152552992105484,
      "kl": 0.018463134765625,
      "learning_rate": 4.3134328358208956e-07,
      "loss": 0.0512,
      "reward": 0.443080373108387,
      "reward_std": 0.12782950140535831,
      "rewards/accuracy_reward": 0.11383929569274187,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.329241082072258,
      "step": 289
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1861.1161499023438,
      "epoch": 0.0866253453812262,
      "grad_norm": 0.1719377040863037,
      "kl": 0.02117919921875,
      "learning_rate": 4.3283582089552234e-07,
      "loss": 0.0622,
      "reward": 0.4765625223517418,
      "reward_std": 0.16266490519046783,
      "rewards/accuracy_reward": 0.12723214738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.349330373108387,
      "step": 290
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1811.7366943359375,
      "epoch": 0.08692405346874767,
      "grad_norm": 0.1808566302061081,
      "kl": 0.022918701171875,
      "learning_rate": 4.343283582089552e-07,
      "loss": 0.0661,
      "reward": 0.494419664144516,
      "reward_std": 0.14984366297721863,
      "rewards/accuracy_reward": 0.1428571492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625149011612,
      "step": 291
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1776.18310546875,
      "epoch": 0.08722276155626914,
      "grad_norm": 0.17152146995067596,
      "kl": 0.024078369140625,
      "learning_rate": 4.3582089552238805e-07,
      "loss": 0.0508,
      "reward": 0.4017857313156128,
      "reward_std": 0.13406887464225292,
      "rewards/accuracy_reward": 0.033482144586741924,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.368303582072258,
      "step": 292
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1864.5916137695312,
      "epoch": 0.08752146964379061,
      "grad_norm": 0.18318171799182892,
      "kl": 0.023651123046875,
      "learning_rate": 4.373134328358209e-07,
      "loss": 0.0787,
      "reward": 0.5340402126312256,
      "reward_std": 0.13480005599558353,
      "rewards/accuracy_reward": 0.1718750111758709,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3621651902794838,
      "step": 293
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1870.7366943359375,
      "epoch": 0.08782017773131208,
      "grad_norm": 0.17291474342346191,
      "kl": 0.02362060546875,
      "learning_rate": 4.388059701492537e-07,
      "loss": 0.0645,
      "reward": 0.3699776977300644,
      "reward_std": 0.14799103513360023,
      "rewards/accuracy_reward": 0.020089287078008056,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.349888414144516,
      "step": 294
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1889.7567749023438,
      "epoch": 0.08811888581883355,
      "grad_norm": 0.16443517804145813,
      "kl": 0.02313232421875,
      "learning_rate": 4.4029850746268654e-07,
      "loss": 0.0547,
      "reward": 0.4090401902794838,
      "reward_std": 0.1376936323940754,
      "rewards/accuracy_reward": 0.055803574388846755,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.353236623108387,
      "step": 295
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1825.0670776367188,
      "epoch": 0.08841759390635502,
      "grad_norm": 0.1642121970653534,
      "kl": 0.025177001953125,
      "learning_rate": 4.417910447761194e-07,
      "loss": 0.0426,
      "reward": 0.3928571566939354,
      "reward_std": 0.14122464135289192,
      "rewards/accuracy_reward": 0.0357142873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3571428656578064,
      "step": 296
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1829.2099304199219,
      "epoch": 0.08871630199387648,
      "grad_norm": 0.17570245265960693,
      "kl": 0.027008056640625,
      "learning_rate": 4.4328358208955225e-07,
      "loss": 0.0623,
      "reward": 0.4849330484867096,
      "reward_std": 0.1573524996638298,
      "rewards/accuracy_reward": 0.11607143469154835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.368861623108387,
      "step": 297
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1836.1630554199219,
      "epoch": 0.08901501008139795,
      "grad_norm": 0.18993113934993744,
      "kl": 0.0281982421875,
      "learning_rate": 4.44776119402985e-07,
      "loss": 0.0595,
      "reward": 0.4090401902794838,
      "reward_std": 0.16793609783053398,
      "rewards/accuracy_reward": 0.044642860535532236,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3643973395228386,
      "step": 298
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1814.8460693359375,
      "epoch": 0.08931371816891942,
      "grad_norm": 0.20305633544921875,
      "kl": 0.0294189453125,
      "learning_rate": 4.462686567164179e-07,
      "loss": 0.061,
      "reward": 0.428571455180645,
      "reward_std": 0.12973556481301785,
      "rewards/accuracy_reward": 0.05133928777649999,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.377232164144516,
      "step": 299
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1752.0068054199219,
      "epoch": 0.08961242625644089,
      "grad_norm": 0.19890770316123962,
      "kl": 0.03204345703125,
      "learning_rate": 4.4776119402985074e-07,
      "loss": 0.0752,
      "reward": 0.564174123108387,
      "reward_std": 0.13523666746914387,
      "rewards/accuracy_reward": 0.19642857206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3677455559372902,
      "step": 300
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1866.7545471191406,
      "epoch": 0.08991113434396236,
      "grad_norm": 0.1725112944841385,
      "kl": 0.029510498046875,
      "learning_rate": 4.492537313432835e-07,
      "loss": 0.0422,
      "reward": 0.400669664144516,
      "reward_std": 0.12570929899811745,
      "rewards/accuracy_reward": 0.04910714365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625149011612,
      "step": 301
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1807.4264221191406,
      "epoch": 0.09020984243148383,
      "grad_norm": 0.18205706775188446,
      "kl": 0.032073974609375,
      "learning_rate": 4.507462686567164e-07,
      "loss": 0.0644,
      "reward": 0.434709832072258,
      "reward_std": 0.15390844270586967,
      "rewards/accuracy_reward": 0.07142857508733869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3632812649011612,
      "step": 302
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1794.4107971191406,
      "epoch": 0.0905085505190053,
      "grad_norm": 0.18576619029045105,
      "kl": 0.03338623046875,
      "learning_rate": 4.522388059701492e-07,
      "loss": 0.0694,
      "reward": 0.4185268133878708,
      "reward_std": 0.14628276601433754,
      "rewards/accuracy_reward": 0.04910714365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.369419664144516,
      "step": 303
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1854.3973999023438,
      "epoch": 0.09080725860652678,
      "grad_norm": 0.19164440035820007,
      "kl": 0.032928466796875,
      "learning_rate": 4.537313432835821e-07,
      "loss": 0.0607,
      "reward": 0.3945312649011612,
      "reward_std": 0.13554842211306095,
      "rewards/accuracy_reward": 0.0223214291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3722098395228386,
      "step": 304
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1732.3996276855469,
      "epoch": 0.09110596669404825,
      "grad_norm": 0.1819790154695511,
      "kl": 0.03582763671875,
      "learning_rate": 4.552238805970149e-07,
      "loss": 0.0548,
      "reward": 0.4017857387661934,
      "reward_std": 0.14014336094260216,
      "rewards/accuracy_reward": 0.024553573224693537,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.377232164144516,
      "step": 305
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1821.6786804199219,
      "epoch": 0.09140467478156972,
      "grad_norm": 0.18072210252285004,
      "kl": 0.0360107421875,
      "learning_rate": 4.567164179104477e-07,
      "loss": 0.0614,
      "reward": 0.4776785969734192,
      "reward_std": 0.17274628207087517,
      "rewards/accuracy_reward": 0.10491071874275804,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3727678805589676,
      "step": 306
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1810.9978332519531,
      "epoch": 0.09170338286909117,
      "grad_norm": 0.2081269770860672,
      "kl": 0.03955078125,
      "learning_rate": 4.582089552238806e-07,
      "loss": 0.0743,
      "reward": 0.5055803805589676,
      "reward_std": 0.17858647927641869,
      "rewards/accuracy_reward": 0.12500000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3805803805589676,
      "step": 307
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1865.5469360351562,
      "epoch": 0.09200209095661264,
      "grad_norm": 0.18621991574764252,
      "kl": 0.0396728515625,
      "learning_rate": 4.5970149253731337e-07,
      "loss": 0.0589,
      "reward": 0.4146205484867096,
      "reward_std": 0.173295671120286,
      "rewards/accuracy_reward": 0.04241071571595967,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3722098395228386,
      "step": 308
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1689.26123046875,
      "epoch": 0.09230079904413412,
      "grad_norm": 0.21320658922195435,
      "kl": 0.04302978515625,
      "learning_rate": 4.6119402985074625e-07,
      "loss": 0.0762,
      "reward": 0.4832589402794838,
      "reward_std": 0.1881660707294941,
      "rewards/accuracy_reward": 0.07589285937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4073660969734192,
      "step": 309
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1722.2500610351562,
      "epoch": 0.09259950713165559,
      "grad_norm": 0.18839113414287567,
      "kl": 0.04547119140625,
      "learning_rate": 4.626865671641791e-07,
      "loss": 0.0672,
      "reward": 0.4888393059372902,
      "reward_std": 0.10311116091907024,
      "rewards/accuracy_reward": 0.11160714668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3772321566939354,
      "step": 310
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1770.6295471191406,
      "epoch": 0.09289821521917706,
      "grad_norm": 0.19653482735157013,
      "kl": 0.0462646484375,
      "learning_rate": 4.641791044776119e-07,
      "loss": 0.0656,
      "reward": 0.3934151977300644,
      "reward_std": 0.11920988373458385,
      "rewards/accuracy_reward": 0.022321430034935474,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3710937649011612,
      "step": 311
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1748.5647888183594,
      "epoch": 0.09319692330669853,
      "grad_norm": 0.17870937287807465,
      "kl": 0.0498046875,
      "learning_rate": 4.6567164179104474e-07,
      "loss": 0.05,
      "reward": 0.5050223395228386,
      "reward_std": 0.1616399846971035,
      "rewards/accuracy_reward": 0.12500000558793545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3800223395228386,
      "step": 312
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1728.7679138183594,
      "epoch": 0.09349563139422,
      "grad_norm": 0.22175025939941406,
      "kl": 0.0509033203125,
      "learning_rate": 4.6716417910447757e-07,
      "loss": 0.0752,
      "reward": 0.454799123108387,
      "reward_std": 0.1308348085731268,
      "rewards/accuracy_reward": 0.06026785937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3945312723517418,
      "step": 313
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1712.8594970703125,
      "epoch": 0.09379433948174147,
      "grad_norm": 0.277186781167984,
      "kl": 0.050048828125,
      "learning_rate": 4.6865671641791045e-07,
      "loss": 0.109,
      "reward": 0.4760044813156128,
      "reward_std": 0.14633923582732677,
      "rewards/accuracy_reward": 0.0736607164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4023437723517418,
      "step": 314
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1684.7322387695312,
      "epoch": 0.09409304756926294,
      "grad_norm": 0.20811457931995392,
      "kl": 0.052490234375,
      "learning_rate": 4.701492537313433e-07,
      "loss": 0.0649,
      "reward": 0.4810268059372902,
      "reward_std": 0.1761021763086319,
      "rewards/accuracy_reward": 0.053571430034935474,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4274553805589676,
      "step": 315
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1752.2456359863281,
      "epoch": 0.09439175565678441,
      "grad_norm": 0.21474464237689972,
      "kl": 0.05609130859375,
      "learning_rate": 4.7164179104477606e-07,
      "loss": 0.0747,
      "reward": 0.5669642984867096,
      "reward_std": 0.18811174109578133,
      "rewards/accuracy_reward": 0.1517857238650322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4151785969734192,
      "step": 316
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1734.7322387695312,
      "epoch": 0.09469046374430588,
      "grad_norm": 0.18936482071876526,
      "kl": 0.0614013671875,
      "learning_rate": 4.7313432835820894e-07,
      "loss": 0.0559,
      "reward": 0.4547991305589676,
      "reward_std": 0.12690494582057,
      "rewards/accuracy_reward": 0.07142857275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3833705484867096,
      "step": 317
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1835.7098693847656,
      "epoch": 0.09498917183182734,
      "grad_norm": 0.2056136578321457,
      "kl": 0.0660400390625,
      "learning_rate": 4.7462686567164177e-07,
      "loss": 0.0638,
      "reward": 0.396205373108387,
      "reward_std": 0.14485053718090057,
      "rewards/accuracy_reward": 0.02232142980210483,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3738839477300644,
      "step": 318
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1714.7009887695312,
      "epoch": 0.09528787991934881,
      "grad_norm": 0.25511667132377625,
      "kl": 0.06439208984375,
      "learning_rate": 4.761194029850746e-07,
      "loss": 0.0798,
      "reward": 0.467075914144516,
      "reward_std": 0.15433309227228165,
      "rewards/accuracy_reward": 0.060267857974395156,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4068080559372902,
      "step": 319
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1714.1228332519531,
      "epoch": 0.09558658800687028,
      "grad_norm": 0.23663757741451263,
      "kl": 0.06732177734375,
      "learning_rate": 4.776119402985074e-07,
      "loss": 0.0842,
      "reward": 0.4179687649011612,
      "reward_std": 0.12431371957063675,
      "rewards/accuracy_reward": 0.011160715017467737,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4068080559372902,
      "step": 320
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1771.9777526855469,
      "epoch": 0.09588529609439175,
      "grad_norm": 0.24057133495807648,
      "kl": 0.073974609375,
      "learning_rate": 4.791044776119403e-07,
      "loss": 0.074,
      "reward": 0.4581473395228386,
      "reward_std": 0.1377399731427431,
      "rewards/accuracy_reward": 0.06696428824216127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3911830484867096,
      "step": 321
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1752.3973999023438,
      "epoch": 0.09618400418191322,
      "grad_norm": 0.2211206555366516,
      "kl": 0.0755615234375,
      "learning_rate": 4.805970149253731e-07,
      "loss": 0.0675,
      "reward": 0.4603794738650322,
      "reward_std": 0.148013174533844,
      "rewards/accuracy_reward": 0.06473214761354029,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3956473395228386,
      "step": 322
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1853.9889221191406,
      "epoch": 0.0964827122694347,
      "grad_norm": 0.2801364064216614,
      "kl": 0.0860595703125,
      "learning_rate": 4.820895522388059e-07,
      "loss": 0.0911,
      "reward": 0.4581473469734192,
      "reward_std": 0.13438985869288445,
      "rewards/accuracy_reward": 0.08482143143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.373325914144516,
      "step": 323
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1773.8929138183594,
      "epoch": 0.09678142035695617,
      "grad_norm": 0.259675532579422,
      "kl": 0.0850830078125,
      "learning_rate": 4.835820895522387e-07,
      "loss": 0.0754,
      "reward": 0.400111623108387,
      "reward_std": 0.11311941407620907,
      "rewards/accuracy_reward": 0.008928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3911830484867096,
      "step": 324
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1688.2277526855469,
      "epoch": 0.09708012844447764,
      "grad_norm": 0.2579168677330017,
      "kl": 0.09033203125,
      "learning_rate": 4.850746268656717e-07,
      "loss": 0.0653,
      "reward": 0.5011161044239998,
      "reward_std": 0.1576140597462654,
      "rewards/accuracy_reward": 0.0781250037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 325
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1763.8639221191406,
      "epoch": 0.09737883653199911,
      "grad_norm": 0.25518760085105896,
      "kl": 0.0960693359375,
      "learning_rate": 4.865671641791044e-07,
      "loss": 0.0593,
      "reward": 0.5189732387661934,
      "reward_std": 0.19704899564385414,
      "rewards/accuracy_reward": 0.11607143096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4029017984867096,
      "step": 326
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1743.3192749023438,
      "epoch": 0.09767754461952058,
      "grad_norm": 0.30585822463035583,
      "kl": 0.101806640625,
      "learning_rate": 4.880597014925372e-07,
      "loss": 0.0767,
      "reward": 0.5251116380095482,
      "reward_std": 0.16676925867795944,
      "rewards/accuracy_reward": 0.10714286379516125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4179687723517418,
      "step": 327
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1614.1050109863281,
      "epoch": 0.09797625270704205,
      "grad_norm": 0.2727620303630829,
      "kl": 0.1014404296875,
      "learning_rate": 4.895522388059702e-07,
      "loss": 0.0724,
      "reward": 0.5585937649011612,
      "reward_std": 0.14034350588917732,
      "rewards/accuracy_reward": 0.1183035783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4402901977300644,
      "step": 328
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1653.1094055175781,
      "epoch": 0.0982749607945635,
      "grad_norm": 0.2347412258386612,
      "kl": 0.10986328125,
      "learning_rate": 4.91044776119403e-07,
      "loss": 0.0454,
      "reward": 0.5641741454601288,
      "reward_std": 0.1534999329596758,
      "rewards/accuracy_reward": 0.13392857648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455633878708,
      "step": 329
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1732.0536193847656,
      "epoch": 0.09857366888208498,
      "grad_norm": 0.2581503391265869,
      "kl": 0.1248779296875,
      "learning_rate": 4.925373134328357e-07,
      "loss": 0.0574,
      "reward": 0.4486607313156128,
      "reward_std": 0.14246662892401218,
      "rewards/accuracy_reward": 0.026785715715959668,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 330
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1610.3683776855469,
      "epoch": 0.09887237696960645,
      "grad_norm": 0.2832465171813965,
      "kl": 0.121337890625,
      "learning_rate": 4.940298507462687e-07,
      "loss": 0.0659,
      "reward": 0.5178571566939354,
      "reward_std": 0.12371615506708622,
      "rewards/accuracy_reward": 0.06696429033763707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.450892873108387,
      "step": 331
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1716.2835693359375,
      "epoch": 0.09917108505712792,
      "grad_norm": 0.25233137607574463,
      "kl": 0.133056640625,
      "learning_rate": 4.955223880597015e-07,
      "loss": 0.0446,
      "reward": 0.5502232313156128,
      "reward_std": 0.12946095131337643,
      "rewards/accuracy_reward": 0.10267857392318547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.447544664144516,
      "step": 332
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1556.8638916015625,
      "epoch": 0.09946979314464939,
      "grad_norm": 0.219815194606781,
      "kl": 0.1318359375,
      "learning_rate": 4.970149253731343e-07,
      "loss": 0.0492,
      "reward": 0.5156250223517418,
      "reward_std": 0.13348649349063635,
      "rewards/accuracy_reward": 0.04017857415601611,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4754464477300644,
      "step": 333
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1565.5157165527344,
      "epoch": 0.09976850123217086,
      "grad_norm": 0.250499427318573,
      "kl": 0.1373291015625,
      "learning_rate": 4.985074626865671e-07,
      "loss": 0.0394,
      "reward": 0.5485491305589676,
      "reward_std": 0.13868961296975613,
      "rewards/accuracy_reward": 0.08258928963914514,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4659598469734192,
      "step": 334
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1544.6228332519531,
      "epoch": 0.10006720931969233,
      "grad_norm": 0.25107425451278687,
      "kl": 0.152099609375,
      "learning_rate": 5e-07,
      "loss": 0.035,
      "reward": 0.5558035969734192,
      "reward_std": 0.1405537649989128,
      "rewards/accuracy_reward": 0.0803571455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4754464477300644,
      "step": 335
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1516.6094360351562,
      "epoch": 0.1003659174072138,
      "grad_norm": 0.2001575380563736,
      "kl": 0.1507568359375,
      "learning_rate": 4.999998640122513e-07,
      "loss": 0.0491,
      "reward": 0.510602705180645,
      "reward_std": 0.10413178242743015,
      "rewards/accuracy_reward": 0.029017858440056443,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4815848469734192,
      "step": 336
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1602.8058471679688,
      "epoch": 0.10066462549473527,
      "grad_norm": 0.21289537847042084,
      "kl": 0.162353515625,
      "learning_rate": 4.999994560491531e-07,
      "loss": 0.037,
      "reward": 0.5156250223517418,
      "reward_std": 0.0598137229681015,
      "rewards/accuracy_reward": 0.0379464291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4776785969734192,
      "step": 337
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1461.9353332519531,
      "epoch": 0.10096333358225675,
      "grad_norm": 0.19642992317676544,
      "kl": 0.148681640625,
      "learning_rate": 4.999987761111494e-07,
      "loss": 0.0374,
      "reward": 0.579241082072258,
      "reward_std": 0.08721552789211273,
      "rewards/accuracy_reward": 0.09375000116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.485491082072258,
      "step": 338
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1540.4866638183594,
      "epoch": 0.10126204166977822,
      "grad_norm": 0.22309331595897675,
      "kl": 0.160400390625,
      "learning_rate": 4.999978241989798e-07,
      "loss": 0.0356,
      "reward": 0.5306919813156128,
      "reward_std": 0.07228913344442844,
      "rewards/accuracy_reward": 0.04687500232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4838169813156128,
      "step": 339
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1526.8706359863281,
      "epoch": 0.10156074975729967,
      "grad_norm": 0.2615892291069031,
      "kl": 0.162353515625,
      "learning_rate": 4.999966003136799e-07,
      "loss": 0.0214,
      "reward": 0.560825914144516,
      "reward_std": 0.1372564658522606,
      "rewards/accuracy_reward": 0.07589286006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4849330559372902,
      "step": 340
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1486.16748046875,
      "epoch": 0.10185945784482114,
      "grad_norm": 0.2156354933977127,
      "kl": 0.158935546875,
      "learning_rate": 4.999951044565813e-07,
      "loss": 0.0273,
      "reward": 0.667410746216774,
      "reward_std": 0.10443050414323807,
      "rewards/accuracy_reward": 0.1741071529686451,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4933035969734192,
      "step": 341
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1449.8861999511719,
      "epoch": 0.10215816593234262,
      "grad_norm": 0.21398331224918365,
      "kl": 0.15478515625,
      "learning_rate": 4.99993336629311e-07,
      "loss": 0.043,
      "reward": 0.5669643208384514,
      "reward_std": 0.13367497269064188,
      "rewards/accuracy_reward": 0.08482143515720963,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.482142873108387,
      "step": 342
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1495.2590026855469,
      "epoch": 0.10245687401986409,
      "grad_norm": 0.24133774638175964,
      "kl": 0.16748046875,
      "learning_rate": 4.999912968337924e-07,
      "loss": 0.0337,
      "reward": 0.5920759290456772,
      "reward_std": 0.0974612906575203,
      "rewards/accuracy_reward": 0.10714286286383867,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4849330633878708,
      "step": 343
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1573.4353332519531,
      "epoch": 0.10275558210738556,
      "grad_norm": 0.20990975201129913,
      "kl": 0.1728515625,
      "learning_rate": 4.999889850722448e-07,
      "loss": 0.034,
      "reward": 0.572544664144516,
      "reward_std": 0.11599421501159668,
      "rewards/accuracy_reward": 0.08705357694998384,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4854910895228386,
      "step": 344
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1439.26123046875,
      "epoch": 0.10305429019490703,
      "grad_norm": 0.20129843056201935,
      "kl": 0.1591796875,
      "learning_rate": 4.999864013471828e-07,
      "loss": 0.0263,
      "reward": 0.5301339477300644,
      "reward_std": 0.09216642938554287,
      "rewards/accuracy_reward": 0.03794643050059676,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4921875223517418,
      "step": 345
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1588.8304443359375,
      "epoch": 0.1033529982824285,
      "grad_norm": 0.24424806237220764,
      "kl": 0.16796875,
      "learning_rate": 4.999835456614175e-07,
      "loss": 0.0271,
      "reward": 0.5412946566939354,
      "reward_std": 0.08315367996692657,
      "rewards/accuracy_reward": 0.05580357206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4854910895228386,
      "step": 346
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1638.3973693847656,
      "epoch": 0.10365170636994997,
      "grad_norm": 0.22653424739837646,
      "kl": 0.171142578125,
      "learning_rate": 4.999804180180557e-07,
      "loss": 0.025,
      "reward": 0.549107164144516,
      "reward_std": 0.11042520403862,
      "rewards/accuracy_reward": 0.06696428847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4821428805589676,
      "step": 347
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1573.3639221191406,
      "epoch": 0.10395041445747144,
      "grad_norm": 0.2334415465593338,
      "kl": 0.16064453125,
      "learning_rate": 4.999770184204996e-07,
      "loss": 0.0197,
      "reward": 0.573660746216774,
      "reward_std": 0.13343196082860231,
      "rewards/accuracy_reward": 0.08482143096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4888393059372902,
      "step": 348
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1538.7611999511719,
      "epoch": 0.10424912254499291,
      "grad_norm": 0.21041445434093475,
      "kl": 0.15380859375,
      "learning_rate": 4.999733468724479e-07,
      "loss": 0.0195,
      "reward": 0.5770089477300644,
      "reward_std": 0.15266066789627075,
      "rewards/accuracy_reward": 0.0892857164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4877232313156128,
      "step": 349
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1588.4085693359375,
      "epoch": 0.10454783063251437,
      "grad_norm": 0.24560904502868652,
      "kl": 0.158447265625,
      "learning_rate": 4.999694033778949e-07,
      "loss": 0.0166,
      "reward": 0.643973246216774,
      "reward_std": 0.10717487148940563,
      "rewards/accuracy_reward": 0.160714291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4832589477300644,
      "step": 350
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1590.0313415527344,
      "epoch": 0.10484653872003584,
      "grad_norm": 0.229527086019516,
      "kl": 0.1650390625,
      "learning_rate": 4.999651879411306e-07,
      "loss": 0.0158,
      "reward": 0.5954241454601288,
      "reward_std": 0.05622940510511398,
      "rewards/accuracy_reward": 0.1116071455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4838169887661934,
      "step": 351
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1580.8839721679688,
      "epoch": 0.10514524680755731,
      "grad_norm": 0.2322050780057907,
      "kl": 0.157470703125,
      "learning_rate": 4.999607005667411e-07,
      "loss": 0.0269,
      "reward": 0.592075914144516,
      "reward_std": 0.13284130953252316,
      "rewards/accuracy_reward": 0.10714286006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4849330559372902,
      "step": 352
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1606.4197082519531,
      "epoch": 0.10544395489507878,
      "grad_norm": 0.2224269062280655,
      "kl": 0.14599609375,
      "learning_rate": 4.99955941259608e-07,
      "loss": 0.0405,
      "reward": 0.5703125149011612,
      "reward_std": 0.1476429346948862,
      "rewards/accuracy_reward": 0.09375000325962901,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765625223517418,
      "step": 353
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1645.9866943359375,
      "epoch": 0.10574266298260025,
      "grad_norm": 0.20698876678943634,
      "kl": 0.15283203125,
      "learning_rate": 4.999509100249093e-07,
      "loss": 0.0223,
      "reward": 0.6473214477300644,
      "reward_std": 0.12398159131407738,
      "rewards/accuracy_reward": 0.16294643515720963,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4843750223517418,
      "step": 354
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1606.7835693359375,
      "epoch": 0.10604137107012172,
      "grad_norm": 0.22010749578475952,
      "kl": 0.14794921875,
      "learning_rate": 4.999456068681183e-07,
      "loss": 0.0103,
      "reward": 0.521205373108387,
      "reward_std": 0.05684919096529484,
      "rewards/accuracy_reward": 0.04017857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4810267984867096,
      "step": 355
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1661.2969665527344,
      "epoch": 0.1063400791576432,
      "grad_norm": 0.22447185218334198,
      "kl": 0.146484375,
      "learning_rate": 4.999400317950042e-07,
      "loss": 0.0407,
      "reward": 0.5424107313156128,
      "reward_std": 0.10662781819701195,
      "rewards/accuracy_reward": 0.06696428917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4754464477300644,
      "step": 356
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1757.6786804199219,
      "epoch": 0.10663878724516467,
      "grad_norm": 0.2322104573249817,
      "kl": 0.154541015625,
      "learning_rate": 4.999341848116324e-07,
      "loss": 0.0247,
      "reward": 0.5301339626312256,
      "reward_std": 0.10411343909800053,
      "rewards/accuracy_reward": 0.05357143236324191,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765625223517418,
      "step": 357
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1714.8460388183594,
      "epoch": 0.10693749533268614,
      "grad_norm": 0.20849138498306274,
      "kl": 0.14892578125,
      "learning_rate": 4.999280659243637e-07,
      "loss": 0.0362,
      "reward": 0.5156250298023224,
      "reward_std": 0.15004741959273815,
      "rewards/accuracy_reward": 0.053571431897580624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4620535969734192,
      "step": 358
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1749.5781860351562,
      "epoch": 0.10723620342020761,
      "grad_norm": 0.2080266773700714,
      "kl": 0.15087890625,
      "learning_rate": 4.999216751398547e-07,
      "loss": 0.0252,
      "reward": 0.5440848469734192,
      "reward_std": 0.16194296814501286,
      "rewards/accuracy_reward": 0.07366071757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4704241305589676,
      "step": 359
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1712.55810546875,
      "epoch": 0.10753491150772908,
      "grad_norm": 0.24691513180732727,
      "kl": 0.137451171875,
      "learning_rate": 4.999150124650583e-07,
      "loss": 0.0429,
      "reward": 0.5792411044239998,
      "reward_std": 0.1408797036856413,
      "rewards/accuracy_reward": 0.1093750074505806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4698660895228386,
      "step": 360
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1715.5536499023438,
      "epoch": 0.10783361959525053,
      "grad_norm": 0.20999087393283844,
      "kl": 0.138671875,
      "learning_rate": 4.999080779072225e-07,
      "loss": 0.0387,
      "reward": 0.4927455559372902,
      "reward_std": 0.1048511303961277,
      "rewards/accuracy_reward": 0.02008928661234677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4726562798023224,
      "step": 361
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1730.04248046875,
      "epoch": 0.108132327682772,
      "grad_norm": 0.22684308886528015,
      "kl": 0.142822265625,
      "learning_rate": 4.999008714738917e-07,
      "loss": 0.0347,
      "reward": 0.518415205180645,
      "reward_std": 0.13277038373053074,
      "rewards/accuracy_reward": 0.0468750037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4715401977300644,
      "step": 362
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1600.7500915527344,
      "epoch": 0.10843103577029348,
      "grad_norm": 0.2272636741399765,
      "kl": 0.13671875,
      "learning_rate": 4.998933931729055e-07,
      "loss": 0.0348,
      "reward": 0.6417411044239998,
      "reward_std": 0.06803015759214759,
      "rewards/accuracy_reward": 0.1629464365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.478794664144516,
      "step": 363
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1686.4308776855469,
      "epoch": 0.10872974385781495,
      "grad_norm": 0.2002456933259964,
      "kl": 0.13623046875,
      "learning_rate": 4.998856430123998e-07,
      "loss": 0.0255,
      "reward": 0.6707589626312256,
      "reward_std": 0.1025421367958188,
      "rewards/accuracy_reward": 0.1986607238650322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4720982313156128,
      "step": 364
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1701.0514221191406,
      "epoch": 0.10902845194533642,
      "grad_norm": 0.2080105096101761,
      "kl": 0.14697265625,
      "learning_rate": 4.998776210008058e-07,
      "loss": 0.0267,
      "reward": 0.6138393133878708,
      "reward_std": 0.1752930535003543,
      "rewards/accuracy_reward": 0.1540178619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4598214477300644,
      "step": 365
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1713.3750915527344,
      "epoch": 0.10932716003285789,
      "grad_norm": 0.27657926082611084,
      "kl": 0.139892578125,
      "learning_rate": 4.99869327146851e-07,
      "loss": 0.0413,
      "reward": 0.5306919887661934,
      "reward_std": 0.13885026797652245,
      "rewards/accuracy_reward": 0.07589285937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4547991305589676,
      "step": 366
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1821.3125915527344,
      "epoch": 0.10962586812037936,
      "grad_norm": 0.2439846694469452,
      "kl": 0.150634765625,
      "learning_rate": 4.99860761459558e-07,
      "loss": 0.0256,
      "reward": 0.5234375223517418,
      "reward_std": 0.12117623444646597,
      "rewards/accuracy_reward": 0.06919643096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4542410895228386,
      "step": 367
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1756.5134582519531,
      "epoch": 0.10992457620790083,
      "grad_norm": 0.2327897548675537,
      "kl": 0.136962890625,
      "learning_rate": 4.998519239482455e-07,
      "loss": 0.0436,
      "reward": 0.4983259066939354,
      "reward_std": 0.14618432894349098,
      "rewards/accuracy_reward": 0.03571428684517741,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4626116305589676,
      "step": 368
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1784.024658203125,
      "epoch": 0.1102232842954223,
      "grad_norm": 0.23025494813919067,
      "kl": 0.137451171875,
      "learning_rate": 4.998428146225279e-07,
      "loss": 0.0288,
      "reward": 0.5139509215950966,
      "reward_std": 0.1309387218207121,
      "rewards/accuracy_reward": 0.044642859138548374,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4693080633878708,
      "step": 369
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1837.1407165527344,
      "epoch": 0.11052199238294377,
      "grad_norm": 0.2914640009403229,
      "kl": 0.153564453125,
      "learning_rate": 4.998334334923153e-07,
      "loss": 0.0342,
      "reward": 0.6065848544239998,
      "reward_std": 0.13887898437678814,
      "rewards/accuracy_reward": 0.1540178656578064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4525669887661934,
      "step": 370
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1643.8840026855469,
      "epoch": 0.11082070047046524,
      "grad_norm": 0.24380235373973846,
      "kl": 0.1292724609375,
      "learning_rate": 4.998237805678131e-07,
      "loss": 0.0439,
      "reward": 0.5306919887661934,
      "reward_std": 0.16333149373531342,
      "rewards/accuracy_reward": 0.055803574388846755,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.474888414144516,
      "step": 371
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1780.2232971191406,
      "epoch": 0.1111194085579867,
      "grad_norm": 0.35335609316825867,
      "kl": 0.141845703125,
      "learning_rate": 4.998138558595232e-07,
      "loss": 0.0404,
      "reward": 0.5212053880095482,
      "reward_std": 0.1817568503320217,
      "rewards/accuracy_reward": 0.060267859837040305,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375298023224,
      "step": 372
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1721.6853332519531,
      "epoch": 0.11141811664550817,
      "grad_norm": 0.3098659813404083,
      "kl": 0.1392822265625,
      "learning_rate": 4.998036593782426e-07,
      "loss": 0.016,
      "reward": 0.6004464626312256,
      "reward_std": 0.11609909869730473,
      "rewards/accuracy_reward": 0.13169643771834671,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4687500223517418,
      "step": 373
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1813.0223999023438,
      "epoch": 0.11171682473302964,
      "grad_norm": 0.31774982810020447,
      "kl": 0.146240234375,
      "learning_rate": 4.997931911350638e-07,
      "loss": 0.0234,
      "reward": 0.5569196715950966,
      "reward_std": 0.11959274858236313,
      "rewards/accuracy_reward": 0.09375000488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.463169664144516,
      "step": 374
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1698.4732971191406,
      "epoch": 0.11201553282055111,
      "grad_norm": 0.38727355003356934,
      "kl": 0.139892578125,
      "learning_rate": 4.997824511413755e-07,
      "loss": 0.037,
      "reward": 0.6422991305589676,
      "reward_std": 0.18701894208788872,
      "rewards/accuracy_reward": 0.1763392984867096,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4659598395228386,
      "step": 375
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1693.9085693359375,
      "epoch": 0.11231424090807259,
      "grad_norm": 0.5233216881752014,
      "kl": 0.135498046875,
      "learning_rate": 4.997714394088615e-07,
      "loss": 0.0319,
      "reward": 0.5970982387661934,
      "reward_std": 0.16860965825617313,
      "rewards/accuracy_reward": 0.12946429289877415,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.467633955180645,
      "step": 376
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1803.0000915527344,
      "epoch": 0.11261294899559406,
      "grad_norm": 0.9928394556045532,
      "kl": 0.15234375,
      "learning_rate": 4.99760155949502e-07,
      "loss": 0.0308,
      "reward": 0.5212053805589676,
      "reward_std": 0.09945615381002426,
      "rewards/accuracy_reward": 0.04910714412108064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4720982387661934,
      "step": 377
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1747.2701721191406,
      "epoch": 0.11291165708311553,
      "grad_norm": 0.9551793932914734,
      "kl": 0.1475830078125,
      "learning_rate": 4.997486007755717e-07,
      "loss": 0.028,
      "reward": 0.5446428880095482,
      "reward_std": 0.15482873655855656,
      "rewards/accuracy_reward": 0.0937500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.450892873108387,
      "step": 378
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1748.4666137695312,
      "epoch": 0.113210365170637,
      "grad_norm": 0.43657103180885315,
      "kl": 0.16357421875,
      "learning_rate": 4.997367738996418e-07,
      "loss": 0.0317,
      "reward": 0.5563616454601288,
      "reward_std": 0.16272744163870811,
      "rewards/accuracy_reward": 0.09598214365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4603794887661934,
      "step": 379
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1762.6942443847656,
      "epoch": 0.11350907325815847,
      "grad_norm": 0.6815599203109741,
      "kl": 0.1396484375,
      "learning_rate": 4.997246753345788e-07,
      "loss": 0.0329,
      "reward": 0.554129496216774,
      "reward_std": 0.13164770789444447,
      "rewards/accuracy_reward": 0.0848214328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4693080633878708,
      "step": 380
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1746.5045776367188,
      "epoch": 0.11380778134567994,
      "grad_norm": 1.869083046913147,
      "kl": 0.15673828125,
      "learning_rate": 4.997123050935445e-07,
      "loss": 0.0367,
      "reward": 0.6065848469734192,
      "reward_std": 0.14629404246807098,
      "rewards/accuracy_reward": 0.14285715157166123,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.463727705180645,
      "step": 381
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1617.2545471191406,
      "epoch": 0.11410648943320141,
      "grad_norm": 1.0162982940673828,
      "kl": 0.139404296875,
      "learning_rate": 4.996996631899968e-07,
      "loss": 0.0422,
      "reward": 0.5345982387661934,
      "reward_std": 0.15161003172397614,
      "rewards/accuracy_reward": 0.07142857392318547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4631696715950966,
      "step": 382
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1616.4621276855469,
      "epoch": 0.11440519752072287,
      "grad_norm": 0.9915550351142883,
      "kl": 0.1331787109375,
      "learning_rate": 4.996867496376888e-07,
      "loss": 0.0323,
      "reward": 0.6261160969734192,
      "reward_std": 0.13551262579858303,
      "rewards/accuracy_reward": 0.15401786752045155,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4720982387661934,
      "step": 383
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1723.87060546875,
      "epoch": 0.11470390560824434,
      "grad_norm": 2.39125919342041,
      "kl": 0.165283203125,
      "learning_rate": 4.99673564450669e-07,
      "loss": 0.0426,
      "reward": 0.6383928805589676,
      "reward_std": 0.22527769207954407,
      "rewards/accuracy_reward": 0.17410715040750802,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4642857313156128,
      "step": 384
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1731.6630554199219,
      "epoch": 0.11500261369576581,
      "grad_norm": 0.9077367186546326,
      "kl": 0.151123046875,
      "learning_rate": 4.996601076432819e-07,
      "loss": 0.0374,
      "reward": 0.5373884215950966,
      "reward_std": 0.15877192094922066,
      "rewards/accuracy_reward": 0.06696428917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4704241305589676,
      "step": 385
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1788.90185546875,
      "epoch": 0.11530132178328728,
      "grad_norm": 2.999178647994995,
      "kl": 0.16748046875,
      "learning_rate": 4.996463792301669e-07,
      "loss": 0.0492,
      "reward": 0.5948660895228386,
      "reward_std": 0.13126266933977604,
      "rewards/accuracy_reward": 0.14062500605359674,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4542410969734192,
      "step": 386
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1669.9710693359375,
      "epoch": 0.11560002987080875,
      "grad_norm": 1.6843479871749878,
      "kl": 0.164794921875,
      "learning_rate": 4.996323792262593e-07,
      "loss": 0.0594,
      "reward": 0.6545759215950966,
      "reward_std": 0.1734354868531227,
      "rewards/accuracy_reward": 0.1986607238650322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4559151977300644,
      "step": 387
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1723.0983276367188,
      "epoch": 0.11589873795833022,
      "grad_norm": 8.725236892700195,
      "kl": 0.219970703125,
      "learning_rate": 4.996181076467897e-07,
      "loss": 0.049,
      "reward": 0.5758928880095482,
      "reward_std": 0.16588085889816284,
      "rewards/accuracy_reward": 0.1250000095460564,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.450892873108387,
      "step": 388
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1720.8728332519531,
      "epoch": 0.1161974460458517,
      "grad_norm": 3.7546021938323975,
      "kl": 0.15234375,
      "learning_rate": 4.996035645072842e-07,
      "loss": 0.0472,
      "reward": 0.6199777126312256,
      "reward_std": 0.2237107828259468,
      "rewards/accuracy_reward": 0.1674107164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4525669813156128,
      "step": 389
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1721.5715026855469,
      "epoch": 0.11649615413337316,
      "grad_norm": 2.650991439819336,
      "kl": 0.144775390625,
      "learning_rate": 4.995887498235642e-07,
      "loss": 0.0347,
      "reward": 0.5937500298023224,
      "reward_std": 0.20694916136562824,
      "rewards/accuracy_reward": 0.14062500558793545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4531250149011612,
      "step": 390
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1739.0157165527344,
      "epoch": 0.11679486222089464,
      "grad_norm": 5.359323501586914,
      "kl": 0.18017578125,
      "learning_rate": 4.995736636117468e-07,
      "loss": 0.0665,
      "reward": 0.5000000223517418,
      "reward_std": 0.1435579787939787,
      "rewards/accuracy_reward": 0.05133928847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4486607313156128,
      "step": 391
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1737.6116943359375,
      "epoch": 0.1170935703084161,
      "grad_norm": 17.036531448364258,
      "kl": 0.298095703125,
      "learning_rate": 4.995583058882442e-07,
      "loss": 0.0465,
      "reward": 0.5931920036673546,
      "reward_std": 0.14481880329549313,
      "rewards/accuracy_reward": 0.1473214365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4458705633878708,
      "step": 392
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1744.5759582519531,
      "epoch": 0.11739227839593756,
      "grad_norm": 72.09225463867188,
      "kl": 0.545166015625,
      "learning_rate": 4.995426766697641e-07,
      "loss": 0.0643,
      "reward": 0.4581473469734192,
      "reward_std": 0.1382804922759533,
      "rewards/accuracy_reward": 0.013392857741564512,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4447544813156128,
      "step": 393
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1720.3348693847656,
      "epoch": 0.11769098648345903,
      "grad_norm": 8.964143753051758,
      "kl": 0.230224609375,
      "learning_rate": 4.995267759733098e-07,
      "loss": 0.0379,
      "reward": 0.5703125149011612,
      "reward_std": 0.18131808936595917,
      "rewards/accuracy_reward": 0.1227678619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.447544664144516,
      "step": 394
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1819.1473693847656,
      "epoch": 0.1179896945709805,
      "grad_norm": 4.219891548156738,
      "kl": 0.21484375,
      "learning_rate": 4.995106038161793e-07,
      "loss": 0.0494,
      "reward": 0.5128348469734192,
      "reward_std": 0.1619175262749195,
      "rewards/accuracy_reward": 0.07589286006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419887661934,
      "step": 395
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1678.4063110351562,
      "epoch": 0.11828840265850198,
      "grad_norm": 9.964911460876465,
      "kl": 0.28759765625,
      "learning_rate": 4.994941602159665e-07,
      "loss": 0.0561,
      "reward": 0.5954241380095482,
      "reward_std": 0.17944439128041267,
      "rewards/accuracy_reward": 0.15848215157166123,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419813156128,
      "step": 396
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1665.4063110351562,
      "epoch": 0.11858711074602345,
      "grad_norm": 6.321689128875732,
      "kl": 0.24365234375,
      "learning_rate": 4.994774451905606e-07,
      "loss": 0.0578,
      "reward": 0.5691964477300644,
      "reward_std": 0.20852927677333355,
      "rewards/accuracy_reward": 0.12276786030270159,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4464285895228386,
      "step": 397
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1783.6362609863281,
      "epoch": 0.11888581883354492,
      "grad_norm": 23.902652740478516,
      "kl": 0.435546875,
      "learning_rate": 4.994604587581456e-07,
      "loss": 0.057,
      "reward": 0.5563616454601288,
      "reward_std": 0.20811011269688606,
      "rewards/accuracy_reward": 0.12723214738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294887661934,
      "step": 398
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1746.5402526855469,
      "epoch": 0.11918452692106639,
      "grad_norm": 5.511786460876465,
      "kl": 0.239990234375,
      "learning_rate": 4.994432009372012e-07,
      "loss": 0.0547,
      "reward": 0.5100446715950966,
      "reward_std": 0.14884588681161404,
      "rewards/accuracy_reward": 0.06696429033763707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443080373108387,
      "step": 399
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1792.8460693359375,
      "epoch": 0.11948323500858786,
      "grad_norm": 6.302380084991455,
      "kl": 0.267578125,
      "learning_rate": 4.994256717465024e-07,
      "loss": 0.0486,
      "reward": 0.5357143133878708,
      "reward_std": 0.15216713212430477,
      "rewards/accuracy_reward": 0.0959821455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.439732164144516,
      "step": 400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1730.0781860351562,
      "epoch": 0.11978194309610933,
      "grad_norm": 4.950244426727295,
      "kl": 0.23779296875,
      "learning_rate": 4.99407871205119e-07,
      "loss": 0.0591,
      "reward": 0.5446428880095482,
      "reward_std": 0.17566783353686333,
      "rewards/accuracy_reward": 0.11160714970901608,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357238650322,
      "step": 401
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1691.9732971191406,
      "epoch": 0.1200806511836308,
      "grad_norm": 4.869670391082764,
      "kl": 0.2041015625,
      "learning_rate": 4.993897993324163e-07,
      "loss": 0.0691,
      "reward": 0.5094866305589676,
      "reward_std": 0.15183800272643566,
      "rewards/accuracy_reward": 0.06250000302679837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4469866305589676,
      "step": 402
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1793.5558776855469,
      "epoch": 0.12037935927115227,
      "grad_norm": 4.653181076049805,
      "kl": 0.35205078125,
      "learning_rate": 4.993714561480547e-07,
      "loss": 0.0653,
      "reward": 0.4698660895228386,
      "reward_std": 0.16444317623972893,
      "rewards/accuracy_reward": 0.05133928684517741,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268133878708,
      "step": 403
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1730.368408203125,
      "epoch": 0.12067806735867373,
      "grad_norm": 2.515561103820801,
      "kl": 0.248291015625,
      "learning_rate": 4.9935284167199e-07,
      "loss": 0.0418,
      "reward": 0.522321455180645,
      "reward_std": 0.19265655055642128,
      "rewards/accuracy_reward": 0.09151786239817739,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035895228386,
      "step": 404
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1764.7813110351562,
      "epoch": 0.1209767754461952,
      "grad_norm": 4.770638942718506,
      "kl": 0.2587890625,
      "learning_rate": 4.993339559244727e-07,
      "loss": 0.0525,
      "reward": 0.4949776977300644,
      "reward_std": 0.14385286159813404,
      "rewards/accuracy_reward": 0.05803571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419887661934,
      "step": 405
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1728.9197082519531,
      "epoch": 0.12127548353371667,
      "grad_norm": 5.24013090133667,
      "kl": 0.33544921875,
      "learning_rate": 4.993147989260487e-07,
      "loss": 0.0712,
      "reward": 0.5340402126312256,
      "reward_std": 0.180188599973917,
      "rewards/accuracy_reward": 0.10937500675208867,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.424665205180645,
      "step": 406
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1707.6675109863281,
      "epoch": 0.12157419162123814,
      "grad_norm": 6.4574103355407715,
      "kl": 0.30908203125,
      "learning_rate": 4.992953706975589e-07,
      "loss": 0.0739,
      "reward": 0.5479911044239998,
      "reward_std": 0.1909767296165228,
      "rewards/accuracy_reward": 0.1227678619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232313156128,
      "step": 407
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1753.4532165527344,
      "epoch": 0.12187289970875961,
      "grad_norm": 7.233785629272461,
      "kl": 0.32666015625,
      "learning_rate": 4.992756712601395e-07,
      "loss": 0.0609,
      "reward": 0.545200914144516,
      "reward_std": 0.21929758042097092,
      "rewards/accuracy_reward": 0.1071428619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4380580484867096,
      "step": 408
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1683.8438415527344,
      "epoch": 0.12217160779628108,
      "grad_norm": 3.7974467277526855,
      "kl": 0.358154296875,
      "learning_rate": 4.992557006352213e-07,
      "loss": 0.0642,
      "reward": 0.464285746216774,
      "reward_std": 0.16011319309473038,
      "rewards/accuracy_reward": 0.0401785746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4241071715950966,
      "step": 409
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1794.7880249023438,
      "epoch": 0.12247031588380256,
      "grad_norm": 6.321704864501953,
      "kl": 0.35498046875,
      "learning_rate": 4.992354588445304e-07,
      "loss": 0.0716,
      "reward": 0.4637276902794838,
      "reward_std": 0.21002240851521492,
      "rewards/accuracy_reward": 0.06919643213041127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3945312649011612,
      "step": 410
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1760.6384582519531,
      "epoch": 0.12276902397132403,
      "grad_norm": 5.116876602172852,
      "kl": 0.51953125,
      "learning_rate": 4.992149459100881e-07,
      "loss": 0.0869,
      "reward": 0.419642873108387,
      "reward_std": 0.1749109923839569,
      "rewards/accuracy_reward": 0.022321429569274187,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3973214402794838,
      "step": 411
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1694.9665832519531,
      "epoch": 0.1230677320588455,
      "grad_norm": 3.7119460105895996,
      "kl": 0.44677734375,
      "learning_rate": 4.991941618542103e-07,
      "loss": 0.0801,
      "reward": 0.5245535969734192,
      "reward_std": 0.19552000239491463,
      "rewards/accuracy_reward": 0.10267857764847577,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 412
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1696.7076721191406,
      "epoch": 0.12336644014636697,
      "grad_norm": 17.278169631958008,
      "kl": 0.5771484375,
      "learning_rate": 4.991731066995081e-07,
      "loss": 0.0754,
      "reward": 0.4916294887661934,
      "reward_std": 0.2157577946782112,
      "rewards/accuracy_reward": 0.07812500302679837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4135044813156128,
      "step": 413
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1641.4665832519531,
      "epoch": 0.12366514823388844,
      "grad_norm": 6.896962642669678,
      "kl": 0.55859375,
      "learning_rate": 4.991517804688874e-07,
      "loss": 0.1012,
      "reward": 0.5390625074505806,
      "reward_std": 0.1763317808508873,
      "rewards/accuracy_reward": 0.1116071492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4274553805589676,
      "step": 414
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1647.5245971679688,
      "epoch": 0.1239638563214099,
      "grad_norm": 45.734519958496094,
      "kl": 0.98388671875,
      "learning_rate": 4.99130183185549e-07,
      "loss": 0.1151,
      "reward": 0.545200914144516,
      "reward_std": 0.19960248842835426,
      "rewards/accuracy_reward": 0.13392857694998384,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4112723395228386,
      "step": 415
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1768.7813415527344,
      "epoch": 0.12426256440893137,
      "grad_norm": 32.52640914916992,
      "kl": 0.828125,
      "learning_rate": 4.991083148729887e-07,
      "loss": 0.1071,
      "reward": 0.4520089402794838,
      "reward_std": 0.17455745488405228,
      "rewards/accuracy_reward": 0.0468750037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4051339402794838,
      "step": 416
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1768.30810546875,
      "epoch": 0.12456127249645284,
      "grad_norm": 36.33440399169922,
      "kl": 0.9228515625,
      "learning_rate": 4.99086175554997e-07,
      "loss": 0.0832,
      "reward": 0.4642857387661934,
      "reward_std": 0.1830338705331087,
      "rewards/accuracy_reward": 0.06696428847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3973214402794838,
      "step": 417
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1749.8795471191406,
      "epoch": 0.12485998058397431,
      "grad_norm": 26.102466583251953,
      "kl": 0.888671875,
      "learning_rate": 4.990637652556595e-07,
      "loss": 0.1025,
      "reward": 0.486607164144516,
      "reward_std": 0.19966524839401245,
      "rewards/accuracy_reward": 0.07589286053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4107142984867096,
      "step": 418
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1715.1295471191406,
      "epoch": 0.1251586886714958,
      "grad_norm": 18.954999923706055,
      "kl": 0.77734375,
      "learning_rate": 4.990410839993562e-07,
      "loss": 0.0854,
      "reward": 0.509486623108387,
      "reward_std": 0.24269787967205048,
      "rewards/accuracy_reward": 0.09598214644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4135044813156128,
      "step": 419
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1750.8728332519531,
      "epoch": 0.12545739675901724,
      "grad_norm": 16.162120819091797,
      "kl": 0.7392578125,
      "learning_rate": 4.990181318107622e-07,
      "loss": 0.0835,
      "reward": 0.5323660895228386,
      "reward_std": 0.17519783973693848,
      "rewards/accuracy_reward": 0.12946429196745157,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4029018059372902,
      "step": 420
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1725.7567749023438,
      "epoch": 0.1257561048465387,
      "grad_norm": 17.33260154724121,
      "kl": 0.830078125,
      "learning_rate": 4.989949087148471e-07,
      "loss": 0.1055,
      "reward": 0.4380580559372902,
      "reward_std": 0.21490630134940147,
      "rewards/accuracy_reward": 0.040178572526201606,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3978794813156128,
      "step": 421
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1665.6630249023438,
      "epoch": 0.12605481293406018,
      "grad_norm": 4.674254417419434,
      "kl": 0.55615234375,
      "learning_rate": 4.989714147368754e-07,
      "loss": 0.085,
      "reward": 0.5323660969734192,
      "reward_std": 0.23007908090949059,
      "rewards/accuracy_reward": 0.1205357201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4118303656578064,
      "step": 422
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1710.3326416015625,
      "epoch": 0.12635352102158165,
      "grad_norm": 9.157416343688965,
      "kl": 0.62548828125,
      "learning_rate": 4.989476499024064e-07,
      "loss": 0.0786,
      "reward": 0.483258955180645,
      "reward_std": 0.18618686869740486,
      "rewards/accuracy_reward": 0.06919643236324191,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625149011612,
      "step": 423
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1757.4308776855469,
      "epoch": 0.12665222910910312,
      "grad_norm": 8.874431610107422,
      "kl": 0.796875,
      "learning_rate": 4.989236142372937e-07,
      "loss": 0.1091,
      "reward": 0.506138414144516,
      "reward_std": 0.17440990172326565,
      "rewards/accuracy_reward": 0.10267857648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4034598469734192,
      "step": 424
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1796.5514526367188,
      "epoch": 0.1269509371966246,
      "grad_norm": 37.65040969848633,
      "kl": 0.8935546875,
      "learning_rate": 4.98899307767686e-07,
      "loss": 0.1022,
      "reward": 0.4302455559372902,
      "reward_std": 0.18125955015420914,
      "rewards/accuracy_reward": 0.04910714668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3811384066939354,
      "step": 425
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1762.6853332519531,
      "epoch": 0.12724964528414606,
      "grad_norm": 5.7325968742370605,
      "kl": 0.62109375,
      "learning_rate": 4.988747305200261e-07,
      "loss": 0.0898,
      "reward": 0.5234375298023224,
      "reward_std": 0.19826872646808624,
      "rewards/accuracy_reward": 0.12276786519214511,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.400669664144516,
      "step": 426
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1754.0156860351562,
      "epoch": 0.12754835337166753,
      "grad_norm": 9.744460105895996,
      "kl": 0.6396484375,
      "learning_rate": 4.988498825210518e-07,
      "loss": 0.0931,
      "reward": 0.4648437649011612,
      "reward_std": 0.18562539666891098,
      "rewards/accuracy_reward": 0.06250000116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4023437649011612,
      "step": 427
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1726.54248046875,
      "epoch": 0.127847061459189,
      "grad_norm": 9.612737655639648,
      "kl": 0.53271484375,
      "learning_rate": 4.988247637977952e-07,
      "loss": 0.098,
      "reward": 0.4871651902794838,
      "reward_std": 0.1740439496934414,
      "rewards/accuracy_reward": 0.08705357322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.400111623108387,
      "step": 428
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1671.3728942871094,
      "epoch": 0.12814576954671048,
      "grad_norm": 4.392724990844727,
      "kl": 0.53369140625,
      "learning_rate": 4.98799374377583e-07,
      "loss": 0.1116,
      "reward": 0.4564732387661934,
      "reward_std": 0.1938146837055683,
      "rewards/accuracy_reward": 0.0580357164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375149011612,
      "step": 429
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1803.5402526855469,
      "epoch": 0.12844447763423195,
      "grad_norm": 2.9815316200256348,
      "kl": 0.50830078125,
      "learning_rate": 4.987737142880363e-07,
      "loss": 0.0928,
      "reward": 0.4575892984867096,
      "reward_std": 0.21182601898908615,
      "rewards/accuracy_reward": 0.07366071827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3839285895228386,
      "step": 430
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1661.7545471191406,
      "epoch": 0.12874318572175342,
      "grad_norm": 11.794843673706055,
      "kl": 0.41552734375,
      "learning_rate": 4.987477835570711e-07,
      "loss": 0.1018,
      "reward": 0.5452009215950966,
      "reward_std": 0.19809302687644958,
      "rewards/accuracy_reward": 0.1428571455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4023437649011612,
      "step": 431
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1773.05810546875,
      "epoch": 0.1290418938092749,
      "grad_norm": 4.0646514892578125,
      "kl": 0.47607421875,
      "learning_rate": 4.987215822128973e-07,
      "loss": 0.089,
      "reward": 0.4849330633878708,
      "reward_std": 0.19286515936255455,
      "rewards/accuracy_reward": 0.1004464328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.384486623108387,
      "step": 432
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1694.8148193359375,
      "epoch": 0.12934060189679636,
      "grad_norm": 8.928903579711914,
      "kl": 0.3759765625,
      "learning_rate": 4.986951102840193e-07,
      "loss": 0.0979,
      "reward": 0.4235491380095482,
      "reward_std": 0.16311455145478249,
      "rewards/accuracy_reward": 0.015625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4079241305589676,
      "step": 433
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1747.33935546875,
      "epoch": 0.12963930998431783,
      "grad_norm": 5.853902816772461,
      "kl": 0.63525390625,
      "learning_rate": 4.986683677992362e-07,
      "loss": 0.0814,
      "reward": 0.4525669887661934,
      "reward_std": 0.24432093277573586,
      "rewards/accuracy_reward": 0.07366071827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3789062723517418,
      "step": 434
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1778.9063110351562,
      "epoch": 0.1299380180718393,
      "grad_norm": 3.6149532794952393,
      "kl": 0.56103515625,
      "learning_rate": 4.986413547876409e-07,
      "loss": 0.081,
      "reward": 0.5602678805589676,
      "reward_std": 0.2076813280582428,
      "rewards/accuracy_reward": 0.1830357238650322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3772321566939354,
      "step": 435
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1816.8304443359375,
      "epoch": 0.13023672615936077,
      "grad_norm": 7.4070048332214355,
      "kl": 0.705078125,
      "learning_rate": 4.986140712786211e-07,
      "loss": 0.0973,
      "reward": 0.3945312723517418,
      "reward_std": 0.21280937269330025,
      "rewards/accuracy_reward": 0.03348214412108064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.361049123108387,
      "step": 436
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1725.3148193359375,
      "epoch": 0.13053543424688224,
      "grad_norm": 4.981678485870361,
      "kl": 0.58935546875,
      "learning_rate": 4.985865173018585e-07,
      "loss": 0.1003,
      "reward": 0.385044664144516,
      "reward_std": 0.1809232011437416,
      "rewards/accuracy_reward": 0.013392857741564512,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3716518059372902,
      "step": 437
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1792.0625915527344,
      "epoch": 0.1308341423344037,
      "grad_norm": 5.515299320220947,
      "kl": 0.64111328125,
      "learning_rate": 4.985586928873292e-07,
      "loss": 0.0907,
      "reward": 0.4916294887661934,
      "reward_std": 0.20357535406947136,
      "rewards/accuracy_reward": 0.1383928619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3532366305589676,
      "step": 438
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1792.5179138183594,
      "epoch": 0.13113285042192518,
      "grad_norm": 5.567059516906738,
      "kl": 0.52294921875,
      "learning_rate": 4.985305980653035e-07,
      "loss": 0.0777,
      "reward": 0.4430803880095482,
      "reward_std": 0.2120881974697113,
      "rewards/accuracy_reward": 0.07366071827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.369419664144516,
      "step": 439
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1654.5313110351562,
      "epoch": 0.13143155850944666,
      "grad_norm": 9.795083999633789,
      "kl": 0.51708984375,
      "learning_rate": 4.985022328663456e-07,
      "loss": 0.0993,
      "reward": 0.5228794813156128,
      "reward_std": 0.20075957477092743,
      "rewards/accuracy_reward": 0.1517857222352177,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3710937649011612,
      "step": 440
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1753.2255554199219,
      "epoch": 0.13173026659696813,
      "grad_norm": 4.738426208496094,
      "kl": 0.6240234375,
      "learning_rate": 4.984735973213142e-07,
      "loss": 0.1075,
      "reward": 0.3978794813156128,
      "reward_std": 0.21313538402318954,
      "rewards/accuracy_reward": 0.05580357275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3420759066939354,
      "step": 441
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1736.6562805175781,
      "epoch": 0.13202897468448957,
      "grad_norm": 8.45821475982666,
      "kl": 0.6142578125,
      "learning_rate": 4.98444691461362e-07,
      "loss": 0.0975,
      "reward": 0.4408482387661934,
      "reward_std": 0.164394099265337,
      "rewards/accuracy_reward": 0.07589285937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.364955373108387,
      "step": 442
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1688.6786499023438,
      "epoch": 0.13232768277201104,
      "grad_norm": 15.27545166015625,
      "kl": 0.7021484375,
      "learning_rate": 4.984155153179356e-07,
      "loss": 0.1102,
      "reward": 0.4319196715950966,
      "reward_std": 0.17640457674860954,
      "rewards/accuracy_reward": 0.08035714644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625149011612,
      "step": 443
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1777.5358276367188,
      "epoch": 0.1326263908595325,
      "grad_norm": 4.756255626678467,
      "kl": 0.7490234375,
      "learning_rate": 4.98386068922776e-07,
      "loss": 0.1202,
      "reward": 0.3962053805589676,
      "reward_std": 0.22278672456741333,
      "rewards/accuracy_reward": 0.06026785844005644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3359375149011612,
      "step": 444
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1681.3973693847656,
      "epoch": 0.13292509894705398,
      "grad_norm": 4.871476650238037,
      "kl": 0.54296875,
      "learning_rate": 4.983563523079179e-07,
      "loss": 0.0828,
      "reward": 0.5066964477300644,
      "reward_std": 0.23435239493846893,
      "rewards/accuracy_reward": 0.1517857238650322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3549107238650322,
      "step": 445
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1715.7322082519531,
      "epoch": 0.13322380703457545,
      "grad_norm": 7.476255893707275,
      "kl": 0.6259765625,
      "learning_rate": 4.983263655056899e-07,
      "loss": 0.131,
      "reward": 0.4542410895228386,
      "reward_std": 0.1906590349972248,
      "rewards/accuracy_reward": 0.08482143096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3694196566939354,
      "step": 446
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1725.649658203125,
      "epoch": 0.13352251512209692,
      "grad_norm": 6.511068820953369,
      "kl": 0.63720703125,
      "learning_rate": 4.982961085487149e-07,
      "loss": 0.115,
      "reward": 0.5312500298023224,
      "reward_std": 0.22237225249409676,
      "rewards/accuracy_reward": 0.1785714365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3526785895228386,
      "step": 447
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1748.44873046875,
      "epoch": 0.1338212232096184,
      "grad_norm": 3.9968206882476807,
      "kl": 0.642578125,
      "learning_rate": 4.982655814699092e-07,
      "loss": 0.1125,
      "reward": 0.3683035895228386,
      "reward_std": 0.21424997225403786,
      "rewards/accuracy_reward": 0.029017859371379018,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3392857313156128,
      "step": 448
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1733.4643859863281,
      "epoch": 0.13411993129713987,
      "grad_norm": 4.968216896057129,
      "kl": 0.5908203125,
      "learning_rate": 4.982347843024837e-07,
      "loss": 0.0865,
      "reward": 0.447544664144516,
      "reward_std": 0.2222447618842125,
      "rewards/accuracy_reward": 0.10267857578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3448660895228386,
      "step": 449
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1665.49560546875,
      "epoch": 0.13441863938466134,
      "grad_norm": 13.677940368652344,
      "kl": 0.65478515625,
      "learning_rate": 4.982037170799425e-07,
      "loss": 0.1433,
      "reward": 0.4453125298023224,
      "reward_std": 0.2230478599667549,
      "rewards/accuracy_reward": 0.0870535783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3582589477300644,
      "step": 450
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1713.0982971191406,
      "epoch": 0.1347173474721828,
      "grad_norm": 8.526785850524902,
      "kl": 0.60205078125,
      "learning_rate": 4.981723798360837e-07,
      "loss": 0.1122,
      "reward": 0.443080373108387,
      "reward_std": 0.22774287313222885,
      "rewards/accuracy_reward": 0.07812500325962901,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3649553805589676,
      "step": 451
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1652.6563110351562,
      "epoch": 0.13501605555970428,
      "grad_norm": 10.148331642150879,
      "kl": 0.7939453125,
      "learning_rate": 4.981407726049992e-07,
      "loss": 0.1196,
      "reward": 0.3761160895228386,
      "reward_std": 0.20988183096051216,
      "rewards/accuracy_reward": 0.0290178582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3470982313156128,
      "step": 452
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1689.5603637695312,
      "epoch": 0.13531476364722575,
      "grad_norm": 17.340486526489258,
      "kl": 0.966796875,
      "learning_rate": 4.981088954210745e-07,
      "loss": 0.1205,
      "reward": 0.4380580484867096,
      "reward_std": 0.21062801778316498,
      "rewards/accuracy_reward": 0.10714286309666932,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3309151977300644,
      "step": 453
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1721.3058776855469,
      "epoch": 0.13561347173474722,
      "grad_norm": 5.392794132232666,
      "kl": 0.814453125,
      "learning_rate": 4.980767483189888e-07,
      "loss": 0.1183,
      "reward": 0.4425223395228386,
      "reward_std": 0.25322138145565987,
      "rewards/accuracy_reward": 0.10714286426082253,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3353794738650322,
      "step": 454
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1699.13623046875,
      "epoch": 0.1359121798222687,
      "grad_norm": 3.3406989574432373,
      "kl": 0.609375,
      "learning_rate": 4.98044331333715e-07,
      "loss": 0.1061,
      "reward": 0.5139509215950966,
      "reward_std": 0.22931575030088425,
      "rewards/accuracy_reward": 0.1941964365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3197544738650322,
      "step": 455
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1695.7523193359375,
      "epoch": 0.13621088790979016,
      "grad_norm": 11.295357704162598,
      "kl": 0.7421875,
      "learning_rate": 4.980116445005198e-07,
      "loss": 0.101,
      "reward": 0.4095982387661934,
      "reward_std": 0.2645513229072094,
      "rewards/accuracy_reward": 0.08035714761354029,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3292410895228386,
      "step": 456
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1688.6473999023438,
      "epoch": 0.13650959599731163,
      "grad_norm": 10.917984008789062,
      "kl": 0.5927734375,
      "learning_rate": 4.979786878549631e-07,
      "loss": 0.117,
      "reward": 0.415736623108387,
      "reward_std": 0.26218926906585693,
      "rewards/accuracy_reward": 0.0691964328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3465401902794838,
      "step": 457
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1659.9978332519531,
      "epoch": 0.1368083040848331,
      "grad_norm": 4.539723873138428,
      "kl": 0.6806640625,
      "learning_rate": 4.979454614328985e-07,
      "loss": 0.1273,
      "reward": 0.489955373108387,
      "reward_std": 0.24744581058621407,
      "rewards/accuracy_reward": 0.1674107201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3225446566939354,
      "step": 458
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1744.3438415527344,
      "epoch": 0.13710701217235458,
      "grad_norm": 13.337088584899902,
      "kl": 0.994140625,
      "learning_rate": 4.979119652704731e-07,
      "loss": 0.128,
      "reward": 0.3320312649011612,
      "reward_std": 0.20739859715104103,
      "rewards/accuracy_reward": 0.006696428870782256,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.325334832072258,
      "step": 459
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1585.3639221191406,
      "epoch": 0.13740572025987605,
      "grad_norm": 13.52846622467041,
      "kl": 0.8251953125,
      "learning_rate": 4.978781994041276e-07,
      "loss": 0.1287,
      "reward": 0.4648437723517418,
      "reward_std": 0.22151242196559906,
      "rewards/accuracy_reward": 0.11383929033763707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3510044887661934,
      "step": 460
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1607.2835693359375,
      "epoch": 0.13770442834739752,
      "grad_norm": 8.296908378601074,
      "kl": 0.927734375,
      "learning_rate": 4.978441638705957e-07,
      "loss": 0.1619,
      "reward": 0.558035746216774,
      "reward_std": 0.2525344230234623,
      "rewards/accuracy_reward": 0.2410714398138225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3169642984867096,
      "step": 461
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1613.9799499511719,
      "epoch": 0.138003136434919,
      "grad_norm": 5.828124523162842,
      "kl": 0.7685546875,
      "learning_rate": 4.978098587069049e-07,
      "loss": 0.1357,
      "reward": 0.4319196715950966,
      "reward_std": 0.19434111565351486,
      "rewards/accuracy_reward": 0.08482143213041127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3470982238650322,
      "step": 462
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1638.2634582519531,
      "epoch": 0.13830184452244043,
      "grad_norm": 10.005996704101562,
      "kl": 0.5703125,
      "learning_rate": 4.977752839503758e-07,
      "loss": 0.1392,
      "reward": 0.4235491380095482,
      "reward_std": 0.21815768629312515,
      "rewards/accuracy_reward": 0.10491071944124997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.318638414144516,
      "step": 463
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1568.3549499511719,
      "epoch": 0.1386005526099619,
      "grad_norm": 11.636214256286621,
      "kl": 0.4970703125,
      "learning_rate": 4.977404396386224e-07,
      "loss": 0.1128,
      "reward": 0.4564732387661934,
      "reward_std": 0.2784469500184059,
      "rewards/accuracy_reward": 0.12946429336443543,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3270089402794838,
      "step": 464
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1568.1652526855469,
      "epoch": 0.13889926069748337,
      "grad_norm": 6.901561737060547,
      "kl": 0.517578125,
      "learning_rate": 4.977053258095517e-07,
      "loss": 0.1258,
      "reward": 0.498325914144516,
      "reward_std": 0.254276342689991,
      "rewards/accuracy_reward": 0.18080358020961285,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.317522332072258,
      "step": 465
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1642.2523193359375,
      "epoch": 0.13919796878500484,
      "grad_norm": 66.4725341796875,
      "kl": 1.4423828125,
      "learning_rate": 4.976699425013643e-07,
      "loss": 0.1595,
      "reward": 0.3939732387661934,
      "reward_std": 0.2176070772111416,
      "rewards/accuracy_reward": 0.08928571571595967,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3046875149011612,
      "step": 466
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1612.5692749023438,
      "epoch": 0.13949667687252632,
      "grad_norm": 11.518576622009277,
      "kl": 0.53369140625,
      "learning_rate": 4.976342897525538e-07,
      "loss": 0.0957,
      "reward": 0.3727678805589676,
      "reward_std": 0.21623540669679642,
      "rewards/accuracy_reward": 0.055803572526201606,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3169643059372902,
      "step": 467
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1647.4375915527344,
      "epoch": 0.13979538496004779,
      "grad_norm": 38.45896911621094,
      "kl": 1.1640625,
      "learning_rate": 4.975983676019068e-07,
      "loss": 0.1231,
      "reward": 0.3498884066939354,
      "reward_std": 0.22987286746501923,
      "rewards/accuracy_reward": 0.05133928847499192,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.2963169738650322,
      "step": 468
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1556.4353332519531,
      "epoch": 0.14009409304756926,
      "grad_norm": 24.09501838684082,
      "kl": 1.166015625,
      "learning_rate": 4.97562176088503e-07,
      "loss": 0.1488,
      "reward": 0.3281250149011612,
      "reward_std": 0.22574611008167267,
      "rewards/accuracy_reward": 0.029017858672887087,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2991071566939354,
      "step": 469
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1565.5380249023438,
      "epoch": 0.14039280113509073,
      "grad_norm": 21.94568634033203,
      "kl": 0.5517578125,
      "learning_rate": 4.975257152517154e-07,
      "loss": 0.1194,
      "reward": 0.3839285969734192,
      "reward_std": 0.23307275772094727,
      "rewards/accuracy_reward": 0.06250000465661287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.321428582072258,
      "step": 470
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1493.5625610351562,
      "epoch": 0.1406915092226122,
      "grad_norm": 12.730708122253418,
      "kl": 1.01953125,
      "learning_rate": 4.974889851312097e-07,
      "loss": 0.1514,
      "reward": 0.4006696566939354,
      "reward_std": 0.24730517342686653,
      "rewards/accuracy_reward": 0.08258929220028222,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.318080373108387,
      "step": 471
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1497.8973693847656,
      "epoch": 0.14099021731013367,
      "grad_norm": 17.26862144470215,
      "kl": 0.54248046875,
      "learning_rate": 4.974519857669448e-07,
      "loss": 0.1055,
      "reward": 0.487165205180645,
      "reward_std": 0.24169136211276054,
      "rewards/accuracy_reward": 0.1830357201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3041294738650322,
      "step": 472
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1490.7210693359375,
      "epoch": 0.14128892539765514,
      "grad_norm": 5.557093620300293,
      "kl": 0.8603515625,
      "learning_rate": 4.974147171991722e-07,
      "loss": 0.1112,
      "reward": 0.3515625074505806,
      "reward_std": 0.22987275943160057,
      "rewards/accuracy_reward": 0.037946430034935474,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.3113839402794838,
      "step": 473
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1503.5625610351562,
      "epoch": 0.1415876334851766,
      "grad_norm": 25.97781753540039,
      "kl": 1.28125,
      "learning_rate": 4.973771794684367e-07,
      "loss": 0.1219,
      "reward": 0.3616071566939354,
      "reward_std": 0.21219830960035324,
      "rewards/accuracy_reward": 0.08705357694998384,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.274553582072258,
      "step": 474
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1544.5871276855469,
      "epoch": 0.14188634157269808,
      "grad_norm": 17.041881561279297,
      "kl": 0.998046875,
      "learning_rate": 4.973393726155752e-07,
      "loss": 0.1317,
      "reward": 0.3487723395228386,
      "reward_std": 0.2655651904642582,
      "rewards/accuracy_reward": 0.04687500116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.301897332072258,
      "step": 475
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1534.8348999023438,
      "epoch": 0.14218504966021955,
      "grad_norm": 20.357866287231445,
      "kl": 1.12109375,
      "learning_rate": 4.973012966817184e-07,
      "loss": 0.1337,
      "reward": 0.329241082072258,
      "reward_std": 0.21531317383050919,
      "rewards/accuracy_reward": 0.042410717345774174,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2868303656578064,
      "step": 476
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1507.9486999511719,
      "epoch": 0.14248375774774102,
      "grad_norm": 8.733418464660645,
      "kl": 0.51806640625,
      "learning_rate": 4.972629517082888e-07,
      "loss": 0.0852,
      "reward": 0.4693080633878708,
      "reward_std": 0.24232514947652817,
      "rewards/accuracy_reward": 0.14955357927829027,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.317522332072258,
      "step": 477
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1479.0469360351562,
      "epoch": 0.1427824658352625,
      "grad_norm": 80.32256317138672,
      "kl": 1.44921875,
      "learning_rate": 4.972243377370022e-07,
      "loss": 0.1608,
      "reward": 0.407924123108387,
      "reward_std": 0.2274959348142147,
      "rewards/accuracy_reward": 0.09598214738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3119419738650322,
      "step": 478
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1485.1986999511719,
      "epoch": 0.14308117392278397,
      "grad_norm": 14.516999244689941,
      "kl": 1.1171875,
      "learning_rate": 4.971854548098667e-07,
      "loss": 0.1287,
      "reward": 0.3030134066939354,
      "reward_std": 0.19861429184675217,
      "rewards/accuracy_reward": 0.008928572060540318,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2940848246216774,
      "step": 479
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1449.9375915527344,
      "epoch": 0.14337988201030544,
      "grad_norm": 25.714197158813477,
      "kl": 1.138671875,
      "learning_rate": 4.971463029691832e-07,
      "loss": 0.149,
      "reward": 0.4045759066939354,
      "reward_std": 0.22242727130651474,
      "rewards/accuracy_reward": 0.10491071757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2996651902794838,
      "step": 480
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1539.18310546875,
      "epoch": 0.1436785900978269,
      "grad_norm": 56.93959426879883,
      "kl": 0.9384765625,
      "learning_rate": 4.97106882257545e-07,
      "loss": 0.1047,
      "reward": 0.3939732313156128,
      "reward_std": 0.21351967379450798,
      "rewards/accuracy_reward": 0.08705357555299997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3069196566939354,
      "step": 481
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1378.513427734375,
      "epoch": 0.14397729818534838,
      "grad_norm": 14.244688034057617,
      "kl": 1.12109375,
      "learning_rate": 4.970671927178378e-07,
      "loss": 0.1364,
      "reward": 0.4648437574505806,
      "reward_std": 0.22593215480446815,
      "rewards/accuracy_reward": 0.1674107201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.297433041036129,
      "step": 482
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1489.7545166015625,
      "epoch": 0.14427600627286985,
      "grad_norm": 17.129865646362305,
      "kl": 1.19921875,
      "learning_rate": 4.970272343932403e-07,
      "loss": 0.1345,
      "reward": 0.3431919813156128,
      "reward_std": 0.21285830065608025,
      "rewards/accuracy_reward": 0.058035716181620955,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2851562649011612,
      "step": 483
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1426.2032165527344,
      "epoch": 0.14457471436039132,
      "grad_norm": 23.218088150024414,
      "kl": 0.94140625,
      "learning_rate": 4.96987007327223e-07,
      "loss": 0.1078,
      "reward": 0.440290205180645,
      "reward_std": 0.2618703097105026,
      "rewards/accuracy_reward": 0.1450892947614193,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2952008992433548,
      "step": 484
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1483.5067443847656,
      "epoch": 0.14487342244791276,
      "grad_norm": 39.45608901977539,
      "kl": 1.1689453125,
      "learning_rate": 4.96946511563549e-07,
      "loss": 0.0991,
      "reward": 0.368861623108387,
      "reward_std": 0.2229001373052597,
      "rewards/accuracy_reward": 0.06919643143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2996651902794838,
      "step": 485
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1526.5335693359375,
      "epoch": 0.14517213053543424,
      "grad_norm": 47.41693115234375,
      "kl": 1.34375,
      "learning_rate": 4.969057471462739e-07,
      "loss": 0.1216,
      "reward": 0.286830373108387,
      "reward_std": 0.20712244510650635,
      "rewards/accuracy_reward": 0.011160715017467737,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2756696566939354,
      "step": 486
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1455.1317749023438,
      "epoch": 0.1454708386229557,
      "grad_norm": 22.495946884155273,
      "kl": 1.2294921875,
      "learning_rate": 4.968647141197452e-07,
      "loss": 0.16,
      "reward": 0.3275669738650322,
      "reward_std": 0.19334674626588821,
      "rewards/accuracy_reward": 0.046875003492459655,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2806919738650322,
      "step": 487
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1488.3616943359375,
      "epoch": 0.14576954671047718,
      "grad_norm": 10.561921119689941,
      "kl": 0.82421875,
      "learning_rate": 4.968234125286029e-07,
      "loss": 0.1156,
      "reward": 0.3716517984867096,
      "reward_std": 0.18082809075713158,
      "rewards/accuracy_reward": 0.042410716181620955,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.329241082072258,
      "step": 488
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1505.5224304199219,
      "epoch": 0.14606825479799865,
      "grad_norm": 12.997093200683594,
      "kl": 0.8125,
      "learning_rate": 4.96781842417779e-07,
      "loss": 0.0919,
      "reward": 0.3677455484867096,
      "reward_std": 0.22588007152080536,
      "rewards/accuracy_reward": 0.06919643096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.298549123108387,
      "step": 489
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1422.8973999023438,
      "epoch": 0.14636696288552012,
      "grad_norm": 29.225215911865234,
      "kl": 0.5908203125,
      "learning_rate": 4.967400038324979e-07,
      "loss": 0.1163,
      "reward": 0.3286830484867096,
      "reward_std": 0.19791864231228828,
      "rewards/accuracy_reward": 0.013392857508733869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3152901902794838,
      "step": 490
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1534.2076416015625,
      "epoch": 0.1466656709730416,
      "grad_norm": 20.25705337524414,
      "kl": 0.7294921875,
      "learning_rate": 4.966978968182757e-07,
      "loss": 0.1147,
      "reward": 0.3052455484867096,
      "reward_std": 0.21112031117081642,
      "rewards/accuracy_reward": 0.04910714505240321,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.2539062686264515,
      "step": 491
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1450.1161499023438,
      "epoch": 0.14696437906056306,
      "grad_norm": 21.163368225097656,
      "kl": 0.62109375,
      "learning_rate": 4.966555214209207e-07,
      "loss": 0.1054,
      "reward": 0.4787946715950966,
      "reward_std": 0.24121013283729553,
      "rewards/accuracy_reward": 0.1785714402794838,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3002232275903225,
      "step": 492
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1477.8661193847656,
      "epoch": 0.14726308714808453,
      "grad_norm": 35.966796875,
      "kl": 1.251953125,
      "learning_rate": 4.966128776865333e-07,
      "loss": 0.0999,
      "reward": 0.3632812649011612,
      "reward_std": 0.20130787789821625,
      "rewards/accuracy_reward": 0.08705357508733869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2762276828289032,
      "step": 493
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1399.7879943847656,
      "epoch": 0.147561795235606,
      "grad_norm": 28.23116111755371,
      "kl": 0.736328125,
      "learning_rate": 4.965699656615056e-07,
      "loss": 0.1284,
      "reward": 0.4363839477300644,
      "reward_std": 0.22000691294670105,
      "rewards/accuracy_reward": 0.12946429220028222,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3069196566939354,
      "step": 494
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1441.05810546875,
      "epoch": 0.14786050332312747,
      "grad_norm": 12.888809204101562,
      "kl": 1.13671875,
      "learning_rate": 4.965267853925216e-07,
      "loss": 0.1038,
      "reward": 0.3186384066939354,
      "reward_std": 0.22457966953516006,
      "rewards/accuracy_reward": 0.033482144586741924,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2851562574505806,
      "step": 495
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1453.5157165527344,
      "epoch": 0.14815921141064894,
      "grad_norm": 28.30711555480957,
      "kl": 1.3046875,
      "learning_rate": 4.964833369265575e-07,
      "loss": 0.128,
      "reward": 0.3331473395228386,
      "reward_std": 0.20996088534593582,
      "rewards/accuracy_reward": 0.06250000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2706473283469677,
      "step": 496
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1473.6942749023438,
      "epoch": 0.14845791949817042,
      "grad_norm": 15.71290111541748,
      "kl": 1.07421875,
      "learning_rate": 4.964396203108806e-07,
      "loss": 0.1029,
      "reward": 0.449776791036129,
      "reward_std": 0.2157180830836296,
      "rewards/accuracy_reward": 0.16517857555299997,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.2823660783469677,
      "step": 497
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1457.9063110351562,
      "epoch": 0.1487566275856919,
      "grad_norm": 26.50455665588379,
      "kl": 1.439453125,
      "learning_rate": 4.963956355930506e-07,
      "loss": 0.1227,
      "reward": 0.3309151902794838,
      "reward_std": 0.2338428609073162,
      "rewards/accuracy_reward": 0.03794643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2929687574505806,
      "step": 498
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1450.2545471191406,
      "epoch": 0.14905533567321336,
      "grad_norm": 15.583983421325684,
      "kl": 1.0234375,
      "learning_rate": 4.963513828209184e-07,
      "loss": 0.1333,
      "reward": 0.3627232313156128,
      "reward_std": 0.20912770181894302,
      "rewards/accuracy_reward": 0.05357143096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.309151791036129,
      "step": 499
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1392.3393249511719,
      "epoch": 0.14935404376073483,
      "grad_norm": 15.106719970703125,
      "kl": 1.0703125,
      "learning_rate": 4.963068620426269e-07,
      "loss": 0.0984,
      "reward": 0.2873884066939354,
      "reward_std": 0.1699320189654827,
      "rewards/accuracy_reward": 0.0022321429569274187,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.2829241193830967,
      "step": 500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1476.6139221191406,
      "epoch": 0.1496527518482563,
      "grad_norm": 20.554685592651367,
      "kl": 1.041015625,
      "learning_rate": 4.962620733066099e-07,
      "loss": 0.1098,
      "reward": 0.3404018059372902,
      "reward_std": 0.18493600189685822,
      "rewards/accuracy_reward": 0.0647321455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2756696529686451,
      "step": 501
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1431.8750610351562,
      "epoch": 0.14995145993577777,
      "grad_norm": 33.422760009765625,
      "kl": 0.646484375,
      "learning_rate": 4.962170166615936e-07,
      "loss": 0.0978,
      "reward": 0.3281250074505806,
      "reward_std": 0.22296306490898132,
      "rewards/accuracy_reward": 0.026785714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3013392984867096,
      "step": 502
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1513.4710388183594,
      "epoch": 0.15025016802329924,
      "grad_norm": 16.91933822631836,
      "kl": 0.8740234375,
      "learning_rate": 4.96171692156595e-07,
      "loss": 0.1201,
      "reward": 0.3867187723517418,
      "reward_std": 0.22634423896670341,
      "rewards/accuracy_reward": 0.07812500279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3085937649011612,
      "step": 503
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1416.993408203125,
      "epoch": 0.1505488761108207,
      "grad_norm": 19.677053451538086,
      "kl": 0.68798828125,
      "learning_rate": 4.961260998409227e-07,
      "loss": 0.1043,
      "reward": 0.3733259066939354,
      "reward_std": 0.21079107746481895,
      "rewards/accuracy_reward": 0.0669642873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3063616156578064,
      "step": 504
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1319.5335388183594,
      "epoch": 0.15084758419834218,
      "grad_norm": 32.37833023071289,
      "kl": 0.69921875,
      "learning_rate": 4.960802397641768e-07,
      "loss": 0.0922,
      "reward": 0.4547991305589676,
      "reward_std": 0.21234611794352531,
      "rewards/accuracy_reward": 0.10937500232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.345424123108387,
      "step": 505
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1444.8683776855469,
      "epoch": 0.15114629228586363,
      "grad_norm": 20.803768157958984,
      "kl": 1.267578125,
      "learning_rate": 4.960341119762484e-07,
      "loss": 0.1059,
      "reward": 0.3459821566939354,
      "reward_std": 0.1845408733934164,
      "rewards/accuracy_reward": 0.049107145285233855,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2968750149011612,
      "step": 506
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1336.2634582519531,
      "epoch": 0.1514450003733851,
      "grad_norm": 8.381172180175781,
      "kl": 1.0302734375,
      "learning_rate": 4.959877165273202e-07,
      "loss": 0.0995,
      "reward": 0.4040178805589676,
      "reward_std": 0.17987819015979767,
      "rewards/accuracy_reward": 0.08482143143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3191964402794838,
      "step": 507
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1408.6786193847656,
      "epoch": 0.15174370846090657,
      "grad_norm": 22.949893951416016,
      "kl": 0.9365234375,
      "learning_rate": 4.959410534678657e-07,
      "loss": 0.1031,
      "reward": 0.517299123108387,
      "reward_std": 0.2119394987821579,
      "rewards/accuracy_reward": 0.1964285816065967,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3208705559372902,
      "step": 508
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1461.1340026855469,
      "epoch": 0.15204241654842804,
      "grad_norm": 22.094566345214844,
      "kl": 1.0107421875,
      "learning_rate": 4.958941228486499e-07,
      "loss": 0.0809,
      "reward": 0.3465401977300644,
      "reward_std": 0.2078564167022705,
      "rewards/accuracy_reward": 0.05580357415601611,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.290736623108387,
      "step": 509
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1457.2813110351562,
      "epoch": 0.1523411246359495,
      "grad_norm": 19.875404357910156,
      "kl": 1.435546875,
      "learning_rate": 4.958469247207286e-07,
      "loss": 0.1254,
      "reward": 0.3219866156578064,
      "reward_std": 0.1806906796991825,
      "rewards/accuracy_reward": 0.01785714365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3041294738650322,
      "step": 510
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1412.7255554199219,
      "epoch": 0.15263983272347098,
      "grad_norm": 28.24956512451172,
      "kl": 1.41015625,
      "learning_rate": 4.957994591354489e-07,
      "loss": 0.1272,
      "reward": 0.5200893133878708,
      "reward_std": 0.19330596551299095,
      "rewards/accuracy_reward": 0.2098214402794838,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.310267873108387,
      "step": 511
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1407.7679138183594,
      "epoch": 0.15293854081099245,
      "grad_norm": 990.5584716796875,
      "kl": 1.849609375,
      "learning_rate": 4.957517261444485e-07,
      "loss": 0.1465,
      "reward": 0.4665178880095482,
      "reward_std": 0.24038105085492134,
      "rewards/accuracy_reward": 0.1808035783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2857142984867096,
      "step": 512
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1433.0223693847656,
      "epoch": 0.15323724889851392,
      "grad_norm": 30.94546890258789,
      "kl": 1.681640625,
      "learning_rate": 4.957037257996563e-07,
      "loss": 0.1415,
      "reward": 0.3465401902794838,
      "reward_std": 0.19393234699964523,
      "rewards/accuracy_reward": 0.04687500232830644,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.2974330484867096,
      "step": 513
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1350.0893249511719,
      "epoch": 0.1535359569860354,
      "grad_norm": 10.932440757751465,
      "kl": 1.296875,
      "learning_rate": 4.956554581532922e-07,
      "loss": 0.1203,
      "reward": 0.3671875149011612,
      "reward_std": 0.19484785944223404,
      "rewards/accuracy_reward": 0.04910714668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.318080373108387,
      "step": 514
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1375.40185546875,
      "epoch": 0.15383466507355686,
      "grad_norm": 51.63142776489258,
      "kl": 1.654296875,
      "learning_rate": 4.956069232578661e-07,
      "loss": 0.127,
      "reward": 0.317522332072258,
      "reward_std": 0.19570493698120117,
      "rewards/accuracy_reward": 0.0223214291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2952009066939354,
      "step": 515
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1448.38623046875,
      "epoch": 0.15413337316107834,
      "grad_norm": 32.530250549316406,
      "kl": 1.572265625,
      "learning_rate": 4.955581211661798e-07,
      "loss": 0.1306,
      "reward": 0.3080357313156128,
      "reward_std": 0.16375143639743328,
      "rewards/accuracy_reward": 0.004464285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3035714477300644,
      "step": 516
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1302.4665832519531,
      "epoch": 0.1544320812485998,
      "grad_norm": 22.07048225402832,
      "kl": 1.30078125,
      "learning_rate": 4.955090519313249e-07,
      "loss": 0.1116,
      "reward": 0.4453125149011612,
      "reward_std": 0.2152816653251648,
      "rewards/accuracy_reward": 0.11830357555299997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3270089402794838,
      "step": 517
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1405.3014221191406,
      "epoch": 0.15473078933612128,
      "grad_norm": 9.912904739379883,
      "kl": 1.556640625,
      "learning_rate": 4.95459715606684e-07,
      "loss": 0.1221,
      "reward": 0.3158482313156128,
      "reward_std": 0.19546522945165634,
      "rewards/accuracy_reward": 0.058035717345774174,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2578125074505806,
      "step": 518
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1340.5246276855469,
      "epoch": 0.15502949742364275,
      "grad_norm": 7.609091758728027,
      "kl": 1.154296875,
      "learning_rate": 4.954101122459301e-07,
      "loss": 0.121,
      "reward": 0.4235491380095482,
      "reward_std": 0.23425734415650368,
      "rewards/accuracy_reward": 0.11160714738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3119419813156128,
      "step": 519
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1503.0536499023438,
      "epoch": 0.15532820551116422,
      "grad_norm": 22.38677978515625,
      "kl": 1.197265625,
      "learning_rate": 4.953602419030269e-07,
      "loss": 0.1032,
      "reward": 0.3666294813156128,
      "reward_std": 0.22174959629774094,
      "rewards/accuracy_reward": 0.0803571455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.286272332072258,
      "step": 520
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1454.6674499511719,
      "epoch": 0.1556269135986857,
      "grad_norm": 27.314897537231445,
      "kl": 1.3515625,
      "learning_rate": 4.953101046322284e-07,
      "loss": 0.1391,
      "reward": 0.3443080559372902,
      "reward_std": 0.2016202248632908,
      "rewards/accuracy_reward": 0.051339289639145136,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.2907366156578064,
      "step": 521
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1353.0625915527344,
      "epoch": 0.15592562168620716,
      "grad_norm": 43.14972686767578,
      "kl": 0.986328125,
      "learning_rate": 4.95259700488079e-07,
      "loss": 0.1288,
      "reward": 0.3984375298023224,
      "reward_std": 0.19918563589453697,
      "rewards/accuracy_reward": 0.08928571944124997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3091517984867096,
      "step": 522
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1430.7500610351562,
      "epoch": 0.15622432977372863,
      "grad_norm": 23.774337768554688,
      "kl": 1.201171875,
      "learning_rate": 4.952090295254136e-07,
      "loss": 0.1175,
      "reward": 0.428013414144516,
      "reward_std": 0.21897517517209053,
      "rewards/accuracy_reward": 0.1071428619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3208705559372902,
      "step": 523
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1433.8326721191406,
      "epoch": 0.1565230378612501,
      "grad_norm": 26.849700927734375,
      "kl": 1.06640625,
      "learning_rate": 4.95158091799357e-07,
      "loss": 0.1182,
      "reward": 0.428571455180645,
      "reward_std": 0.19215301051735878,
      "rewards/accuracy_reward": 0.11607143469154835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3125000149011612,
      "step": 524
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1522.0290832519531,
      "epoch": 0.15682174594877157,
      "grad_norm": 28.29598045349121,
      "kl": 1.9296875,
      "learning_rate": 4.951068873653246e-07,
      "loss": 0.148,
      "reward": 0.4095982313156128,
      "reward_std": 0.21544695273041725,
      "rewards/accuracy_reward": 0.11160714738070965,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.2957589402794838,
      "step": 525
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1546.1942749023438,
      "epoch": 0.15712045403629304,
      "grad_norm": 37.72453308105469,
      "kl": 2.142578125,
      "learning_rate": 4.95055416279022e-07,
      "loss": 0.1554,
      "reward": 0.3404017984867096,
      "reward_std": 0.19783299416303635,
      "rewards/accuracy_reward": 0.0491071455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2912946566939354,
      "step": 526
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1500.4554443359375,
      "epoch": 0.15741916212381452,
      "grad_norm": 22.83420753479004,
      "kl": 1.9140625,
      "learning_rate": 4.950036785964443e-07,
      "loss": 0.1622,
      "reward": 0.3482143059372902,
      "reward_std": 0.19142743572592735,
      "rewards/accuracy_reward": 0.05133928777649999,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2968750149011612,
      "step": 527
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1438.2054138183594,
      "epoch": 0.15771787021133596,
      "grad_norm": 21.790014266967773,
      "kl": 1.798828125,
      "learning_rate": 4.949516743738772e-07,
      "loss": 0.1524,
      "reward": 0.4815848469734192,
      "reward_std": 0.24426716193556786,
      "rewards/accuracy_reward": 0.16071429196745157,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3208705484867096,
      "step": 528
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1436.12060546875,
      "epoch": 0.15801657829885743,
      "grad_norm": 11.602551460266113,
      "kl": 1.78125,
      "learning_rate": 4.948994036678964e-07,
      "loss": 0.1906,
      "reward": 0.3867187649011612,
      "reward_std": 0.2183646634221077,
      "rewards/accuracy_reward": 0.08482143096625805,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.2996651902794838,
      "step": 529
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1489.7902526855469,
      "epoch": 0.1583152863863789,
      "grad_norm": 22.823286056518555,
      "kl": 1.677734375,
      "learning_rate": 4.94846866535367e-07,
      "loss": 0.1272,
      "reward": 0.3459821566939354,
      "reward_std": 0.17157990112900734,
      "rewards/accuracy_reward": 0.04687500232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2991071492433548,
      "step": 530
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1460.2857666015625,
      "epoch": 0.15861399447390037,
      "grad_norm": 15.884933471679688,
      "kl": 1.68359375,
      "learning_rate": 4.947940630334444e-07,
      "loss": 0.1715,
      "reward": 0.3945312574505806,
      "reward_std": 0.17972556129097939,
      "rewards/accuracy_reward": 0.07812500419095159,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3164062574505806,
      "step": 531
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1517.0826416015625,
      "epoch": 0.15891270256142184,
      "grad_norm": 11.859461784362793,
      "kl": 2.005859375,
      "learning_rate": 4.947409932195736e-07,
      "loss": 0.1911,
      "reward": 0.3214285895228386,
      "reward_std": 0.17129936441779137,
      "rewards/accuracy_reward": 0.03794643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2834821566939354,
      "step": 532
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1460.2590026855469,
      "epoch": 0.1592114106489433,
      "grad_norm": 25.551998138427734,
      "kl": 1.923828125,
      "learning_rate": 4.946876571514895e-07,
      "loss": 0.1592,
      "reward": 0.3945312649011612,
      "reward_std": 0.18359490856528282,
      "rewards/accuracy_reward": 0.10044643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.294084832072258,
      "step": 533
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1549.4286193847656,
      "epoch": 0.15951011873646478,
      "grad_norm": 22.828031539916992,
      "kl": 1.845703125,
      "learning_rate": 4.946340548872164e-07,
      "loss": 0.1384,
      "reward": 0.3510044738650322,
      "reward_std": 0.18982669338583946,
      "rewards/accuracy_reward": 0.07589286006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2751116119325161,
      "step": 534
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1493.1451416015625,
      "epoch": 0.15980882682398626,
      "grad_norm": 20.755876541137695,
      "kl": 1.3623046875,
      "learning_rate": 4.945801864850681e-07,
      "loss": 0.1415,
      "reward": 0.337611623108387,
      "reward_std": 0.20516351610422134,
      "rewards/accuracy_reward": 0.029017859371379018,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.306361623108387,
      "step": 535
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1485.9063415527344,
      "epoch": 0.16010753491150773,
      "grad_norm": 53.59226989746094,
      "kl": 1.203125,
      "learning_rate": 4.945260520036484e-07,
      "loss": 0.1705,
      "reward": 0.5251116305589676,
      "reward_std": 0.22791927307844162,
      "rewards/accuracy_reward": 0.20535714738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3197544813156128,
      "step": 536
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1546.279052734375,
      "epoch": 0.1604062429990292,
      "grad_norm": 32.348567962646484,
      "kl": 1.568359375,
      "learning_rate": 4.944716515018504e-07,
      "loss": 0.1693,
      "reward": 0.3705357313156128,
      "reward_std": 0.17491189762949944,
      "rewards/accuracy_reward": 0.08482143399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.285714291036129,
      "step": 537
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1533.1920166015625,
      "epoch": 0.16070495108655067,
      "grad_norm": 41.89881896972656,
      "kl": 1.3671875,
      "learning_rate": 4.944169850388562e-07,
      "loss": 0.1411,
      "reward": 0.341517873108387,
      "reward_std": 0.20202600583434105,
      "rewards/accuracy_reward": 0.05357143213041127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2879464402794838,
      "step": 538
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1529.1295166015625,
      "epoch": 0.16100365917407214,
      "grad_norm": 31.795007705688477,
      "kl": 1.5390625,
      "learning_rate": 4.943620526741378e-07,
      "loss": 0.158,
      "reward": 0.4492187649011612,
      "reward_std": 0.2079201228916645,
      "rewards/accuracy_reward": 0.1316964311990887,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3175223395228386,
      "step": 539
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1533.3929138183594,
      "epoch": 0.1613023672615936,
      "grad_norm": 16.42098045349121,
      "kl": 1.88671875,
      "learning_rate": 4.94306854467456e-07,
      "loss": 0.1687,
      "reward": 0.3677455559372902,
      "reward_std": 0.20785117149353027,
      "rewards/accuracy_reward": 0.06250000116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3052455484867096,
      "step": 540
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1448.6965026855469,
      "epoch": 0.16160107534911508,
      "grad_norm": 15.577332496643066,
      "kl": 1.65625,
      "learning_rate": 4.942513904788613e-07,
      "loss": 0.1797,
      "reward": 0.3560268059372902,
      "reward_std": 0.1952895149588585,
      "rewards/accuracy_reward": 0.03571428847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3203125149011612,
      "step": 541
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1505.7210388183594,
      "epoch": 0.16189978343663655,
      "grad_norm": 16.7977294921875,
      "kl": 1.857421875,
      "learning_rate": 4.94195660768693e-07,
      "loss": 0.1521,
      "reward": 0.3783482238650322,
      "reward_std": 0.1823890469968319,
      "rewards/accuracy_reward": 0.0669642873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3113839477300644,
      "step": 542
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1512.1429443359375,
      "epoch": 0.16219849152415802,
      "grad_norm": 25.266433715820312,
      "kl": 2.20703125,
      "learning_rate": 4.941396653975795e-07,
      "loss": 0.1702,
      "reward": 0.314174123108387,
      "reward_std": 0.16804957389831543,
      "rewards/accuracy_reward": 0.04241071501746774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2717634066939354,
      "step": 543
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1451.107177734375,
      "epoch": 0.1624971996116795,
      "grad_norm": 11.879313468933105,
      "kl": 1.896484375,
      "learning_rate": 4.940834044264383e-07,
      "loss": 0.152,
      "reward": 0.466517873108387,
      "reward_std": 0.20944871753454208,
      "rewards/accuracy_reward": 0.1473214402794838,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3191964402794838,
      "step": 544
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1511.9040832519531,
      "epoch": 0.16279590769920096,
      "grad_norm": 10.97879409790039,
      "kl": 1.787109375,
      "learning_rate": 4.940268779164758e-07,
      "loss": 0.1443,
      "reward": 0.334263414144516,
      "reward_std": 0.17915084585547447,
      "rewards/accuracy_reward": 0.053571430034935474,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2806919738650322,
      "step": 545
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1536.2300109863281,
      "epoch": 0.16309461578672244,
      "grad_norm": 15.401771545410156,
      "kl": 2.119140625,
      "learning_rate": 4.939700859291875e-07,
      "loss": 0.209,
      "reward": 0.3292410895228386,
      "reward_std": 0.18664926663041115,
      "rewards/accuracy_reward": 0.022321430034935474,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.306919664144516,
      "step": 546
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1569.24560546875,
      "epoch": 0.1633933238742439,
      "grad_norm": 11.521446228027344,
      "kl": 1.966796875,
      "learning_rate": 4.939130285263572e-07,
      "loss": 0.1948,
      "reward": 0.3699776977300644,
      "reward_std": 0.2225136049091816,
      "rewards/accuracy_reward": 0.07589286006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.294084832072258,
      "step": 547
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1644.4263916015625,
      "epoch": 0.16369203196176538,
      "grad_norm": 15.928544998168945,
      "kl": 2.09765625,
      "learning_rate": 4.938557057700579e-07,
      "loss": 0.1666,
      "reward": 0.3125000111758709,
      "reward_std": 0.1819944977760315,
      "rewards/accuracy_reward": 0.0491071455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2633928693830967,
      "step": 548
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1513.60498046875,
      "epoch": 0.16399074004928682,
      "grad_norm": 26.027103424072266,
      "kl": 1.626953125,
      "learning_rate": 4.937981177226511e-07,
      "loss": 0.1506,
      "reward": 0.4308035895228386,
      "reward_std": 0.2502877898514271,
      "rewards/accuracy_reward": 0.10714286286383867,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3236607238650322,
      "step": 549
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1564.0491943359375,
      "epoch": 0.1642894481368083,
      "grad_norm": 14.689818382263184,
      "kl": 1.7265625,
      "learning_rate": 4.937402644467871e-07,
      "loss": 0.1784,
      "reward": 0.3878348395228386,
      "reward_std": 0.18370545655488968,
      "rewards/accuracy_reward": 0.0870535746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3007812649011612,
      "step": 550
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1587.7277526855469,
      "epoch": 0.16458815622432976,
      "grad_norm": 27.457515716552734,
      "kl": 1.79296875,
      "learning_rate": 4.936821460054045e-07,
      "loss": 0.1951,
      "reward": 0.4056919738650322,
      "reward_std": 0.16137132421135902,
      "rewards/accuracy_reward": 0.1093750037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2963169738650322,
      "step": 551
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1511.7902526855469,
      "epoch": 0.16488686431185123,
      "grad_norm": 13.077816009521484,
      "kl": 1.66015625,
      "learning_rate": 4.936237624617304e-07,
      "loss": 0.1805,
      "reward": 0.423549123108387,
      "reward_std": 0.1953897513449192,
      "rewards/accuracy_reward": 0.1026785746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3208705484867096,
      "step": 552
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1540.5268249511719,
      "epoch": 0.1651855723993727,
      "grad_norm": 9.963757514953613,
      "kl": 1.8359375,
      "learning_rate": 4.935651138792804e-07,
      "loss": 0.192,
      "reward": 0.3694196566939354,
      "reward_std": 0.1632193885743618,
      "rewards/accuracy_reward": 0.07142857578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2979910895228386,
      "step": 553
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1568.3036193847656,
      "epoch": 0.16548428048689418,
      "grad_norm": 32.14925003051758,
      "kl": 1.638671875,
      "learning_rate": 4.935062003218585e-07,
      "loss": 0.1768,
      "reward": 0.3660714402794838,
      "reward_std": 0.22965658456087112,
      "rewards/accuracy_reward": 0.042410716880112886,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.3214285895228386,
      "step": 554
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1624.5581359863281,
      "epoch": 0.16578298857441565,
      "grad_norm": 15.839299201965332,
      "kl": 2.44140625,
      "learning_rate": 4.934470218535569e-07,
      "loss": 0.1964,
      "reward": 0.4101562649011612,
      "reward_std": 0.1923549398779869,
      "rewards/accuracy_reward": 0.12723215040750802,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2829241156578064,
      "step": 555
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1590.352783203125,
      "epoch": 0.16608169666193712,
      "grad_norm": 10.260693550109863,
      "kl": 2.029296875,
      "learning_rate": 4.933875785387558e-07,
      "loss": 0.2125,
      "reward": 0.353236623108387,
      "reward_std": 0.2079397663474083,
      "rewards/accuracy_reward": 0.06696428917348385,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.2840401902794838,
      "step": 556
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1621.4598999023438,
      "epoch": 0.1663804047494586,
      "grad_norm": 13.340351104736328,
      "kl": 2.34765625,
      "learning_rate": 4.933278704421239e-07,
      "loss": 0.2067,
      "reward": 0.3225446566939354,
      "reward_std": 0.20206910744309425,
      "rewards/accuracy_reward": 0.05580357275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.266741082072258,
      "step": 557
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1574.0134582519531,
      "epoch": 0.16667911283698006,
      "grad_norm": 8.78235149383545,
      "kl": 2.28125,
      "learning_rate": 4.932678976286176e-07,
      "loss": 0.216,
      "reward": 0.321986623108387,
      "reward_std": 0.20052982866764069,
      "rewards/accuracy_reward": 0.022321430034935474,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2996651902794838,
      "step": 558
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1611.2545471191406,
      "epoch": 0.16697782092450153,
      "grad_norm": 10.4319486618042,
      "kl": 2.61328125,
      "learning_rate": 4.932076601634814e-07,
      "loss": 0.2176,
      "reward": 0.2912946566939354,
      "reward_std": 0.1844063363969326,
      "rewards/accuracy_reward": 0.011160715017467737,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2801339477300644,
      "step": 559
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1638.82373046875,
      "epoch": 0.167276529012023,
      "grad_norm": 32.21158218383789,
      "kl": 2.63671875,
      "learning_rate": 4.931471581122479e-07,
      "loss": 0.2147,
      "reward": 0.3593750149011612,
      "reward_std": 0.23315341398119926,
      "rewards/accuracy_reward": 0.07589286006987095,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.2812500074505806,
      "step": 560
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1587.6473999023438,
      "epoch": 0.16757523709954447,
      "grad_norm": 13.846587181091309,
      "kl": 2.36328125,
      "learning_rate": 4.930863915407374e-07,
      "loss": 0.2594,
      "reward": 0.353236623108387,
      "reward_std": 0.2137494459748268,
      "rewards/accuracy_reward": 0.058035718742758036,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2952009066939354,
      "step": 561
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1594.10498046875,
      "epoch": 0.16787394518706594,
      "grad_norm": 15.19684886932373,
      "kl": 1.755859375,
      "learning_rate": 4.930253605150579e-07,
      "loss": 0.1738,
      "reward": 0.318638414144516,
      "reward_std": 0.2119026742875576,
      "rewards/accuracy_reward": 0.022321429336443543,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2963169738650322,
      "step": 562
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1627.1652526855469,
      "epoch": 0.1681726532745874,
      "grad_norm": 21.628976821899414,
      "kl": 2.6171875,
      "learning_rate": 4.929640651016053e-07,
      "loss": 0.2196,
      "reward": 0.3722098395228386,
      "reward_std": 0.2102942168712616,
      "rewards/accuracy_reward": 0.1004464328289032,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.2695312649011612,
      "step": 563
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1618.1697387695312,
      "epoch": 0.16847136136210888,
      "grad_norm": 10.55736255645752,
      "kl": 2.32421875,
      "learning_rate": 4.929025053670627e-07,
      "loss": 0.2267,
      "reward": 0.3381696566939354,
      "reward_std": 0.16556742787361145,
      "rewards/accuracy_reward": 0.04017857206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.297991082072258,
      "step": 564
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1565.60498046875,
      "epoch": 0.16877006944963036,
      "grad_norm": 6.295408248901367,
      "kl": 2.40625,
      "learning_rate": 4.928406813784015e-07,
      "loss": 0.2379,
      "reward": 0.3850446566939354,
      "reward_std": 0.19622733816504478,
      "rewards/accuracy_reward": 0.0870535746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.297991082072258,
      "step": 565
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1597.0022888183594,
      "epoch": 0.16906877753715183,
      "grad_norm": 20.413482666015625,
      "kl": 1.958984375,
      "learning_rate": 4.927785932028799e-07,
      "loss": 0.2254,
      "reward": 0.3504464402794838,
      "reward_std": 0.20354238897562027,
      "rewards/accuracy_reward": 0.04687500209547579,
      "rewards/format_reward": 0.004464285913854837,
      "rewards/tag_count_reward": 0.2991071566939354,
      "step": 566
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1602.435302734375,
      "epoch": 0.1693674856246733,
      "grad_norm": 10.642688751220703,
      "kl": 2.298828125,
      "learning_rate": 4.927162409080436e-07,
      "loss": 0.2131,
      "reward": 0.3493303656578064,
      "reward_std": 0.22611487284302711,
      "rewards/accuracy_reward": 0.042410716181620955,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.3046875223517418,
      "step": 567
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1595.0491638183594,
      "epoch": 0.16966619371219477,
      "grad_norm": 17.878570556640625,
      "kl": 2.2890625,
      "learning_rate": 4.92653624561726e-07,
      "loss": 0.2408,
      "reward": 0.3582589328289032,
      "reward_std": 0.19988222047686577,
      "rewards/accuracy_reward": 0.06919643026776612,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.2868303656578064,
      "step": 568
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1564.5067443847656,
      "epoch": 0.16996490179971624,
      "grad_norm": 8.563328742980957,
      "kl": 2.42578125,
      "learning_rate": 4.925907442320475e-07,
      "loss": 0.2268,
      "reward": 0.364955373108387,
      "reward_std": 0.2186395600438118,
      "rewards/accuracy_reward": 0.058035717345774174,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3069196492433548,
      "step": 569
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1514.6339721679688,
      "epoch": 0.1702636098872377,
      "grad_norm": 14.028228759765625,
      "kl": 2.359375,
      "learning_rate": 4.925275999874156e-07,
      "loss": 0.2459,
      "reward": 0.4676339477300644,
      "reward_std": 0.22803118452429771,
      "rewards/accuracy_reward": 0.1607142947614193,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3069196566939354,
      "step": 570
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1577.91748046875,
      "epoch": 0.17056231797475915,
      "grad_norm": 10.996620178222656,
      "kl": 2.82421875,
      "learning_rate": 4.924641918965254e-07,
      "loss": 0.2594,
      "reward": 0.3091517984867096,
      "reward_std": 0.19352185726165771,
      "rewards/accuracy_reward": 0.008928571827709675,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.297991082072258,
      "step": 571
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1526.7612609863281,
      "epoch": 0.17086102606228062,
      "grad_norm": 25.230566024780273,
      "kl": 2.8671875,
      "learning_rate": 4.924005200283583e-07,
      "loss": 0.236,
      "reward": 0.377232164144516,
      "reward_std": 0.20791489630937576,
      "rewards/accuracy_reward": 0.08482143119908869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2924107238650322,
      "step": 572
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1561.0089721679688,
      "epoch": 0.1711597341498021,
      "grad_norm": 7.374900817871094,
      "kl": 2.72265625,
      "learning_rate": 4.923365844521832e-07,
      "loss": 0.2421,
      "reward": 0.3560267984867096,
      "reward_std": 0.2072165422141552,
      "rewards/accuracy_reward": 0.06473214598372579,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.2890625149011612,
      "step": 573
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1622.1072082519531,
      "epoch": 0.17145844223732357,
      "grad_norm": 41.591552734375,
      "kl": 3.40234375,
      "learning_rate": 4.922723852375559e-07,
      "loss": 0.2648,
      "reward": 0.314174123108387,
      "reward_std": 0.17675819993019104,
      "rewards/accuracy_reward": 0.04017857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2739955484867096,
      "step": 574
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1616.3795471191406,
      "epoch": 0.17175715032484504,
      "grad_norm": 14.09976577758789,
      "kl": 2.2109375,
      "learning_rate": 4.922079224543185e-07,
      "loss": 0.2336,
      "reward": 0.3431919813156128,
      "reward_std": 0.212439626455307,
      "rewards/accuracy_reward": 0.03571428777649999,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3074776902794838,
      "step": 575
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1580.3371276855469,
      "epoch": 0.1720558584123665,
      "grad_norm": 15.591771125793457,
      "kl": 2.064453125,
      "learning_rate": 4.921431961726006e-07,
      "loss": 0.2128,
      "reward": 0.3459821566939354,
      "reward_std": 0.18646401166915894,
      "rewards/accuracy_reward": 0.0558035746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.290178582072258,
      "step": 576
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1602.5134582519531,
      "epoch": 0.17235456649988798,
      "grad_norm": 11.33595085144043,
      "kl": 2.130859375,
      "learning_rate": 4.920782064628177e-07,
      "loss": 0.2221,
      "reward": 0.344866082072258,
      "reward_std": 0.2150227315723896,
      "rewards/accuracy_reward": 0.040178574388846755,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.302455373108387,
      "step": 577
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1604.3371276855469,
      "epoch": 0.17265327458740945,
      "grad_norm": 8.615790367126465,
      "kl": 2.71484375,
      "learning_rate": 4.920129533956725e-07,
      "loss": 0.2498,
      "reward": 0.356584832072258,
      "reward_std": 0.2187850959599018,
      "rewards/accuracy_reward": 0.06026786100119352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2963169887661934,
      "step": 578
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1670.72998046875,
      "epoch": 0.17295198267493092,
      "grad_norm": 13.563003540039062,
      "kl": 2.765625,
      "learning_rate": 4.919474370421538e-07,
      "loss": 0.2533,
      "reward": 0.3364955559372902,
      "reward_std": 0.22764960676431656,
      "rewards/accuracy_reward": 0.0424107164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.294084832072258,
      "step": 579
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1630.2411499023438,
      "epoch": 0.1732506907624524,
      "grad_norm": 13.145462989807129,
      "kl": 2.67578125,
      "learning_rate": 4.918816574735369e-07,
      "loss": 0.2387,
      "reward": 0.3616071566939354,
      "reward_std": 0.22869007661938667,
      "rewards/accuracy_reward": 0.05580357206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3058035746216774,
      "step": 580
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1694.3906860351562,
      "epoch": 0.17354939884997386,
      "grad_norm": 20.640827178955078,
      "kl": 3.359375,
      "learning_rate": 4.918156147613837e-07,
      "loss": 0.2687,
      "reward": 0.290736623108387,
      "reward_std": 0.20415988191962242,
      "rewards/accuracy_reward": 0.01785714365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2728794813156128,
      "step": 581
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1678.6875610351562,
      "epoch": 0.17384810693749533,
      "grad_norm": 7.124320983886719,
      "kl": 2.53125,
      "learning_rate": 4.917493089775421e-07,
      "loss": 0.2339,
      "reward": 0.3549107313156128,
      "reward_std": 0.19793832302093506,
      "rewards/accuracy_reward": 0.08705357555299997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2678571492433548,
      "step": 582
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1692.9108276367188,
      "epoch": 0.1741468150250168,
      "grad_norm": 13.515953063964844,
      "kl": 2.8515625,
      "learning_rate": 4.916827401941464e-07,
      "loss": 0.245,
      "reward": 0.3733259066939354,
      "reward_std": 0.2015364058315754,
      "rewards/accuracy_reward": 0.09598214738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2773437611758709,
      "step": 583
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1643.9442749023438,
      "epoch": 0.17444552311253828,
      "grad_norm": 13.006391525268555,
      "kl": 2.79296875,
      "learning_rate": 4.916159084836166e-07,
      "loss": 0.2416,
      "reward": 0.3789062723517418,
      "reward_std": 0.16606170311570168,
      "rewards/accuracy_reward": 0.10937500488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2695312649011612,
      "step": 584
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1644.3840026855469,
      "epoch": 0.17474423120005975,
      "grad_norm": 8.40479850769043,
      "kl": 2.8515625,
      "learning_rate": 4.915488139186594e-07,
      "loss": 0.2629,
      "reward": 0.349330373108387,
      "reward_std": 0.2074294127523899,
      "rewards/accuracy_reward": 0.04687500232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.302455373108387,
      "step": 585
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1646.1540832519531,
      "epoch": 0.17504293928758122,
      "grad_norm": 18.074647903442383,
      "kl": 3.30078125,
      "learning_rate": 4.91481456572267e-07,
      "loss": 0.2596,
      "reward": 0.3225446492433548,
      "reward_std": 0.20551162213087082,
      "rewards/accuracy_reward": 0.03125000116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2912946492433548,
      "step": 586
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1656.9577026367188,
      "epoch": 0.1753416473751027,
      "grad_norm": 6.897797107696533,
      "kl": 2.8515625,
      "learning_rate": 4.914138365177176e-07,
      "loss": 0.26,
      "reward": 0.3141741193830967,
      "reward_std": 0.22537293657660484,
      "rewards/accuracy_reward": 0.03125000139698386,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.2806919775903225,
      "step": 587
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1574.5960693359375,
      "epoch": 0.17564035546262416,
      "grad_norm": 13.853618621826172,
      "kl": 2.349609375,
      "learning_rate": 4.913459538285751e-07,
      "loss": 0.2598,
      "reward": 0.3722098395228386,
      "reward_std": 0.23297493532299995,
      "rewards/accuracy_reward": 0.06919643376022577,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.3007812649011612,
      "step": 588
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1621.83935546875,
      "epoch": 0.17593906355014563,
      "grad_norm": 16.838857650756836,
      "kl": 2.328125,
      "learning_rate": 4.912778085786893e-07,
      "loss": 0.2318,
      "reward": 0.3448660969734192,
      "reward_std": 0.20028283074498177,
      "rewards/accuracy_reward": 0.051339288242161274,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.293526791036129,
      "step": 589
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1692.7076721191406,
      "epoch": 0.1762377716376671,
      "grad_norm": 17.532466888427734,
      "kl": 3.38671875,
      "learning_rate": 4.912094008421956e-07,
      "loss": 0.2709,
      "reward": 0.317522332072258,
      "reward_std": 0.17356320843100548,
      "rewards/accuracy_reward": 0.03794643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2795758992433548,
      "step": 590
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1581.6116638183594,
      "epoch": 0.17653647972518857,
      "grad_norm": 11.020123481750488,
      "kl": 2.76171875,
      "learning_rate": 4.911407306935148e-07,
      "loss": 0.2541,
      "reward": 0.3537946566939354,
      "reward_std": 0.17900799587368965,
      "rewards/accuracy_reward": 0.044642859138548374,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3091518059372902,
      "step": 591
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1668.1473999023438,
      "epoch": 0.17683518781271004,
      "grad_norm": 37.67497253417969,
      "kl": 1.87890625,
      "learning_rate": 4.910717982073534e-07,
      "loss": 0.2106,
      "reward": 0.2991071566939354,
      "reward_std": 0.1814970299601555,
      "rewards/accuracy_reward": 0.0022321429569274187,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.294642873108387,
      "step": 592
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1638.9375915527344,
      "epoch": 0.17713389590023149,
      "grad_norm": 25.699308395385742,
      "kl": 2.04296875,
      "learning_rate": 4.91002603458703e-07,
      "loss": 0.231,
      "reward": 0.3549107238650322,
      "reward_std": 0.22091063484549522,
      "rewards/accuracy_reward": 0.05580357555299997,
      "rewards/format_reward": 0.006696428870782256,
      "rewards/tag_count_reward": 0.2924107238650322,
      "step": 593
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1667.4554138183594,
      "epoch": 0.17743260398775296,
      "grad_norm": 6.941884994506836,
      "kl": 2.3984375,
      "learning_rate": 4.909331465228409e-07,
      "loss": 0.2225,
      "reward": 0.4179687649011612,
      "reward_std": 0.2182648368179798,
      "rewards/accuracy_reward": 0.13616071827709675,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.2795759066939354,
      "step": 594
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1666.7701721191406,
      "epoch": 0.17773131207527443,
      "grad_norm": 52.901390075683594,
      "kl": 4.2734375,
      "learning_rate": 4.908634274753294e-07,
      "loss": 0.3053,
      "reward": 0.3236607313156128,
      "reward_std": 0.18835443258285522,
      "rewards/accuracy_reward": 0.051339289639145136,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2723214402794838,
      "step": 595
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1513.2567749023438,
      "epoch": 0.1780300201627959,
      "grad_norm": 10.335769653320312,
      "kl": 2.78125,
      "learning_rate": 4.907934463920159e-07,
      "loss": 0.2762,
      "reward": 0.404017873108387,
      "reward_std": 0.2034577988088131,
      "rewards/accuracy_reward": 0.08482143376022577,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3191964477300644,
      "step": 596
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1700.5335388183594,
      "epoch": 0.17832872825031737,
      "grad_norm": 28.970849990844727,
      "kl": 3.8515625,
      "learning_rate": 4.907232033490331e-07,
      "loss": 0.3015,
      "reward": 0.3510044813156128,
      "reward_std": 0.17583762481808662,
      "rewards/accuracy_reward": 0.0781250037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2728794738650322,
      "step": 597
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1686.16748046875,
      "epoch": 0.17862743633783884,
      "grad_norm": 22.58493423461914,
      "kl": 3.79296875,
      "learning_rate": 4.906526984227984e-07,
      "loss": 0.3228,
      "reward": 0.3738839402794838,
      "reward_std": 0.22883356735110283,
      "rewards/accuracy_reward": 0.08705357578583062,
      "rewards/format_reward": 0.006696428870782256,
      "rewards/tag_count_reward": 0.2801339440047741,
      "step": 598
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1569.5826416015625,
      "epoch": 0.1789261444253603,
      "grad_norm": 14.223552703857422,
      "kl": 2.9765625,
      "learning_rate": 4.905819316900142e-07,
      "loss": 0.3145,
      "reward": 0.337611623108387,
      "reward_std": 0.2109643742442131,
      "rewards/accuracy_reward": 0.022321430267766118,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3152901977300644,
      "step": 599
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1622.274658203125,
      "epoch": 0.17922485251288178,
      "grad_norm": 9.300459861755371,
      "kl": 2.63671875,
      "learning_rate": 4.905109032276681e-07,
      "loss": 0.2732,
      "reward": 0.372767873108387,
      "reward_std": 0.21699827164411545,
      "rewards/accuracy_reward": 0.0848214328289032,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.285714291036129,
      "step": 600
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1651.21435546875,
      "epoch": 0.17952356060040325,
      "grad_norm": 51.11439895629883,
      "kl": 5.03125,
      "learning_rate": 4.904396131130317e-07,
      "loss": 0.3929,
      "reward": 0.317522332072258,
      "reward_std": 0.23021812364459038,
      "rewards/accuracy_reward": 0.035714287078008056,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2818080484867096,
      "step": 601
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1674.1719360351562,
      "epoch": 0.17982226868792472,
      "grad_norm": 20.548410415649414,
      "kl": 3.171875,
      "learning_rate": 4.90368061423662e-07,
      "loss": 0.2202,
      "reward": 0.3766741305589676,
      "reward_std": 0.1925434172153473,
      "rewards/accuracy_reward": 0.08928571501746774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2873884066939354,
      "step": 602
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1580.935302734375,
      "epoch": 0.1801209767754462,
      "grad_norm": 16.438804626464844,
      "kl": 2.796875,
      "learning_rate": 4.902962482374e-07,
      "loss": 0.2856,
      "reward": 0.3582589402794838,
      "reward_std": 0.21278730779886246,
      "rewards/accuracy_reward": 0.0357142873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.322544664144516,
      "step": 603
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1576.2232971191406,
      "epoch": 0.18041968486296767,
      "grad_norm": 21.912410736083984,
      "kl": 2.5234375,
      "learning_rate": 4.902241736323715e-07,
      "loss": 0.2472,
      "reward": 0.4162946566939354,
      "reward_std": 0.18398118019104004,
      "rewards/accuracy_reward": 0.09598214668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3203125074505806,
      "step": 604
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1618.6072387695312,
      "epoch": 0.18071839295048914,
      "grad_norm": 10.333484649658203,
      "kl": 3.0390625,
      "learning_rate": 4.901518376869866e-07,
      "loss": 0.2857,
      "reward": 0.3565848395228386,
      "reward_std": 0.18648212775588036,
      "rewards/accuracy_reward": 0.05357143096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3030134066939354,
      "step": 605
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1692.0357971191406,
      "epoch": 0.1810171010380106,
      "grad_norm": 39.94902420043945,
      "kl": 4.359375,
      "learning_rate": 4.900792404799396e-07,
      "loss": 0.3473,
      "reward": 0.2929687649011612,
      "reward_std": 0.20128410309553146,
      "rewards/accuracy_reward": 0.01562500116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2773437574505806,
      "step": 606
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1629.6027526855469,
      "epoch": 0.18131580912553208,
      "grad_norm": 26.49481201171875,
      "kl": 3.83203125,
      "learning_rate": 4.900063820902094e-07,
      "loss": 0.3042,
      "reward": 0.436383955180645,
      "reward_std": 0.2070818915963173,
      "rewards/accuracy_reward": 0.12723214668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3091517984867096,
      "step": 607
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1623.35498046875,
      "epoch": 0.18161451721305355,
      "grad_norm": 14.277666091918945,
      "kl": 4.11328125,
      "learning_rate": 4.899332625970586e-07,
      "loss": 0.3485,
      "reward": 0.3074776902794838,
      "reward_std": 0.2183593437075615,
      "rewards/accuracy_reward": 0.015625000931322575,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.2896205484867096,
      "step": 608
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1687.8460693359375,
      "epoch": 0.18191322530057502,
      "grad_norm": 5.5822553634643555,
      "kl": 3.25,
      "learning_rate": 4.89859882080034e-07,
      "loss": 0.2597,
      "reward": 0.3030133992433548,
      "reward_std": 0.22221703827381134,
      "rewards/accuracy_reward": 0.02232142980210483,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2806919813156128,
      "step": 609
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1618.9397888183594,
      "epoch": 0.1822119333880965,
      "grad_norm": 10.38758373260498,
      "kl": 3.109375,
      "learning_rate": 4.897862406189667e-07,
      "loss": 0.3041,
      "reward": 0.3392857238650322,
      "reward_std": 0.24260738864541054,
      "rewards/accuracy_reward": 0.04910714365541935,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.2879464477300644,
      "step": 610
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1577.9286499023438,
      "epoch": 0.18251064147561796,
      "grad_norm": 9.135944366455078,
      "kl": 2.833984375,
      "learning_rate": 4.89712338293971e-07,
      "loss": 0.2745,
      "reward": 0.4447544887661934,
      "reward_std": 0.21125075966119766,
      "rewards/accuracy_reward": 0.14062500558793545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3041294738650322,
      "step": 611
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1580.3103332519531,
      "epoch": 0.18280934956313943,
      "grad_norm": 29.207996368408203,
      "kl": 2.73828125,
      "learning_rate": 4.896381751854456e-07,
      "loss": 0.2872,
      "reward": 0.3175223395228386,
      "reward_std": 0.2046714834868908,
      "rewards/accuracy_reward": 0.013392857974395156,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3041294813156128,
      "step": 612
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1483.1540832519531,
      "epoch": 0.1831080576506609,
      "grad_norm": 26.977313995361328,
      "kl": 2.46484375,
      "learning_rate": 4.895637513740727e-07,
      "loss": 0.2681,
      "reward": 0.4207589477300644,
      "reward_std": 0.17794334888458252,
      "rewards/accuracy_reward": 0.08705357694998384,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.333705373108387,
      "step": 613
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1615.3906860351562,
      "epoch": 0.18340676573818235,
      "grad_norm": 10.343697547912598,
      "kl": 2.9921875,
      "learning_rate": 4.89489066940818e-07,
      "loss": 0.2736,
      "reward": 0.3794643059372902,
      "reward_std": 0.20134438574314117,
      "rewards/accuracy_reward": 0.06250000116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3169643059372902,
      "step": 614
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1681.935302734375,
      "epoch": 0.18370547382570382,
      "grad_norm": 14.852068901062012,
      "kl": 3.83203125,
      "learning_rate": 4.89414121966931e-07,
      "loss": 0.3302,
      "reward": 0.299107164144516,
      "reward_std": 0.2102036066353321,
      "rewards/accuracy_reward": 0.01562500116415322,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.2812500223517418,
      "step": 615
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1658.2188415527344,
      "epoch": 0.1840041819132253,
      "grad_norm": 16.310871124267578,
      "kl": 3.6484375,
      "learning_rate": 4.893389165339443e-07,
      "loss": 0.2814,
      "reward": 0.3348214477300644,
      "reward_std": 0.19258644804358482,
      "rewards/accuracy_reward": 0.04687500116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2879464402794838,
      "step": 616
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1569.5804138183594,
      "epoch": 0.18430289000074676,
      "grad_norm": 21.803630828857422,
      "kl": 3.1484375,
      "learning_rate": 4.892634507236742e-07,
      "loss": 0.3106,
      "reward": 0.3822544813156128,
      "reward_std": 0.1858588345348835,
      "rewards/accuracy_reward": 0.07589286053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.306361623108387,
      "step": 617
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1668.4822387695312,
      "epoch": 0.18460159808826823,
      "grad_norm": 14.831948280334473,
      "kl": 3.86328125,
      "learning_rate": 4.891877246182202e-07,
      "loss": 0.3119,
      "reward": 0.3783482164144516,
      "reward_std": 0.1814921796321869,
      "rewards/accuracy_reward": 0.0803571492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2979910746216774,
      "step": 618
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1591.1339721679688,
      "epoch": 0.1849003061757897,
      "grad_norm": 39.02629852294922,
      "kl": 4.34375,
      "learning_rate": 4.891117382999646e-07,
      "loss": 0.3558,
      "reward": 0.3275669813156128,
      "reward_std": 0.2049858383834362,
      "rewards/accuracy_reward": 0.03348214412108064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.294084832072258,
      "step": 619
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1618.7143859863281,
      "epoch": 0.18519901426331117,
      "grad_norm": 26.222776412963867,
      "kl": 3.7421875,
      "learning_rate": 4.890354918515731e-07,
      "loss": 0.3142,
      "reward": 0.3967634066939354,
      "reward_std": 0.21178822219371796,
      "rewards/accuracy_reward": 0.09151786309666932,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.3030134066939354,
      "step": 620
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1662.9130249023438,
      "epoch": 0.18549772235083264,
      "grad_norm": 8.32137680053711,
      "kl": 3.53125,
      "learning_rate": 4.889589853559946e-07,
      "loss": 0.2963,
      "reward": 0.3125000149011612,
      "reward_std": 0.20667875930666924,
      "rewards/accuracy_reward": 0.017857143422588706,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2946428656578064,
      "step": 621
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1503.6317749023438,
      "epoch": 0.18579643043835412,
      "grad_norm": 43.870338439941406,
      "kl": 1.9375,
      "learning_rate": 4.888822188964606e-07,
      "loss": 0.2487,
      "reward": 0.4508928805589676,
      "reward_std": 0.2510574646294117,
      "rewards/accuracy_reward": 0.11607143399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3348214402794838,
      "step": 622
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1484.8259887695312,
      "epoch": 0.1860951385258756,
      "grad_norm": 33.6615104675293,
      "kl": 2.12109375,
      "learning_rate": 4.888051925564853e-07,
      "loss": 0.2328,
      "reward": 0.440848246216774,
      "reward_std": 0.23657935112714767,
      "rewards/accuracy_reward": 0.10937500605359674,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.329241082072258,
      "step": 623
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1610.1005249023438,
      "epoch": 0.18639384661339706,
      "grad_norm": 32.655677795410156,
      "kl": 2.302734375,
      "learning_rate": 4.887279064198659e-07,
      "loss": 0.2492,
      "reward": 0.4090401977300644,
      "reward_std": 0.2252359464764595,
      "rewards/accuracy_reward": 0.10714286309666932,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3018973395228386,
      "step": 624
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1678.9241638183594,
      "epoch": 0.18669255470091853,
      "grad_norm": 22.373123168945312,
      "kl": 2.83203125,
      "learning_rate": 4.886503605706821e-07,
      "loss": 0.2782,
      "reward": 0.325334832072258,
      "reward_std": 0.20584102347493172,
      "rewards/accuracy_reward": 0.026785715715959668,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2985491156578064,
      "step": 625
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1638.8482971191406,
      "epoch": 0.18699126278844,
      "grad_norm": 15.191483497619629,
      "kl": 2.65234375,
      "learning_rate": 4.885725550932964e-07,
      "loss": 0.2436,
      "reward": 0.3655134066939354,
      "reward_std": 0.21288607269525528,
      "rewards/accuracy_reward": 0.06026786123402417,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3052455484867096,
      "step": 626
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1627.3973999023438,
      "epoch": 0.18728997087596147,
      "grad_norm": 15.465039253234863,
      "kl": 3.78515625,
      "learning_rate": 4.884944900723533e-07,
      "loss": 0.3171,
      "reward": 0.3872768059372902,
      "reward_std": 0.19754140451550484,
      "rewards/accuracy_reward": 0.07812500349245965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3091517984867096,
      "step": 627
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1637.3370971679688,
      "epoch": 0.18758867896348294,
      "grad_norm": 33.3397216796875,
      "kl": 4.16796875,
      "learning_rate": 4.884161655927801e-07,
      "loss": 0.3018,
      "reward": 0.3085937574505806,
      "reward_std": 0.20259520784020424,
      "rewards/accuracy_reward": 0.0200892873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2885044738650322,
      "step": 628
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1671.247802734375,
      "epoch": 0.1878873870510044,
      "grad_norm": 37.147525787353516,
      "kl": 4.47265625,
      "learning_rate": 4.883375817397859e-07,
      "loss": 0.3346,
      "reward": 0.4190848395228386,
      "reward_std": 0.21098532527685165,
      "rewards/accuracy_reward": 0.13169643143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2873883992433548,
      "step": 629
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1632.8840026855469,
      "epoch": 0.18818609513852588,
      "grad_norm": 27.872949600219727,
      "kl": 4.00390625,
      "learning_rate": 4.882587385988623e-07,
      "loss": 0.3139,
      "reward": 0.318080373108387,
      "reward_std": 0.2043987177312374,
      "rewards/accuracy_reward": 0.026785716181620955,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2912946492433548,
      "step": 630
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1567.22998046875,
      "epoch": 0.18848480322604735,
      "grad_norm": 10.563283920288086,
      "kl": 3.765625,
      "learning_rate": 4.881796362557832e-07,
      "loss": 0.3254,
      "reward": 0.3247767984867096,
      "reward_std": 0.20603583008050919,
      "rewards/accuracy_reward": 0.0267857164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2979910746216774,
      "step": 631
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1497.3326721191406,
      "epoch": 0.18878351131356882,
      "grad_norm": 13.469141006469727,
      "kl": 2.61328125,
      "learning_rate": 4.881002747966038e-07,
      "loss": 0.2605,
      "reward": 0.4218750149011612,
      "reward_std": 0.19058503955602646,
      "rewards/accuracy_reward": 0.08258928917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3392857238650322,
      "step": 632
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1568.7969360351562,
      "epoch": 0.1890822194010903,
      "grad_norm": 25.033424377441406,
      "kl": 2.59765625,
      "learning_rate": 4.880206543076617e-07,
      "loss": 0.26,
      "reward": 0.3420759066939354,
      "reward_std": 0.20988080650568008,
      "rewards/accuracy_reward": 0.017857144121080637,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3242187649011612,
      "step": 633
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1613.1407165527344,
      "epoch": 0.18938092748861177,
      "grad_norm": 13.932612419128418,
      "kl": 3.41796875,
      "learning_rate": 4.879407748755764e-07,
      "loss": 0.2831,
      "reward": 0.3872768059372902,
      "reward_std": 0.20878805965185165,
      "rewards/accuracy_reward": 0.08482143026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.302455373108387,
      "step": 634
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1649.4308776855469,
      "epoch": 0.18967963557613324,
      "grad_norm": 4.663081645965576,
      "kl": 3.28515625,
      "learning_rate": 4.878606365872486e-07,
      "loss": 0.2992,
      "reward": 0.3030134066939354,
      "reward_std": 0.19142171368002892,
      "rewards/accuracy_reward": 0.01562500069849193,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.287388414144516,
      "step": 635
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1674.6920471191406,
      "epoch": 0.18997834366365468,
      "grad_norm": 44.26906204223633,
      "kl": 2.3515625,
      "learning_rate": 4.877802395298612e-07,
      "loss": 0.2654,
      "reward": 0.3281250223517418,
      "reward_std": 0.22746697440743446,
      "rewards/accuracy_reward": 0.03794643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.290178582072258,
      "step": 636
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1646.3996276855469,
      "epoch": 0.19027705175117615,
      "grad_norm": 42.17460632324219,
      "kl": 2.333984375,
      "learning_rate": 4.87699583790878e-07,
      "loss": 0.2495,
      "reward": 0.3515625223517418,
      "reward_std": 0.19890738278627396,
      "rewards/accuracy_reward": 0.04241071501746774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3091517984867096,
      "step": 637
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1622.7098999023438,
      "epoch": 0.19057575983869762,
      "grad_norm": 12.488162994384766,
      "kl": 2.77734375,
      "learning_rate": 4.876186694580448e-07,
      "loss": 0.2436,
      "reward": 0.401227705180645,
      "reward_std": 0.17957303300499916,
      "rewards/accuracy_reward": 0.08705357578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.314174123108387,
      "step": 638
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1588.1897888183594,
      "epoch": 0.1908744679262191,
      "grad_norm": 19.67913246154785,
      "kl": 3.0703125,
      "learning_rate": 4.875374966193883e-07,
      "loss": 0.3204,
      "reward": 0.4252232238650322,
      "reward_std": 0.20518175140023232,
      "rewards/accuracy_reward": 0.11160715040750802,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.313616082072258,
      "step": 639
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1665.0648193359375,
      "epoch": 0.19117317601374056,
      "grad_norm": 32.708717346191406,
      "kl": 4.12109375,
      "learning_rate": 4.874560653632167e-07,
      "loss": 0.3251,
      "reward": 0.3593750223517418,
      "reward_std": 0.23873838409781456,
      "rewards/accuracy_reward": 0.07589286006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2834821492433548,
      "step": 640
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1676.6630249023438,
      "epoch": 0.19147188410126204,
      "grad_norm": 32.92763137817383,
      "kl": 4.4765625,
      "learning_rate": 4.873743757781191e-07,
      "loss": 0.3681,
      "reward": 0.3420759066939354,
      "reward_std": 0.21803581342101097,
      "rewards/accuracy_reward": 0.06473214644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2773437649011612,
      "step": 641
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1685.747802734375,
      "epoch": 0.1917705921887835,
      "grad_norm": 27.858640670776367,
      "kl": 4.21875,
      "learning_rate": 4.87292427952966e-07,
      "loss": 0.3191,
      "reward": 0.3543526977300644,
      "reward_std": 0.19386287406086922,
      "rewards/accuracy_reward": 0.05133928661234677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3030134066939354,
      "step": 642
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1563.3170166015625,
      "epoch": 0.19206930027630498,
      "grad_norm": 19.176326751708984,
      "kl": 4.1171875,
      "learning_rate": 4.872102219769083e-07,
      "loss": 0.3384,
      "reward": 0.432477705180645,
      "reward_std": 0.16325363144278526,
      "rewards/accuracy_reward": 0.11160715157166123,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3208705559372902,
      "step": 643
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1591.0246276855469,
      "epoch": 0.19236800836382645,
      "grad_norm": 5.66936731338501,
      "kl": 3.38671875,
      "learning_rate": 4.871277579393782e-07,
      "loss": 0.3103,
      "reward": 0.4196428880095482,
      "reward_std": 0.19601045176386833,
      "rewards/accuracy_reward": 0.1026785746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.316964291036129,
      "step": 644
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1606.3951721191406,
      "epoch": 0.19266671645134792,
      "grad_norm": 14.642088890075684,
      "kl": 3.49609375,
      "learning_rate": 4.870450359300886e-07,
      "loss": 0.2979,
      "reward": 0.3861607313156128,
      "reward_std": 0.22472280263900757,
      "rewards/accuracy_reward": 0.07142857369035482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3147321566939354,
      "step": 645
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1710.2188110351562,
      "epoch": 0.1929654245388694,
      "grad_norm": 6.783075332641602,
      "kl": 3.484375,
      "learning_rate": 4.869620560390327e-07,
      "loss": 0.2848,
      "reward": 0.424107164144516,
      "reward_std": 0.2475946806371212,
      "rewards/accuracy_reward": 0.1227678619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3013392984867096,
      "step": 646
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1600.3973693847656,
      "epoch": 0.19326413262639086,
      "grad_norm": 12.36570930480957,
      "kl": 3.5859375,
      "learning_rate": 4.868788183564847e-07,
      "loss": 0.3079,
      "reward": 0.3443080484867096,
      "reward_std": 0.15416168048977852,
      "rewards/accuracy_reward": 0.0357142873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3085937574505806,
      "step": 647
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1593.4040832519531,
      "epoch": 0.19356284071391233,
      "grad_norm": 22.541234970092773,
      "kl": 2.82421875,
      "learning_rate": 4.867953229729988e-07,
      "loss": 0.2725,
      "reward": 0.388950914144516,
      "reward_std": 0.1738101914525032,
      "rewards/accuracy_reward": 0.08035714644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3085937574505806,
      "step": 648
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1596.3839721679688,
      "epoch": 0.1938615488014338,
      "grad_norm": 29.291837692260742,
      "kl": 2.734375,
      "learning_rate": 4.867115699794101e-07,
      "loss": 0.2901,
      "reward": 0.4068080484867096,
      "reward_std": 0.1978348344564438,
      "rewards/accuracy_reward": 0.0915178619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3152901977300644,
      "step": 649
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1603.6340026855469,
      "epoch": 0.19416025688895527,
      "grad_norm": 32.39169692993164,
      "kl": 2.275390625,
      "learning_rate": 4.866275594668335e-07,
      "loss": 0.237,
      "reward": 0.451450914144516,
      "reward_std": 0.2406710982322693,
      "rewards/accuracy_reward": 0.12276786146685481,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.326450914144516,
      "step": 650
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1638.8013916015625,
      "epoch": 0.19445896497647674,
      "grad_norm": 26.57655143737793,
      "kl": 2.890625,
      "learning_rate": 4.865432915266641e-07,
      "loss": 0.2803,
      "reward": 0.3962053656578064,
      "reward_std": 0.21351991221308708,
      "rewards/accuracy_reward": 0.0937500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3024553656578064,
      "step": 651
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1655.0938415527344,
      "epoch": 0.19475767306399822,
      "grad_norm": 5.260843753814697,
      "kl": 3.4765625,
      "learning_rate": 4.864587662505773e-07,
      "loss": 0.2902,
      "reward": 0.388392873108387,
      "reward_std": 0.18621479347348213,
      "rewards/accuracy_reward": 0.08258928940631449,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.305803582072258,
      "step": 652
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1654.3817749023438,
      "epoch": 0.1950563811515197,
      "grad_norm": 13.082962036132812,
      "kl": 3.69921875,
      "learning_rate": 4.863739837305282e-07,
      "loss": 0.2962,
      "reward": 0.3750000149011612,
      "reward_std": 0.2190452516078949,
      "rewards/accuracy_reward": 0.0848214328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.290178582072258,
      "step": 653
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1635.0402526855469,
      "epoch": 0.19535508923904116,
      "grad_norm": 4.247363090515137,
      "kl": 3.17578125,
      "learning_rate": 4.86288944058752e-07,
      "loss": 0.2875,
      "reward": 0.330357164144516,
      "reward_std": 0.2030586116015911,
      "rewards/accuracy_reward": 0.026785715715959668,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3035714402794838,
      "step": 654
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1734.6116943359375,
      "epoch": 0.19565379732656263,
      "grad_norm": 23.231365203857422,
      "kl": 4.1328125,
      "learning_rate": 4.862036473277634e-07,
      "loss": 0.3105,
      "reward": 0.2890625149011612,
      "reward_std": 0.1819249652326107,
      "rewards/accuracy_reward": 0.004464285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2845982313156128,
      "step": 655
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1635.8572082519531,
      "epoch": 0.1959525054140841,
      "grad_norm": 8.652819633483887,
      "kl": 3.58984375,
      "learning_rate": 4.86118093630357e-07,
      "loss": 0.3167,
      "reward": 0.3833705559372902,
      "reward_std": 0.2126331329345703,
      "rewards/accuracy_reward": 0.09151786309666932,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2918526902794838,
      "step": 656
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1560.5915832519531,
      "epoch": 0.19625121350160554,
      "grad_norm": 16.258974075317383,
      "kl": 3.546875,
      "learning_rate": 4.860322830596069e-07,
      "loss": 0.3565,
      "reward": 0.3895089477300644,
      "reward_std": 0.21800580248236656,
      "rewards/accuracy_reward": 0.07142857648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3180803656578064,
      "step": 657
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1796.0581359863281,
      "epoch": 0.196549921589127,
      "grad_norm": 66.2879638671875,
      "kl": 4.84375,
      "learning_rate": 4.859462157088663e-07,
      "loss": 0.3193,
      "reward": 0.349330373108387,
      "reward_std": 0.17484188824892044,
      "rewards/accuracy_reward": 0.07589286053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2734375223517418,
      "step": 658
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1577.77685546875,
      "epoch": 0.19684862967664848,
      "grad_norm": 11.260113716125488,
      "kl": 3.58984375,
      "learning_rate": 4.858598916717683e-07,
      "loss": 0.34,
      "reward": 0.4112723395228386,
      "reward_std": 0.17002077400684357,
      "rewards/accuracy_reward": 0.08035714668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3309151902794838,
      "step": 659
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1647.8951721191406,
      "epoch": 0.19714733776416996,
      "grad_norm": 33.16048812866211,
      "kl": 4.39453125,
      "learning_rate": 4.857733110422249e-07,
      "loss": 0.3565,
      "reward": 0.3270089402794838,
      "reward_std": 0.2113390676677227,
      "rewards/accuracy_reward": 0.022321429569274187,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3046875074505806,
      "step": 660
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1621.7232666015625,
      "epoch": 0.19744604585169143,
      "grad_norm": 30.13826560974121,
      "kl": 4.03515625,
      "learning_rate": 4.856864739144274e-07,
      "loss": 0.3086,
      "reward": 0.498325914144516,
      "reward_std": 0.16964975371956825,
      "rewards/accuracy_reward": 0.18973215110599995,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3085937574505806,
      "step": 661
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1749.1563415527344,
      "epoch": 0.1977447539392129,
      "grad_norm": 16.775165557861328,
      "kl": 3.8671875,
      "learning_rate": 4.85599380382846e-07,
      "loss": 0.3028,
      "reward": 0.2957589477300644,
      "reward_std": 0.21433276683092117,
      "rewards/accuracy_reward": 0.024553571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2712053656578064,
      "step": 662
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1697.16748046875,
      "epoch": 0.19804346202673437,
      "grad_norm": 4.305462837219238,
      "kl": 3.37890625,
      "learning_rate": 4.8551203054223e-07,
      "loss": 0.2979,
      "reward": 0.3007812723517418,
      "reward_std": 0.18942279741168022,
      "rewards/accuracy_reward": 0.011160715017467737,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2896205484867096,
      "step": 663
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1588.7388916015625,
      "epoch": 0.19834217011425584,
      "grad_norm": 77.33695983886719,
      "kl": 2.1015625,
      "learning_rate": 4.854244244876073e-07,
      "loss": 0.2773,
      "reward": 0.3671875149011612,
      "reward_std": 0.18020589649677277,
      "rewards/accuracy_reward": 0.0424107164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3247767984867096,
      "step": 664
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1656.0246276855469,
      "epoch": 0.1986408782017773,
      "grad_norm": 4.391027450561523,
      "kl": 3.1796875,
      "learning_rate": 4.85336562314285e-07,
      "loss": 0.2796,
      "reward": 0.372209832072258,
      "reward_std": 0.20364606007933617,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.309709832072258,
      "step": 665
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1601.0491943359375,
      "epoch": 0.19893958628929878,
      "grad_norm": 10.950960159301758,
      "kl": 3.20703125,
      "learning_rate": 4.852484441178482e-07,
      "loss": 0.2908,
      "reward": 0.4263393059372902,
      "reward_std": 0.2214193306863308,
      "rewards/accuracy_reward": 0.11383929289877415,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3125000149011612,
      "step": 666
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1626.7120971679688,
      "epoch": 0.19923829437682025,
      "grad_norm": 20.771196365356445,
      "kl": 3.421875,
      "learning_rate": 4.851600699941612e-07,
      "loss": 0.3336,
      "reward": 0.3309151977300644,
      "reward_std": 0.19198953732848167,
      "rewards/accuracy_reward": 0.013392857508733869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3175223395228386,
      "step": 667
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1625.7991638183594,
      "epoch": 0.19953700246434172,
      "grad_norm": 6.68268346786499,
      "kl": 3.83984375,
      "learning_rate": 4.850714400393661e-07,
      "loss": 0.3348,
      "reward": 0.4687500298023224,
      "reward_std": 0.2117708921432495,
      "rewards/accuracy_reward": 0.16517858067527413,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3035714402794838,
      "step": 668
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1598.1273193359375,
      "epoch": 0.1998357105518632,
      "grad_norm": 21.330608367919922,
      "kl": 3.3515625,
      "learning_rate": 4.849825543498837e-07,
      "loss": 0.3204,
      "reward": 0.487723246216774,
      "reward_std": 0.18235694989562035,
      "rewards/accuracy_reward": 0.15848214738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.329241082072258,
      "step": 669
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1643.3594665527344,
      "epoch": 0.20013441863938466,
      "grad_norm": 9.666836738586426,
      "kl": 3.4453125,
      "learning_rate": 4.848934130224131e-07,
      "loss": 0.3204,
      "reward": 0.3459821566939354,
      "reward_std": 0.2026408426463604,
      "rewards/accuracy_reward": 0.029017858440056443,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3169643059372902,
      "step": 670
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1564.1072082519531,
      "epoch": 0.20043312672690614,
      "grad_norm": 27.48702049255371,
      "kl": 4.5,
      "learning_rate": 4.84804016153931e-07,
      "loss": 0.3625,
      "reward": 0.444196455180645,
      "reward_std": 0.20088427513837814,
      "rewards/accuracy_reward": 0.11383928824216127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3303571566939354,
      "step": 671
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1523.6585693359375,
      "epoch": 0.2007318348144276,
      "grad_norm": 14.57810115814209,
      "kl": 3.83984375,
      "learning_rate": 4.847143638416927e-07,
      "loss": 0.3373,
      "reward": 0.477678582072258,
      "reward_std": 0.19053140096366405,
      "rewards/accuracy_reward": 0.13169643399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3459821566939354,
      "step": 672
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1587.7165832519531,
      "epoch": 0.20103054290194908,
      "grad_norm": 31.174837112426758,
      "kl": 4.59375,
      "learning_rate": 4.846244561832309e-07,
      "loss": 0.3924,
      "reward": 0.4017857387661934,
      "reward_std": 0.2314198613166809,
      "rewards/accuracy_reward": 0.07812500465661287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3236607238650322,
      "step": 673
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1596.8505249023438,
      "epoch": 0.20132925098947055,
      "grad_norm": 23.666088104248047,
      "kl": 4.76953125,
      "learning_rate": 4.845342932763565e-07,
      "loss": 0.4063,
      "reward": 0.3593750149011612,
      "reward_std": 0.24428743869066238,
      "rewards/accuracy_reward": 0.04687500209547579,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3125000149011612,
      "step": 674
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1566.3929443359375,
      "epoch": 0.20162795907699202,
      "grad_norm": 12.397934913635254,
      "kl": 4.01953125,
      "learning_rate": 4.844438752191579e-07,
      "loss": 0.3537,
      "reward": 0.341517873108387,
      "reward_std": 0.1769193708896637,
      "rewards/accuracy_reward": 0.008928572060540318,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.332589291036129,
      "step": 675
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1613.3416137695312,
      "epoch": 0.2019266671645135,
      "grad_norm": 8.899201393127441,
      "kl": 3.98046875,
      "learning_rate": 4.843532021100009e-07,
      "loss": 0.3494,
      "reward": 0.3967634066939354,
      "reward_std": 0.22016541659832,
      "rewards/accuracy_reward": 0.08035714738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3164062574505806,
      "step": 676
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1548.21435546875,
      "epoch": 0.20222537525203496,
      "grad_norm": 27.794336318969727,
      "kl": 3.22265625,
      "learning_rate": 4.842622740475291e-07,
      "loss": 0.3359,
      "reward": 0.428013414144516,
      "reward_std": 0.19770359247922897,
      "rewards/accuracy_reward": 0.0937500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3342633992433548,
      "step": 677
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1570.52685546875,
      "epoch": 0.20252408333955643,
      "grad_norm": 25.658205032348633,
      "kl": 3.5703125,
      "learning_rate": 4.841710911306634e-07,
      "loss": 0.3606,
      "reward": 0.4564732313156128,
      "reward_std": 0.16606799513101578,
      "rewards/accuracy_reward": 0.1205357201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3359375149011612,
      "step": 678
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1532.40185546875,
      "epoch": 0.20282279142707788,
      "grad_norm": 25.947763442993164,
      "kl": 3.12109375,
      "learning_rate": 4.840796534586018e-07,
      "loss": 0.3106,
      "reward": 0.3978794887661934,
      "reward_std": 0.19667642191052437,
      "rewards/accuracy_reward": 0.05133928917348385,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.3443080484867096,
      "step": 679
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1696.8773193359375,
      "epoch": 0.20312149951459935,
      "grad_norm": 25.272611618041992,
      "kl": 4.109375,
      "learning_rate": 4.839879611308194e-07,
      "loss": 0.3037,
      "reward": 0.419642873108387,
      "reward_std": 0.2102460116147995,
      "rewards/accuracy_reward": 0.12500000232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.294642873108387,
      "step": 680
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1451.6630249023438,
      "epoch": 0.20342020760212082,
      "grad_norm": 26.330211639404297,
      "kl": 3.33984375,
      "learning_rate": 4.838960142470687e-07,
      "loss": 0.3343,
      "reward": 0.4827009215950966,
      "reward_std": 0.22188091650605202,
      "rewards/accuracy_reward": 0.12500000558793545,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.3554687649011612,
      "step": 681
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1607.9978332519531,
      "epoch": 0.2037189156896423,
      "grad_norm": 9.8863525390625,
      "kl": 3.77734375,
      "learning_rate": 4.838038129073787e-07,
      "loss": 0.3168,
      "reward": 0.3995535895228386,
      "reward_std": 0.19469749927520752,
      "rewards/accuracy_reward": 0.060267860535532236,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3392857313156128,
      "step": 682
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1570.2254943847656,
      "epoch": 0.20401762377716376,
      "grad_norm": 11.120509147644043,
      "kl": 3.73046875,
      "learning_rate": 4.837113572120555e-07,
      "loss": 0.3472,
      "reward": 0.4202009066939354,
      "reward_std": 0.19817324727773666,
      "rewards/accuracy_reward": 0.09151786006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3286830559372902,
      "step": 683
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1587.0603637695312,
      "epoch": 0.20431633186468523,
      "grad_norm": 49.873897552490234,
      "kl": 4.390625,
      "learning_rate": 4.836186472616818e-07,
      "loss": 0.3218,
      "reward": 0.3984375223517418,
      "reward_std": 0.19624651595950127,
      "rewards/accuracy_reward": 0.07142857369035482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3270089477300644,
      "step": 684
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1582.4800109863281,
      "epoch": 0.2046150399522067,
      "grad_norm": 16.45689582824707,
      "kl": 4.1796875,
      "learning_rate": 4.83525683157117e-07,
      "loss": 0.3553,
      "reward": 0.3911830484867096,
      "reward_std": 0.22156401351094246,
      "rewards/accuracy_reward": 0.06026785937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3309151902794838,
      "step": 685
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1588.60498046875,
      "epoch": 0.20491374803972817,
      "grad_norm": 19.330488204956055,
      "kl": 4.28515625,
      "learning_rate": 4.834324649994969e-07,
      "loss": 0.3524,
      "reward": 0.4330357387661934,
      "reward_std": 0.2463468685746193,
      "rewards/accuracy_reward": 0.0982142873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3348214328289032,
      "step": 686
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1684.2857971191406,
      "epoch": 0.20521245612724964,
      "grad_norm": 31.813735961914062,
      "kl": 4.96875,
      "learning_rate": 4.833389928902337e-07,
      "loss": 0.391,
      "reward": 0.3314732313156128,
      "reward_std": 0.22441529482603073,
      "rewards/accuracy_reward": 0.026785715017467737,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3046875149011612,
      "step": 687
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1545.7232666015625,
      "epoch": 0.2055111642147711,
      "grad_norm": 7.393466472625732,
      "kl": 4.18359375,
      "learning_rate": 4.83245266931016e-07,
      "loss": 0.3733,
      "reward": 0.435267873108387,
      "reward_std": 0.21350396052002907,
      "rewards/accuracy_reward": 0.09598214738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3392857313156128,
      "step": 688
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1604.4398193359375,
      "epoch": 0.20580987230229258,
      "grad_norm": 5.005248069763184,
      "kl": 4.1328125,
      "learning_rate": 4.831512872238082e-07,
      "loss": 0.3578,
      "reward": 0.4095982387661934,
      "reward_std": 0.2491498589515686,
      "rewards/accuracy_reward": 0.08928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3203125149011612,
      "step": 689
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1585.0938110351562,
      "epoch": 0.20610858038981406,
      "grad_norm": 14.290268898010254,
      "kl": 3.97265625,
      "learning_rate": 4.830570538708513e-07,
      "loss": 0.3358,
      "reward": 0.3822544738650322,
      "reward_std": 0.22422278299927711,
      "rewards/accuracy_reward": 0.07142857392318547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3108259066939354,
      "step": 690
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1676.305908203125,
      "epoch": 0.20640728847733553,
      "grad_norm": 22.375356674194336,
      "kl": 3.16015625,
      "learning_rate": 4.829625669746619e-07,
      "loss": 0.2748,
      "reward": 0.4799107313156128,
      "reward_std": 0.1814117282629013,
      "rewards/accuracy_reward": 0.15178571757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3281250149011612,
      "step": 691
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1566.1317443847656,
      "epoch": 0.206705996564857,
      "grad_norm": 36.52425765991211,
      "kl": 2.884765625,
      "learning_rate": 4.828678266380323e-07,
      "loss": 0.2878,
      "reward": 0.3738839402794838,
      "reward_std": 0.19423465430736542,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3426339402794838,
      "step": 692
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1623.6406860351562,
      "epoch": 0.20700470465237847,
      "grad_norm": 43.555179595947266,
      "kl": 2.8203125,
      "learning_rate": 4.827728329640308e-07,
      "loss": 0.2763,
      "reward": 0.3906250149011612,
      "reward_std": 0.211230106651783,
      "rewards/accuracy_reward": 0.05803571850992739,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3325893059372902,
      "step": 693
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1623.8393249511719,
      "epoch": 0.20730341273989994,
      "grad_norm": 27.840177536010742,
      "kl": 3.28515625,
      "learning_rate": 4.826775860560013e-07,
      "loss": 0.3078,
      "reward": 0.415736623108387,
      "reward_std": 0.20240634679794312,
      "rewards/accuracy_reward": 0.0959821455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3197544738650322,
      "step": 694
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1613.8817749023438,
      "epoch": 0.2076021208274214,
      "grad_norm": 6.394109725952148,
      "kl": 4.0078125,
      "learning_rate": 4.82582086017563e-07,
      "loss": 0.3677,
      "reward": 0.4581473469734192,
      "reward_std": 0.17779334634542465,
      "rewards/accuracy_reward": 0.1517857201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3063616156578064,
      "step": 695
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1609.3996276855469,
      "epoch": 0.20790082891494288,
      "grad_norm": 15.590879440307617,
      "kl": 3.41796875,
      "learning_rate": 4.824863329526106e-07,
      "loss": 0.3195,
      "reward": 0.3231026977300644,
      "reward_std": 0.1993384137749672,
      "rewards/accuracy_reward": 0.015625000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3074776902794838,
      "step": 696
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1616.2388916015625,
      "epoch": 0.20819953700246435,
      "grad_norm": 14.615315437316895,
      "kl": 3.7578125,
      "learning_rate": 4.823903269653141e-07,
      "loss": 0.306,
      "reward": 0.419642873108387,
      "reward_std": 0.18257762491703033,
      "rewards/accuracy_reward": 0.09598215040750802,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3236607238650322,
      "step": 697
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1609.6072082519531,
      "epoch": 0.20849824508998582,
      "grad_norm": 5.701258659362793,
      "kl": 3.609375,
      "learning_rate": 4.822940681601186e-07,
      "loss": 0.3122,
      "reward": 0.3744419887661934,
      "reward_std": 0.1936628483235836,
      "rewards/accuracy_reward": 0.053571431431919336,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3208705484867096,
      "step": 698
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1593.5781860351562,
      "epoch": 0.2087969531775073,
      "grad_norm": 27.374202728271484,
      "kl": 3.2734375,
      "learning_rate": 4.821975566417441e-07,
      "loss": 0.3038,
      "reward": 0.389508955180645,
      "reward_std": 0.20435847714543343,
      "rewards/accuracy_reward": 0.05803571571595967,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3314732238650322,
      "step": 699
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1679.4107666015625,
      "epoch": 0.20909566126502874,
      "grad_norm": 9.274701118469238,
      "kl": 3.80078125,
      "learning_rate": 4.821007925151858e-07,
      "loss": 0.3045,
      "reward": 0.3861607313156128,
      "reward_std": 0.2269614152610302,
      "rewards/accuracy_reward": 0.06473214668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.321428582072258,
      "step": 700
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1566.6161193847656,
      "epoch": 0.2093943693525502,
      "grad_norm": 5.112265586853027,
      "kl": 3.99609375,
      "learning_rate": 4.820037758857136e-07,
      "loss": 0.3589,
      "reward": 0.4408482238650322,
      "reward_std": 0.19334249570965767,
      "rewards/accuracy_reward": 0.11160714761354029,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.329241082072258,
      "step": 701
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1665.2366943359375,
      "epoch": 0.20969307744007168,
      "grad_norm": 31.22503089904785,
      "kl": 4.40625,
      "learning_rate": 4.81906506858872e-07,
      "loss": 0.3442,
      "reward": 0.309709832072258,
      "reward_std": 0.168434988707304,
      "rewards/accuracy_reward": 0.004464285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3052455484867096,
      "step": 702
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1616.1585388183594,
      "epoch": 0.20999178552759315,
      "grad_norm": 48.63649368286133,
      "kl": 5.0234375,
      "learning_rate": 4.818089855404803e-07,
      "loss": 0.3901,
      "reward": 0.4174107313156128,
      "reward_std": 0.24856522306799889,
      "rewards/accuracy_reward": 0.09821429196745157,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3191964477300644,
      "step": 703
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1623.3817749023438,
      "epoch": 0.21029049361511462,
      "grad_norm": 63.81969451904297,
      "kl": 5.32421875,
      "learning_rate": 4.817112120366319e-07,
      "loss": 0.4024,
      "reward": 0.353236623108387,
      "reward_std": 0.20233870670199394,
      "rewards/accuracy_reward": 0.05133928847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3018973395228386,
      "step": 704
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1628.1719665527344,
      "epoch": 0.2105892017026361,
      "grad_norm": 5.314508438110352,
      "kl": 3.8984375,
      "learning_rate": 4.81613186453695e-07,
      "loss": 0.3341,
      "reward": 0.338169664144516,
      "reward_std": 0.19756008312106133,
      "rewards/accuracy_reward": 0.017857144121080637,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3203125074505806,
      "step": 705
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1602.12060546875,
      "epoch": 0.21088790979015756,
      "grad_norm": 30.735994338989258,
      "kl": 4.5078125,
      "learning_rate": 4.815149088983119e-07,
      "loss": 0.3567,
      "reward": 0.4520089477300644,
      "reward_std": 0.2243431620299816,
      "rewards/accuracy_reward": 0.12276786309666932,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.329241082072258,
      "step": 706
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1666.4665832519531,
      "epoch": 0.21118661787767903,
      "grad_norm": 13.175734519958496,
      "kl": 4.37109375,
      "learning_rate": 4.814163794773988e-07,
      "loss": 0.3431,
      "reward": 0.3203125074505806,
      "reward_std": 0.1819184236228466,
      "rewards/accuracy_reward": 0.004464285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3158482238650322,
      "step": 707
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1579.6585388183594,
      "epoch": 0.2114853259652005,
      "grad_norm": 48.19032669067383,
      "kl": 3.0,
      "learning_rate": 4.81317598298146e-07,
      "loss": 0.3184,
      "reward": 0.3705357387661934,
      "reward_std": 0.20747651532292366,
      "rewards/accuracy_reward": 0.037946430733427405,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.332589291036129,
      "step": 708
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1580.1719360351562,
      "epoch": 0.21178403405272198,
      "grad_norm": 53.444732666015625,
      "kl": 2.94921875,
      "learning_rate": 4.812185654680178e-07,
      "loss": 0.3213,
      "reward": 0.424665205180645,
      "reward_std": 0.2092951312661171,
      "rewards/accuracy_reward": 0.0959821492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3286830484867096,
      "step": 709
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1668.0871276855469,
      "epoch": 0.21208274214024345,
      "grad_norm": 22.1900577545166,
      "kl": 3.0703125,
      "learning_rate": 4.811192810947523e-07,
      "loss": 0.2699,
      "reward": 0.4073660895228386,
      "reward_std": 0.19444474577903748,
      "rewards/accuracy_reward": 0.08258928963914514,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3247767984867096,
      "step": 710
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1588.9754943847656,
      "epoch": 0.21238145022776492,
      "grad_norm": 9.09578800201416,
      "kl": 3.46875,
      "learning_rate": 4.810197452863611e-07,
      "loss": 0.3286,
      "reward": 0.3856026902794838,
      "reward_std": 0.1920848786830902,
      "rewards/accuracy_reward": 0.05133928847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3342634066939354,
      "step": 711
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1494.9420166015625,
      "epoch": 0.2126801583152864,
      "grad_norm": 19.644140243530273,
      "kl": 3.8671875,
      "learning_rate": 4.809199581511292e-07,
      "loss": 0.3953,
      "reward": 0.4508928805589676,
      "reward_std": 0.22199459746479988,
      "rewards/accuracy_reward": 0.1093750074505806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3415178656578064,
      "step": 712
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1640.3125610351562,
      "epoch": 0.21297886640280786,
      "grad_norm": 30.95254898071289,
      "kl": 4.4453125,
      "learning_rate": 4.808199197976156e-07,
      "loss": 0.3397,
      "reward": 0.4257812723517418,
      "reward_std": 0.1747324801981449,
      "rewards/accuracy_reward": 0.11383929080329835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3119419664144516,
      "step": 713
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1628.8103332519531,
      "epoch": 0.21327757449032933,
      "grad_norm": 148.57635498046875,
      "kl": 9.671875,
      "learning_rate": 4.80719630334652e-07,
      "loss": 0.5103,
      "reward": 0.3643973395228386,
      "reward_std": 0.19513047859072685,
      "rewards/accuracy_reward": 0.046875003492459655,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3175223395228386,
      "step": 714
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1615.200927734375,
      "epoch": 0.2135762825778508,
      "grad_norm": 53.69288635253906,
      "kl": 5.2109375,
      "learning_rate": 4.806190898713435e-07,
      "loss": 0.3921,
      "reward": 0.3510044813156128,
      "reward_std": 0.22202418744564056,
      "rewards/accuracy_reward": 0.04017857275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3108258992433548,
      "step": 715
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1625.6541137695312,
      "epoch": 0.21387499066537227,
      "grad_norm": 46.037628173828125,
      "kl": 5.1953125,
      "learning_rate": 4.805182985170684e-07,
      "loss": 0.4016,
      "reward": 0.415736623108387,
      "reward_std": 0.20253733545541763,
      "rewards/accuracy_reward": 0.10714285937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3085937649011612,
      "step": 716
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1598.2835693359375,
      "epoch": 0.21417369875289374,
      "grad_norm": 7.247920989990234,
      "kl": 4.4140625,
      "learning_rate": 4.804172563814777e-07,
      "loss": 0.379,
      "reward": 0.3733259066939354,
      "reward_std": 0.19638732820749283,
      "rewards/accuracy_reward": 0.046875003492459655,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3264509066939354,
      "step": 717
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1609.94873046875,
      "epoch": 0.21447240684041521,
      "grad_norm": 23.84544563293457,
      "kl": 4.33203125,
      "learning_rate": 4.803159635744954e-07,
      "loss": 0.349,
      "reward": 0.392857164144516,
      "reward_std": 0.21716271713376045,
      "rewards/accuracy_reward": 0.06696428684517741,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.325892873108387,
      "step": 718
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1632.18310546875,
      "epoch": 0.21477111492793668,
      "grad_norm": 45.51645278930664,
      "kl": 4.6953125,
      "learning_rate": 4.80214420206318e-07,
      "loss": 0.3727,
      "reward": 0.3839285895228386,
      "reward_std": 0.17488805204629898,
      "rewards/accuracy_reward": 0.07812500232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3058035895228386,
      "step": 719
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1620.3371276855469,
      "epoch": 0.21506982301545816,
      "grad_norm": 5.110437870025635,
      "kl": 4.0,
      "learning_rate": 4.801126263874151e-07,
      "loss": 0.3379,
      "reward": 0.3526785895228386,
      "reward_std": 0.21703342348337173,
      "rewards/accuracy_reward": 0.037946431431919336,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3147321566939354,
      "step": 720
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1612.5826721191406,
      "epoch": 0.21536853110297963,
      "grad_norm": 32.55705261230469,
      "kl": 3.30078125,
      "learning_rate": 4.80010582228528e-07,
      "loss": 0.3226,
      "reward": 0.3978794738650322,
      "reward_std": 0.1938558891415596,
      "rewards/accuracy_reward": 0.08035714644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3175223395228386,
      "step": 721
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1530.5402526855469,
      "epoch": 0.21566723919050107,
      "grad_norm": 48.4517936706543,
      "kl": 2.59765625,
      "learning_rate": 4.799082878406709e-07,
      "loss": 0.2687,
      "reward": 0.3537946566939354,
      "reward_std": 0.19409164786338806,
      "rewards/accuracy_reward": 0.022321429336443543,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3314732313156128,
      "step": 722
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1551.0871276855469,
      "epoch": 0.21596594727802254,
      "grad_norm": 52.26213455200195,
      "kl": 2.74609375,
      "learning_rate": 4.7980574333513e-07,
      "loss": 0.2981,
      "reward": 0.4709821566939354,
      "reward_std": 0.18524416908621788,
      "rewards/accuracy_reward": 0.1361607201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3348214477300644,
      "step": 723
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1634.4308471679688,
      "epoch": 0.216264655365544,
      "grad_norm": 24.10224723815918,
      "kl": 3.08984375,
      "learning_rate": 4.797029488234639e-07,
      "loss": 0.2747,
      "reward": 0.376116082072258,
      "reward_std": 0.19443616643548012,
      "rewards/accuracy_reward": 0.05357143026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3225446492433548,
      "step": 724
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1652.0201721191406,
      "epoch": 0.21656336345306548,
      "grad_norm": 28.628019332885742,
      "kl": 2.87890625,
      "learning_rate": 4.795999044175026e-07,
      "loss": 0.2707,
      "reward": 0.3978794813156128,
      "reward_std": 0.18203013390302658,
      "rewards/accuracy_reward": 0.0870535746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3108259066939354,
      "step": 725
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1641.5246276855469,
      "epoch": 0.21686207154058695,
      "grad_norm": 14.411606788635254,
      "kl": 3.625,
      "learning_rate": 4.794966102293486e-07,
      "loss": 0.3369,
      "reward": 0.3984375223517418,
      "reward_std": 0.2092544436454773,
      "rewards/accuracy_reward": 0.09821428847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3002232238650322,
      "step": 726
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1644.1763916015625,
      "epoch": 0.21716077962810842,
      "grad_norm": 22.025117874145508,
      "kl": 3.9765625,
      "learning_rate": 4.793930663713757e-07,
      "loss": 0.3312,
      "reward": 0.3532366156578064,
      "reward_std": 0.20541147887706757,
      "rewards/accuracy_reward": 0.02901785890571773,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3242187649011612,
      "step": 727
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1612.0201721191406,
      "epoch": 0.2174594877156299,
      "grad_norm": 54.84812545776367,
      "kl": 2.1015625,
      "learning_rate": 4.792892729562296e-07,
      "loss": 0.2401,
      "reward": 0.3895089477300644,
      "reward_std": 0.1836816817522049,
      "rewards/accuracy_reward": 0.04910714505240321,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3404018059372902,
      "step": 728
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1542.7656860351562,
      "epoch": 0.21775819580315137,
      "grad_norm": 23.361364364624023,
      "kl": 3.265625,
      "learning_rate": 4.791852300968273e-07,
      "loss": 0.2968,
      "reward": 0.4564732313156128,
      "reward_std": 0.19759487733244896,
      "rewards/accuracy_reward": 0.10491072130389512,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625223517418,
      "step": 729
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1552.6875305175781,
      "epoch": 0.21805690389067284,
      "grad_norm": 9.52374267578125,
      "kl": 3.7265625,
      "learning_rate": 4.790809379063572e-07,
      "loss": 0.3255,
      "reward": 0.4369419887661934,
      "reward_std": 0.19534089788794518,
      "rewards/accuracy_reward": 0.0915178619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.345424123108387,
      "step": 730
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1571.6161499023438,
      "epoch": 0.2183556119781943,
      "grad_norm": 29.042966842651367,
      "kl": 5.3984375,
      "learning_rate": 4.78976396498279e-07,
      "loss": 0.4659,
      "reward": 0.3856026977300644,
      "reward_std": 0.18712127581238747,
      "rewards/accuracy_reward": 0.05357143026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3320312574505806,
      "step": 731
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1587.2880249023438,
      "epoch": 0.21865432006571578,
      "grad_norm": 53.375308990478516,
      "kl": 5.578125,
      "learning_rate": 4.788716059863235e-07,
      "loss": 0.4411,
      "reward": 0.399553582072258,
      "reward_std": 0.21364906057715416,
      "rewards/accuracy_reward": 0.07142857578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3281250074505806,
      "step": 732
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1539.9375915527344,
      "epoch": 0.21895302815323725,
      "grad_norm": 48.08573913574219,
      "kl": 4.80859375,
      "learning_rate": 4.787665664844925e-07,
      "loss": 0.3709,
      "reward": 0.4419643059372902,
      "reward_std": 0.2109740674495697,
      "rewards/accuracy_reward": 0.08482143376022577,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.357142873108387,
      "step": 733
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1576.7277221679688,
      "epoch": 0.21925173624075872,
      "grad_norm": 22.21114158630371,
      "kl": 4.765625,
      "learning_rate": 4.786612781070587e-07,
      "loss": 0.4177,
      "reward": 0.3794642984867096,
      "reward_std": 0.19743217527866364,
      "rewards/accuracy_reward": 0.05133928661234677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3281250149011612,
      "step": 734
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1646.8014221191406,
      "epoch": 0.2195504443282802,
      "grad_norm": 89.7652816772461,
      "kl": 5.9375,
      "learning_rate": 4.785557409685656e-07,
      "loss": 0.4095,
      "reward": 0.4319196715950966,
      "reward_std": 0.24269666522741318,
      "rewards/accuracy_reward": 0.12500000861473382,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.3046875149011612,
      "step": 735
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1596.9152526855469,
      "epoch": 0.21984915241580166,
      "grad_norm": 18.36240005493164,
      "kl": 4.35546875,
      "learning_rate": 4.784499551838271e-07,
      "loss": 0.3542,
      "reward": 0.3956473395228386,
      "reward_std": 0.20697414502501488,
      "rewards/accuracy_reward": 0.06250000116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3331473395228386,
      "step": 736
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1573.8416137695312,
      "epoch": 0.22014786050332313,
      "grad_norm": 19.221790313720703,
      "kl": 4.40234375,
      "learning_rate": 4.783439208679279e-07,
      "loss": 0.3774,
      "reward": 0.3638393059372902,
      "reward_std": 0.1808967962861061,
      "rewards/accuracy_reward": 0.042410716181620955,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3214285895228386,
      "step": 737
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1573.5603332519531,
      "epoch": 0.2204465685908446,
      "grad_norm": 13.014540672302246,
      "kl": 3.96875,
      "learning_rate": 4.782376381362229e-07,
      "loss": 0.3688,
      "reward": 0.3364955559372902,
      "reward_std": 0.19221926480531693,
      "rewards/accuracy_reward": 0.0133928582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3231026977300644,
      "step": 738
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1618.7344360351562,
      "epoch": 0.22074527667836608,
      "grad_norm": 9.605168342590332,
      "kl": 4.51953125,
      "learning_rate": 4.781311071043373e-07,
      "loss": 0.3848,
      "reward": 0.3777901977300644,
      "reward_std": 0.24306915700435638,
      "rewards/accuracy_reward": 0.06696428963914514,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3108259066939354,
      "step": 739
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1596.5603332519531,
      "epoch": 0.22104398476588755,
      "grad_norm": 39.075279235839844,
      "kl": 3.22265625,
      "learning_rate": 4.780243278881664e-07,
      "loss": 0.3355,
      "reward": 0.465959832072258,
      "reward_std": 0.17779749631881714,
      "rewards/accuracy_reward": 0.12276786426082253,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3431919887661934,
      "step": 740
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1542.259033203125,
      "epoch": 0.22134269285340902,
      "grad_norm": 57.15321350097656,
      "kl": 2.97265625,
      "learning_rate": 4.779173006038756e-07,
      "loss": 0.3336,
      "reward": 0.3856026977300644,
      "reward_std": 0.21280289441347122,
      "rewards/accuracy_reward": 0.035714287078008056,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3498884066939354,
      "step": 741
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1579.43310546875,
      "epoch": 0.2216414009409305,
      "grad_norm": 42.650882720947266,
      "kl": 3.0234375,
      "learning_rate": 4.778100253679e-07,
      "loss": 0.3209,
      "reward": 0.3872768059372902,
      "reward_std": 0.2000235915184021,
      "rewards/accuracy_reward": 0.05580357206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3314732313156128,
      "step": 742
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1604.0491943359375,
      "epoch": 0.22194010902845193,
      "grad_norm": 29.784175872802734,
      "kl": 3.078125,
      "learning_rate": 4.777025022969446e-07,
      "loss": 0.295,
      "reward": 0.3325893059372902,
      "reward_std": 0.18082531541585922,
      "rewards/accuracy_reward": 0.008928572060540318,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3236607313156128,
      "step": 743
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1539.9911193847656,
      "epoch": 0.2222388171159734,
      "grad_norm": 48.358585357666016,
      "kl": 3.24609375,
      "learning_rate": 4.775947315079839e-07,
      "loss": 0.3469,
      "reward": 0.498325914144516,
      "reward_std": 0.2129516378045082,
      "rewards/accuracy_reward": 0.1406250111758709,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.357700914144516,
      "step": 744
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1528.5045166015625,
      "epoch": 0.22253752520349487,
      "grad_norm": 29.147829055786133,
      "kl": 3.35546875,
      "learning_rate": 4.774867131182621e-07,
      "loss": 0.3464,
      "reward": 0.3800223395228386,
      "reward_std": 0.2092142403125763,
      "rewards/accuracy_reward": 0.04464286006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3353794738650322,
      "step": 745
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1569.8348999023438,
      "epoch": 0.22283623329101634,
      "grad_norm": 10.879770278930664,
      "kl": 4.31640625,
      "learning_rate": 4.773784472452924e-07,
      "loss": 0.3961,
      "reward": 0.3750000149011612,
      "reward_std": 0.24804991483688354,
      "rewards/accuracy_reward": 0.040178573690354824,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3348214402794838,
      "step": 746
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1629.0268859863281,
      "epoch": 0.22313494137853782,
      "grad_norm": 36.54946517944336,
      "kl": 3.03125,
      "learning_rate": 4.772699340068576e-07,
      "loss": 0.2981,
      "reward": 0.4241071566939354,
      "reward_std": 0.209950502961874,
      "rewards/accuracy_reward": 0.10044643469154835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3236607238650322,
      "step": 747
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1612.2768249511719,
      "epoch": 0.2234336494660593,
      "grad_norm": 35.734275817871094,
      "kl": 5.3828125,
      "learning_rate": 4.771611735210096e-07,
      "loss": 0.4463,
      "reward": 0.4123884066939354,
      "reward_std": 0.1869749054312706,
      "rewards/accuracy_reward": 0.08928571757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3231026902794838,
      "step": 748
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1578.7120971679688,
      "epoch": 0.22373235755358076,
      "grad_norm": 59.725196838378906,
      "kl": 5.7421875,
      "learning_rate": 4.770521659060689e-07,
      "loss": 0.4366,
      "reward": 0.3699776977300644,
      "reward_std": 0.23239155858755112,
      "rewards/accuracy_reward": 0.04687500186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3231026902794838,
      "step": 749
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1533.1741638183594,
      "epoch": 0.22403106564110223,
      "grad_norm": 119.00836181640625,
      "kl": 6.765625,
      "learning_rate": 4.769429112806251e-07,
      "loss": 0.4908,
      "reward": 0.4118303805589676,
      "reward_std": 0.24525899812579155,
      "rewards/accuracy_reward": 0.08928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3225446566939354,
      "step": 750
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1605.9933776855469,
      "epoch": 0.2243297737286237,
      "grad_norm": 89.06428527832031,
      "kl": 6.15625,
      "learning_rate": 4.768334097635368e-07,
      "loss": 0.4508,
      "reward": 0.3353794813156128,
      "reward_std": 0.21304966509342194,
      "rewards/accuracy_reward": 0.026785715948790312,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3085937649011612,
      "step": 751
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1620.4665832519531,
      "epoch": 0.22462848181614517,
      "grad_norm": 49.671043395996094,
      "kl": 5.2890625,
      "learning_rate": 4.767236614739306e-07,
      "loss": 0.4134,
      "reward": 0.3805803656578064,
      "reward_std": 0.1868327260017395,
      "rewards/accuracy_reward": 0.053571431431919336,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3270089477300644,
      "step": 752
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1662.2634582519531,
      "epoch": 0.22492718990366664,
      "grad_norm": 112.0594253540039,
      "kl": 6.609375,
      "learning_rate": 4.7661366653120204e-07,
      "loss": 0.4643,
      "reward": 0.4129464402794838,
      "reward_std": 0.1965065486729145,
      "rewards/accuracy_reward": 0.1049107201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3080357313156128,
      "step": 753
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1588.6473693847656,
      "epoch": 0.2252258979911881,
      "grad_norm": 62.234649658203125,
      "kl": 5.09375,
      "learning_rate": 4.7650342505501484e-07,
      "loss": 0.372,
      "reward": 0.4559151977300644,
      "reward_std": 0.21867735683918,
      "rewards/accuracy_reward": 0.12500000465661287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3309151902794838,
      "step": 754
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1623.1004943847656,
      "epoch": 0.22552460607870958,
      "grad_norm": 38.875701904296875,
      "kl": 3.05859375,
      "learning_rate": 4.763929371653009e-07,
      "loss": 0.3064,
      "reward": 0.3130580484867096,
      "reward_std": 0.1689031682908535,
      "rewards/accuracy_reward": 0.0022321429569274187,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3108259066939354,
      "step": 755
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1610.5848999023438,
      "epoch": 0.22582331416623105,
      "grad_norm": 45.475440979003906,
      "kl": 2.921875,
      "learning_rate": 4.762822029822603e-07,
      "loss": 0.2885,
      "reward": 0.3738839402794838,
      "reward_std": 0.20382528752088547,
      "rewards/accuracy_reward": 0.04464285937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.329241082072258,
      "step": 756
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1690.5603332519531,
      "epoch": 0.22612202225375252,
      "grad_norm": 32.88844299316406,
      "kl": 3.09765625,
      "learning_rate": 4.761712226263608e-07,
      "loss": 0.2894,
      "reward": 0.3738839477300644,
      "reward_std": 0.23021108284592628,
      "rewards/accuracy_reward": 0.0758928619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.297991082072258,
      "step": 757
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1685.0938110351562,
      "epoch": 0.226420730341274,
      "grad_norm": 42.43009567260742,
      "kl": 4.52734375,
      "learning_rate": 4.760599962183383e-07,
      "loss": 0.3366,
      "reward": 0.4017857387661934,
      "reward_std": 0.1825690008699894,
      "rewards/accuracy_reward": 0.09151786169968545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.310267873108387,
      "step": 758
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1657.6317443847656,
      "epoch": 0.22671943842879547,
      "grad_norm": 26.399612426757812,
      "kl": 4.33984375,
      "learning_rate": 4.7594852387919607e-07,
      "loss": 0.317,
      "reward": 0.3169643059372902,
      "reward_std": 0.16340947151184082,
      "rewards/accuracy_reward": 0.004464285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3125000149011612,
      "step": 759
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1637.7210388183594,
      "epoch": 0.22701814651631694,
      "grad_norm": 3.753678560256958,
      "kl": 3.84375,
      "learning_rate": 4.758368057302053e-07,
      "loss": 0.3228,
      "reward": 0.4821428805589676,
      "reward_std": 0.18982116132974625,
      "rewards/accuracy_reward": 0.16741072200238705,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3147321492433548,
      "step": 760
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1633.2210388183594,
      "epoch": 0.2273168546038384,
      "grad_norm": 7.221460819244385,
      "kl": 3.71875,
      "learning_rate": 4.7572484189290424e-07,
      "loss": 0.3145,
      "reward": 0.3404017984867096,
      "reward_std": 0.2006298042833805,
      "rewards/accuracy_reward": 0.020089285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3203125149011612,
      "step": 761
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1611.0067443847656,
      "epoch": 0.22761556269135988,
      "grad_norm": 6.448947906494141,
      "kl": 3.70703125,
      "learning_rate": 4.7561263248909866e-07,
      "loss": 0.326,
      "reward": 0.4648437723517418,
      "reward_std": 0.1962817832827568,
      "rewards/accuracy_reward": 0.14062500861473382,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3242187649011612,
      "step": 762
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1521.7723999023438,
      "epoch": 0.22791427077888135,
      "grad_norm": 17.45954704284668,
      "kl": 3.74609375,
      "learning_rate": 4.7550017764086126e-07,
      "loss": 0.3607,
      "reward": 0.4581473395228386,
      "reward_std": 0.21560953557491302,
      "rewards/accuracy_reward": 0.1160714365541935,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.3398437723517418,
      "step": 763
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1551.5045166015625,
      "epoch": 0.22821297886640282,
      "grad_norm": 9.536385536193848,
      "kl": 4.9765625,
      "learning_rate": 4.75387477470532e-07,
      "loss": 0.4627,
      "reward": 0.4469866305589676,
      "reward_std": 0.16817675530910492,
      "rewards/accuracy_reward": 0.10937500488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3376116305589676,
      "step": 764
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1631.1429138183594,
      "epoch": 0.22851168695392426,
      "grad_norm": 30.501157760620117,
      "kl": 4.90625,
      "learning_rate": 4.7527453210071764e-07,
      "loss": 0.3982,
      "reward": 0.3850446566939354,
      "reward_std": 0.20481175929307938,
      "rewards/accuracy_reward": 0.07142857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3136160895228386,
      "step": 765
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1486.0826416015625,
      "epoch": 0.22881039504144574,
      "grad_norm": 49.96992874145508,
      "kl": 5.0625,
      "learning_rate": 4.751613416542917e-07,
      "loss": 0.4095,
      "reward": 0.487723246216774,
      "reward_std": 0.25005093216896057,
      "rewards/accuracy_reward": 0.1361607201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625149011612,
      "step": 766
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1432.9822082519531,
      "epoch": 0.2291091031289672,
      "grad_norm": 9.327000617980957,
      "kl": 4.14453125,
      "learning_rate": 4.7504790625439405e-07,
      "loss": 0.3766,
      "reward": 0.5000000074505806,
      "reward_std": 0.21503648161888123,
      "rewards/accuracy_reward": 0.1383928619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3616071566939354,
      "step": 767
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1548.3125915527344,
      "epoch": 0.22940781121648868,
      "grad_norm": 29.45963478088379,
      "kl": 3.85546875,
      "learning_rate": 4.7493422602443156e-07,
      "loss": 0.3849,
      "reward": 0.3934151977300644,
      "reward_std": 0.18705883249640465,
      "rewards/accuracy_reward": 0.0513392873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.342075914144516,
      "step": 768
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1619.2947082519531,
      "epoch": 0.22970651930401015,
      "grad_norm": 13.68345832824707,
      "kl": 4.23046875,
      "learning_rate": 4.748203010880771e-07,
      "loss": 0.3538,
      "reward": 0.4804687798023224,
      "reward_std": 0.21931473538279533,
      "rewards/accuracy_reward": 0.14285715110599995,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.337611623108387,
      "step": 769
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1569.4398193359375,
      "epoch": 0.23000522739153162,
      "grad_norm": 11.466922760009766,
      "kl": 4.3203125,
      "learning_rate": 4.747061315692698e-07,
      "loss": 0.3548,
      "reward": 0.4296875223517418,
      "reward_std": 0.19274571537971497,
      "rewards/accuracy_reward": 0.10267857508733869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3270089402794838,
      "step": 770
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1649.4241638183594,
      "epoch": 0.2303039354790531,
      "grad_norm": 8.567787170410156,
      "kl": 3.92578125,
      "learning_rate": 4.745917175922149e-07,
      "loss": 0.3335,
      "reward": 0.3660714402794838,
      "reward_std": 0.20628343150019646,
      "rewards/accuracy_reward": 0.05803571594879031,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3080357313156128,
      "step": 771
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1638.1473999023438,
      "epoch": 0.23060264356657456,
      "grad_norm": 11.818560600280762,
      "kl": 4.28125,
      "learning_rate": 4.744770592813836e-07,
      "loss": 0.3427,
      "reward": 0.3995535895228386,
      "reward_std": 0.18462025746703148,
      "rewards/accuracy_reward": 0.08035714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3191964402794838,
      "step": 772
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1551.8348999023438,
      "epoch": 0.23090135165409603,
      "grad_norm": 22.21766471862793,
      "kl": 3.640625,
      "learning_rate": 4.743621567615129e-07,
      "loss": 0.3363,
      "reward": 0.3671875223517418,
      "reward_std": 0.21998000144958496,
      "rewards/accuracy_reward": 0.03348214412108064,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.3314732313156128,
      "step": 773
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1607.6272888183594,
      "epoch": 0.2312000597416175,
      "grad_norm": 21.037748336791992,
      "kl": 4.09765625,
      "learning_rate": 4.742470101576056e-07,
      "loss": 0.3941,
      "reward": 0.381138414144516,
      "reward_std": 0.16437776759266853,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.309709832072258,
      "step": 774
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1606.3371276855469,
      "epoch": 0.23149876782913897,
      "grad_norm": 4.347177028656006,
      "kl": 4.0859375,
      "learning_rate": 4.7413161959492976e-07,
      "loss": 0.3588,
      "reward": 0.4536830633878708,
      "reward_std": 0.19590357691049576,
      "rewards/accuracy_reward": 0.1272321492433548,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.3242187649011612,
      "step": 775
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1500.8683471679688,
      "epoch": 0.23179747591666044,
      "grad_norm": 18.262115478515625,
      "kl": 3.125,
      "learning_rate": 4.74015985199019e-07,
      "loss": 0.2909,
      "reward": 0.4386160969734192,
      "reward_std": 0.23202602565288544,
      "rewards/accuracy_reward": 0.08482143236324191,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3537946566939354,
      "step": 776
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1602.9085693359375,
      "epoch": 0.23209618400418192,
      "grad_norm": 29.736433029174805,
      "kl": 3.7265625,
      "learning_rate": 4.7390010709567236e-07,
      "loss": 0.3683,
      "reward": 0.3660714477300644,
      "reward_std": 0.1861266754567623,
      "rewards/accuracy_reward": 0.044642857974395156,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.321428582072258,
      "step": 777
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1493.3973693847656,
      "epoch": 0.2323948920917034,
      "grad_norm": 9.16287612915039,
      "kl": 4.00390625,
      "learning_rate": 4.7378398541095376e-07,
      "loss": 0.3557,
      "reward": 0.4475446566939354,
      "reward_std": 0.21805253997445107,
      "rewards/accuracy_reward": 0.10714286006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3404017984867096,
      "step": 778
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1479.3304138183594,
      "epoch": 0.23269360017922486,
      "grad_norm": 11.847085952758789,
      "kl": 3.7890625,
      "learning_rate": 4.736676202711922e-07,
      "loss": 0.3868,
      "reward": 0.411830373108387,
      "reward_std": 0.2492023967206478,
      "rewards/accuracy_reward": 0.06473214691504836,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3470982313156128,
      "step": 779
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1568.9041137695312,
      "epoch": 0.23299230826674633,
      "grad_norm": 36.21589660644531,
      "kl": 5.4921875,
      "learning_rate": 4.7355101180298153e-07,
      "loss": 0.4737,
      "reward": 0.4107142984867096,
      "reward_std": 0.22869988530874252,
      "rewards/accuracy_reward": 0.08035715040750802,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3303571566939354,
      "step": 780
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1622.7523193359375,
      "epoch": 0.2332910163542678,
      "grad_norm": 40.842498779296875,
      "kl": 5.203125,
      "learning_rate": 4.734341601331804e-07,
      "loss": 0.4187,
      "reward": 0.3945312723517418,
      "reward_std": 0.19413265585899353,
      "rewards/accuracy_reward": 0.082589291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3119419813156128,
      "step": 781
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1591.3236999511719,
      "epoch": 0.23358972444178927,
      "grad_norm": 11.995438575744629,
      "kl": 4.34765625,
      "learning_rate": 4.733170653889119e-07,
      "loss": 0.3508,
      "reward": 0.4469866156578064,
      "reward_std": 0.19743777066469193,
      "rewards/accuracy_reward": 0.1071428656578064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3398437649011612,
      "step": 782
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1600.5603332519531,
      "epoch": 0.23388843252931074,
      "grad_norm": 32.80662155151367,
      "kl": 4.9140625,
      "learning_rate": 4.7319972769756376e-07,
      "loss": 0.387,
      "reward": 0.4051339477300644,
      "reward_std": 0.20813065022230148,
      "rewards/accuracy_reward": 0.0848214328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3203125149011612,
      "step": 783
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1533.3884582519531,
      "epoch": 0.2341871406168322,
      "grad_norm": 49.4031867980957,
      "kl": 3.25390625,
      "learning_rate": 4.730821471867878e-07,
      "loss": 0.3438,
      "reward": 0.3973214477300644,
      "reward_std": 0.20391527190804482,
      "rewards/accuracy_reward": 0.044642859138548374,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.352678582072258,
      "step": 784
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1595.3460693359375,
      "epoch": 0.23448584870435368,
      "grad_norm": 23.638885498046875,
      "kl": 3.921875,
      "learning_rate": 4.729643239845001e-07,
      "loss": 0.385,
      "reward": 0.4196428805589676,
      "reward_std": 0.1922941543161869,
      "rewards/accuracy_reward": 0.08258928963914514,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3370535895228386,
      "step": 785
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1651.57373046875,
      "epoch": 0.23478455679187513,
      "grad_norm": 17.6804141998291,
      "kl": 4.65625,
      "learning_rate": 4.728462582188807e-07,
      "loss": 0.3864,
      "reward": 0.348772332072258,
      "reward_std": 0.17659789323806763,
      "rewards/accuracy_reward": 0.04241071501746774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.306361623108387,
      "step": 786
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1538.9264221191406,
      "epoch": 0.2350832648793966,
      "grad_norm": 10.762480735778809,
      "kl": 4.3046875,
      "learning_rate": 4.7272795001837373e-07,
      "loss": 0.3689,
      "reward": 0.3677455559372902,
      "reward_std": 0.2079564332962036,
      "rewards/accuracy_reward": 0.03348214481957257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3342634066939354,
      "step": 787
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1501.1272888183594,
      "epoch": 0.23538197296691807,
      "grad_norm": 18.453758239746094,
      "kl": 4.10546875,
      "learning_rate": 4.7260939951168677e-07,
      "loss": 0.383,
      "reward": 0.5106026977300644,
      "reward_std": 0.21142082661390305,
      "rewards/accuracy_reward": 0.1785714365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3320312574505806,
      "step": 788
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1579.0647888183594,
      "epoch": 0.23568068105443954,
      "grad_norm": 15.09395980834961,
      "kl": 4.5546875,
      "learning_rate": 4.7249060682779116e-07,
      "loss": 0.3899,
      "reward": 0.3839285895228386,
      "reward_std": 0.22301950678229332,
      "rewards/accuracy_reward": 0.06250000325962901,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3214285895228386,
      "step": 789
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1492.9911193847656,
      "epoch": 0.235979389141961,
      "grad_norm": 35.8751220703125,
      "kl": 3.74609375,
      "learning_rate": 4.7237157209592185e-07,
      "loss": 0.4094,
      "reward": 0.4553571566939354,
      "reward_std": 0.2220921292901039,
      "rewards/accuracy_reward": 0.11830357951112092,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3370535895228386,
      "step": 790
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1661.5804138183594,
      "epoch": 0.23627809722948248,
      "grad_norm": 7.326234817504883,
      "kl": 4.40234375,
      "learning_rate": 4.722522954455768e-07,
      "loss": 0.3756,
      "reward": 0.451450914144516,
      "reward_std": 0.1889878623187542,
      "rewards/accuracy_reward": 0.14955357951112092,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.301897332072258,
      "step": 791
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1653.5648193359375,
      "epoch": 0.23657680531700395,
      "grad_norm": 22.318750381469727,
      "kl": 5.0234375,
      "learning_rate": 4.7213277700651733e-07,
      "loss": 0.4096,
      "reward": 0.3549107313156128,
      "reward_std": 0.21945121884346008,
      "rewards/accuracy_reward": 0.05357143026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3013392984867096,
      "step": 792
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1680.6986999511719,
      "epoch": 0.23687551340452542,
      "grad_norm": 46.658878326416016,
      "kl": 4.9921875,
      "learning_rate": 4.7201301690876787e-07,
      "loss": 0.355,
      "reward": 0.3325892984867096,
      "reward_std": 0.2030538022518158,
      "rewards/accuracy_reward": 0.026785715017467737,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.305803582072258,
      "step": 793
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1473.0782165527344,
      "epoch": 0.2371742214920469,
      "grad_norm": 12.9006986618042,
      "kl": 3.78515625,
      "learning_rate": 4.718930152826156e-07,
      "loss": 0.3483,
      "reward": 0.5150669813156128,
      "reward_std": 0.25594663619995117,
      "rewards/accuracy_reward": 0.1562500074505806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3588169738650322,
      "step": 794
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1606.0491638183594,
      "epoch": 0.23747292957956836,
      "grad_norm": 17.206375122070312,
      "kl": 4.75,
      "learning_rate": 4.7177277225861064e-07,
      "loss": 0.3999,
      "reward": 0.4123884215950966,
      "reward_std": 0.21441031619906425,
      "rewards/accuracy_reward": 0.10491072246804833,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3074776902794838,
      "step": 795
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1437.5514221191406,
      "epoch": 0.23777163766708984,
      "grad_norm": 60.66862487792969,
      "kl": 2.91796875,
      "learning_rate": 4.7165228796756547e-07,
      "loss": 0.3661,
      "reward": 0.4101562723517418,
      "reward_std": 0.20885229483246803,
      "rewards/accuracy_reward": 0.053571432596072555,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3565848469734192,
      "step": 796
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1512.5826416015625,
      "epoch": 0.2380703457546113,
      "grad_norm": 56.03329086303711,
      "kl": 3.19921875,
      "learning_rate": 4.7153156254055524e-07,
      "loss": 0.3607,
      "reward": 0.3459821566939354,
      "reward_std": 0.1770428754389286,
      "rewards/accuracy_reward": 0.008928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3370535895228386,
      "step": 797
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1539.58935546875,
      "epoch": 0.23836905384213278,
      "grad_norm": 40.79336929321289,
      "kl": 3.14453125,
      "learning_rate": 4.7141059610891743e-07,
      "loss": 0.3168,
      "reward": 0.3989955484867096,
      "reward_std": 0.244227334856987,
      "rewards/accuracy_reward": 0.058035718044266105,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.340959832072258,
      "step": 798
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1579.1719360351562,
      "epoch": 0.23866776192965425,
      "grad_norm": 11.844035148620605,
      "kl": 4.140625,
      "learning_rate": 4.7128938880425157e-07,
      "loss": 0.3777,
      "reward": 0.4402901977300644,
      "reward_std": 0.23507649451494217,
      "rewards/accuracy_reward": 0.1183035746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3219866305589676,
      "step": 799
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1451.9308776855469,
      "epoch": 0.23896647001717572,
      "grad_norm": 20.141048431396484,
      "kl": 3.82421875,
      "learning_rate": 4.711679407584194e-07,
      "loss": 0.3857,
      "reward": 0.439732164144516,
      "reward_std": 0.20673994347453117,
      "rewards/accuracy_reward": 0.09821428940631449,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3415178805589676,
      "step": 800
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1562.0291137695312,
      "epoch": 0.2392651781046972,
      "grad_norm": 19.112764358520508,
      "kl": 3.69140625,
      "learning_rate": 4.710462521035445e-07,
      "loss": 0.3362,
      "reward": 0.3878348395228386,
      "reward_std": 0.17721455916762352,
      "rewards/accuracy_reward": 0.046875000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3409598395228386,
      "step": 801
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1535.1808776855469,
      "epoch": 0.23956388619221866,
      "grad_norm": 34.41617202758789,
      "kl": 4.96875,
      "learning_rate": 4.709243229720122e-07,
      "loss": 0.4204,
      "reward": 0.3705357313156128,
      "reward_std": 0.2245902381837368,
      "rewards/accuracy_reward": 0.04910714481957257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3214285895228386,
      "step": 802
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1520.9219665527344,
      "epoch": 0.23986259427974013,
      "grad_norm": 18.671527862548828,
      "kl": 4.44921875,
      "learning_rate": 4.7080215349646934e-07,
      "loss": 0.4096,
      "reward": 0.4335937723517418,
      "reward_std": 0.18695341795682907,
      "rewards/accuracy_reward": 0.08928571850992739,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3443080559372902,
      "step": 803
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1535.9554748535156,
      "epoch": 0.2401613023672616,
      "grad_norm": 21.111595153808594,
      "kl": 4.30859375,
      "learning_rate": 4.7067974380982443e-07,
      "loss": 0.359,
      "reward": 0.3632812649011612,
      "reward_std": 0.22077381238341331,
      "rewards/accuracy_reward": 0.026785715715959668,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3364955484867096,
      "step": 804
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1561.138427734375,
      "epoch": 0.24046001045478307,
      "grad_norm": 31.330413818359375,
      "kl": 5.0390625,
      "learning_rate": 4.705570940452472e-07,
      "loss": 0.4185,
      "reward": 0.4620535969734192,
      "reward_std": 0.22796199470758438,
      "rewards/accuracy_reward": 0.14285715017467737,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3191964328289032,
      "step": 805
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1521.6630249023438,
      "epoch": 0.24075871854230455,
      "grad_norm": 34.99658203125,
      "kl": 5.0234375,
      "learning_rate": 4.7043420433616865e-07,
      "loss": 0.4141,
      "reward": 0.4787946790456772,
      "reward_std": 0.25611385703086853,
      "rewards/accuracy_reward": 0.15401786682195961,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3247767984867096,
      "step": 806
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1560.6741638183594,
      "epoch": 0.24105742662982602,
      "grad_norm": 29.833375930786133,
      "kl": 4.984375,
      "learning_rate": 4.7031107481628053e-07,
      "loss": 0.4151,
      "reward": 0.416294664144516,
      "reward_std": 0.22185399010777473,
      "rewards/accuracy_reward": 0.08928571874275804,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3270089402794838,
      "step": 807
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1542.5402526855469,
      "epoch": 0.24135613471734746,
      "grad_norm": 24.508716583251953,
      "kl": 5.1171875,
      "learning_rate": 4.7018770561953593e-07,
      "loss": 0.4476,
      "reward": 0.3554687723517418,
      "reward_std": 0.1981598436832428,
      "rewards/accuracy_reward": 0.024553571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3309151977300644,
      "step": 808
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1612.4085388183594,
      "epoch": 0.24165484280486893,
      "grad_norm": 25.000417709350586,
      "kl": 5.125,
      "learning_rate": 4.7006409688014823e-07,
      "loss": 0.4508,
      "reward": 0.3320312649011612,
      "reward_std": 0.21699140593409538,
      "rewards/accuracy_reward": 0.03125000139698386,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3007812649011612,
      "step": 809
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1585.2880249023438,
      "epoch": 0.2419535508923904,
      "grad_norm": 19.80357551574707,
      "kl": 3.87109375,
      "learning_rate": 4.6994024873259174e-07,
      "loss": 0.346,
      "reward": 0.3911830484867096,
      "reward_std": 0.1907978244125843,
      "rewards/accuracy_reward": 0.08035714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3108259066939354,
      "step": 810
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1495.6741638183594,
      "epoch": 0.24225225897991187,
      "grad_norm": 53.10260009765625,
      "kl": 3.66796875,
      "learning_rate": 4.698161613116012e-07,
      "loss": 0.3968,
      "reward": 0.423549123108387,
      "reward_std": 0.21475359424948692,
      "rewards/accuracy_reward": 0.0892857164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3342634066939354,
      "step": 811
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1572.0246276855469,
      "epoch": 0.24255096706743334,
      "grad_norm": 38.460811614990234,
      "kl": 3.2109375,
      "learning_rate": 4.6969183475217135e-07,
      "loss": 0.3144,
      "reward": 0.356584832072258,
      "reward_std": 0.1807586532086134,
      "rewards/accuracy_reward": 0.026785715017467737,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3297991156578064,
      "step": 812
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1575.4688415527344,
      "epoch": 0.2428496751549548,
      "grad_norm": 25.007333755493164,
      "kl": 3.671875,
      "learning_rate": 4.695672691895574e-07,
      "loss": 0.3311,
      "reward": 0.3867187798023224,
      "reward_std": 0.20576754212379456,
      "rewards/accuracy_reward": 0.06250000488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3242187649011612,
      "step": 813
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1569.9554138183594,
      "epoch": 0.24314838324247628,
      "grad_norm": 31.91322135925293,
      "kl": 3.52734375,
      "learning_rate": 4.694424647592745e-07,
      "loss": 0.3552,
      "reward": 0.3543526902794838,
      "reward_std": 0.20612715929746628,
      "rewards/accuracy_reward": 0.03125000209547579,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3231026902794838,
      "step": 814
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1570.6808471679688,
      "epoch": 0.24344709132999776,
      "grad_norm": 17.008459091186523,
      "kl": 4.4609375,
      "learning_rate": 4.693174215970976e-07,
      "loss": 0.3895,
      "reward": 0.368861623108387,
      "reward_std": 0.2210695967078209,
      "rewards/accuracy_reward": 0.060267859837040305,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3085937574505806,
      "step": 815
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1614.9130249023438,
      "epoch": 0.24374579941751923,
      "grad_norm": 47.531349182128906,
      "kl": 5.21875,
      "learning_rate": 4.691921398390614e-07,
      "loss": 0.4317,
      "reward": 0.405133955180645,
      "reward_std": 0.21911616995930672,
      "rewards/accuracy_reward": 0.10714286006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2979910895228386,
      "step": 816
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1596.9085693359375,
      "epoch": 0.2440445075050407,
      "grad_norm": 27.729637145996094,
      "kl": 3.40234375,
      "learning_rate": 4.690666196214602e-07,
      "loss": 0.3009,
      "reward": 0.407924123108387,
      "reward_std": 0.1909848153591156,
      "rewards/accuracy_reward": 0.08482143143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3231026902794838,
      "step": 817
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1739.05810546875,
      "epoch": 0.24434321559256217,
      "grad_norm": 47.85274887084961,
      "kl": 4.9453125,
      "learning_rate": 4.689408610808477e-07,
      "loss": 0.3431,
      "reward": 0.294642873108387,
      "reward_std": 0.20507322624325752,
      "rewards/accuracy_reward": 0.022321430034935474,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2723214402794838,
      "step": 818
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1556.3772888183594,
      "epoch": 0.24464192368008364,
      "grad_norm": 32.593231201171875,
      "kl": 3.8984375,
      "learning_rate": 4.6881486435403694e-07,
      "loss": 0.3749,
      "reward": 0.466517873108387,
      "reward_std": 0.22306020930409431,
      "rewards/accuracy_reward": 0.1383928656578064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3281250074505806,
      "step": 819
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1637.7746276855469,
      "epoch": 0.2449406317676051,
      "grad_norm": 37.256160736083984,
      "kl": 4.6796875,
      "learning_rate": 4.6868862957809983e-07,
      "loss": 0.358,
      "reward": 0.3677455484867096,
      "reward_std": 0.24929990246891975,
      "rewards/accuracy_reward": 0.053571431431919336,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.314174123108387,
      "step": 820
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1587.24560546875,
      "epoch": 0.24523933985512658,
      "grad_norm": 10.461759567260742,
      "kl": 3.78515625,
      "learning_rate": 4.685621568903676e-07,
      "loss": 0.3312,
      "reward": 0.3325893133878708,
      "reward_std": 0.22354182228446007,
      "rewards/accuracy_reward": 0.033482143422588706,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2991071492433548,
      "step": 821
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1577.0647888183594,
      "epoch": 0.24553804794264805,
      "grad_norm": 5.627130508422852,
      "kl": 4.0078125,
      "learning_rate": 4.6843544642843003e-07,
      "loss": 0.3336,
      "reward": 0.4001116305589676,
      "reward_std": 0.18443744257092476,
      "rewards/accuracy_reward": 0.058035717345774174,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.3398437574505806,
      "step": 822
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1540.18310546875,
      "epoch": 0.24583675603016952,
      "grad_norm": 26.076204299926758,
      "kl": 5.05078125,
      "learning_rate": 4.683084983301357e-07,
      "loss": 0.4337,
      "reward": 0.391741082072258,
      "reward_std": 0.2098359428346157,
      "rewards/accuracy_reward": 0.07142857369035482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3203125149011612,
      "step": 823
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1632.0960388183594,
      "epoch": 0.246135464117691,
      "grad_norm": 59.55546951293945,
      "kl": 5.28125,
      "learning_rate": 4.681813127335917e-07,
      "loss": 0.3974,
      "reward": 0.376116082072258,
      "reward_std": 0.24109845235943794,
      "rewards/accuracy_reward": 0.06696428707800806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3091517984867096,
      "step": 824
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1578.1384582519531,
      "epoch": 0.24643417220521247,
      "grad_norm": 6.068005561828613,
      "kl": 4.609375,
      "learning_rate": 4.680538897771635e-07,
      "loss": 0.414,
      "reward": 0.3219866156578064,
      "reward_std": 0.1817588433623314,
      "rewards/accuracy_reward": 0.008928572060540318,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.313058041036129,
      "step": 825
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1615.4978332519531,
      "epoch": 0.24673288029273394,
      "grad_norm": 36.44014358520508,
      "kl": 4.6875,
      "learning_rate": 4.6792622959947493e-07,
      "loss": 0.3683,
      "reward": 0.4648437723517418,
      "reward_std": 0.18514490500092506,
      "rewards/accuracy_reward": 0.15178572246804833,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3130580484867096,
      "step": 826
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1645.665283203125,
      "epoch": 0.2470315883802554,
      "grad_norm": 36.95512008666992,
      "kl": 3.15625,
      "learning_rate": 4.677983323394075e-07,
      "loss": 0.2974,
      "reward": 0.385044664144516,
      "reward_std": 0.20364565402269363,
      "rewards/accuracy_reward": 0.0892857164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2957589402794838,
      "step": 827
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1528.716552734375,
      "epoch": 0.24733029646777688,
      "grad_norm": 79.9479751586914,
      "kl": 2.57421875,
      "learning_rate": 4.676701981361011e-07,
      "loss": 0.3113,
      "reward": 0.3811383992433548,
      "reward_std": 0.19384190067648888,
      "rewards/accuracy_reward": 0.04017857206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3409598395228386,
      "step": 828
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1628.4219360351562,
      "epoch": 0.24762900455529832,
      "grad_norm": 13.317482948303223,
      "kl": 4.09765625,
      "learning_rate": 4.6754182712895296e-07,
      "loss": 0.3446,
      "reward": 0.4034598395228386,
      "reward_std": 0.17611131817102432,
      "rewards/accuracy_reward": 0.082589291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3208705484867096,
      "step": 829
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1558.9397583007812,
      "epoch": 0.2479277126428198,
      "grad_norm": 64.40067291259766,
      "kl": 2.71484375,
      "learning_rate": 4.6741321945761845e-07,
      "loss": 0.3161,
      "reward": 0.3699776902794838,
      "reward_std": 0.18701285868883133,
      "rewards/accuracy_reward": 0.05133928661234677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3186383992433548,
      "step": 830
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1620.02685546875,
      "epoch": 0.24822642073034126,
      "grad_norm": 11.651524543762207,
      "kl": 4.09375,
      "learning_rate": 4.6728437526200987e-07,
      "loss": 0.3476,
      "reward": 0.3705357313156128,
      "reward_std": 0.20570750162005424,
      "rewards/accuracy_reward": 0.0625000016298145,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3080357313156128,
      "step": 831
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1573.74560546875,
      "epoch": 0.24852512881786273,
      "grad_norm": 12.349356651306152,
      "kl": 3.87109375,
      "learning_rate": 4.671552946822972e-07,
      "loss": 0.3305,
      "reward": 0.3906250149011612,
      "reward_std": 0.17414593696594238,
      "rewards/accuracy_reward": 0.060267859138548374,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3303571566939354,
      "step": 832
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1655.5514221191406,
      "epoch": 0.2488238369053842,
      "grad_norm": 16.102136611938477,
      "kl": 3.734375,
      "learning_rate": 4.670259778589074e-07,
      "loss": 0.3285,
      "reward": 0.3398437649011612,
      "reward_std": 0.1950475201010704,
      "rewards/accuracy_reward": 0.015625000931322575,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.3219866305589676,
      "step": 833
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1604.5425109863281,
      "epoch": 0.24912254499290568,
      "grad_norm": 22.57285499572754,
      "kl": 4.8828125,
      "learning_rate": 4.668964249325245e-07,
      "loss": 0.3996,
      "reward": 0.3811384066939354,
      "reward_std": 0.1889372430741787,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3186384066939354,
      "step": 834
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1655.0111999511719,
      "epoch": 0.24942125308042715,
      "grad_norm": 98.17493438720703,
      "kl": 6.484375,
      "learning_rate": 4.6676663604408946e-07,
      "loss": 0.4829,
      "reward": 0.3582589477300644,
      "reward_std": 0.1953287236392498,
      "rewards/accuracy_reward": 0.0580357164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3002232313156128,
      "step": 835
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1461.2500610351562,
      "epoch": 0.24971996116794862,
      "grad_norm": 93.57233428955078,
      "kl": 2.65234375,
      "learning_rate": 4.6663661133479977e-07,
      "loss": 0.3361,
      "reward": 0.4034598469734192,
      "reward_std": 0.17028777301311493,
      "rewards/accuracy_reward": 0.04017857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3632812723517418,
      "step": 836
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1485.4978332519531,
      "epoch": 0.2500186692554701,
      "grad_norm": 20.224430084228516,
      "kl": 4.08203125,
      "learning_rate": 4.6650635094610966e-07,
      "loss": 0.3676,
      "reward": 0.4224330484867096,
      "reward_std": 0.19755249843001366,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3510044813156128,
      "step": 837
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1581.3728332519531,
      "epoch": 0.2503173773429916,
      "grad_norm": 26.901081085205078,
      "kl": 4.82421875,
      "learning_rate": 4.663758550197296e-07,
      "loss": 0.4069,
      "reward": 0.4570312649011612,
      "reward_std": 0.20708920061588287,
      "rewards/accuracy_reward": 0.1272321492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.329799123108387,
      "step": 838
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1548.9308776855469,
      "epoch": 0.25061608543051306,
      "grad_norm": 26.57581329345703,
      "kl": 4.27734375,
      "learning_rate": 4.6624512369762637e-07,
      "loss": 0.4071,
      "reward": 0.376674123108387,
      "reward_std": 0.1624312475323677,
      "rewards/accuracy_reward": 0.04017857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3364955559372902,
      "step": 839
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1545.5000610351562,
      "epoch": 0.2509147935180345,
      "grad_norm": 23.26570701599121,
      "kl": 4.6875,
      "learning_rate": 4.661141571220229e-07,
      "loss": 0.3903,
      "reward": 0.455357164144516,
      "reward_std": 0.18889540433883667,
      "rewards/accuracy_reward": 0.12053571967408061,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3348214477300644,
      "step": 840
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1576.0848999023438,
      "epoch": 0.25121350160555594,
      "grad_norm": 10.82313346862793,
      "kl": 4.078125,
      "learning_rate": 4.659829554353979e-07,
      "loss": 0.3409,
      "reward": 0.3755580484867096,
      "reward_std": 0.18060410022735596,
      "rewards/accuracy_reward": 0.037946428870782256,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3376116156578064,
      "step": 841
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1595.7389221191406,
      "epoch": 0.2515122096930774,
      "grad_norm": 61.55778503417969,
      "kl": 5.9921875,
      "learning_rate": 4.65851518780486e-07,
      "loss": 0.4848,
      "reward": 0.398995541036129,
      "reward_std": 0.22408530116081238,
      "rewards/accuracy_reward": 0.08258928917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3164062649011612,
      "step": 842
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1510.1272888183594,
      "epoch": 0.2518109177805989,
      "grad_norm": 31.55916976928711,
      "kl": 5.5625,
      "learning_rate": 4.6571984730027746e-07,
      "loss": 0.4475,
      "reward": 0.385044664144516,
      "reward_std": 0.1869351491332054,
      "rewards/accuracy_reward": 0.05357143096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3314732238650322,
      "step": 843
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1548.4710388183594,
      "epoch": 0.25210962586812036,
      "grad_norm": 25.668895721435547,
      "kl": 5.0859375,
      "learning_rate": 4.655879411380178e-07,
      "loss": 0.4342,
      "reward": 0.3989955484867096,
      "reward_std": 0.2041613571345806,
      "rewards/accuracy_reward": 0.0669642873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3320312649011612,
      "step": 844
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1518.5960693359375,
      "epoch": 0.25240833395564183,
      "grad_norm": 18.992769241333008,
      "kl": 4.625,
      "learning_rate": 4.654558004372081e-07,
      "loss": 0.431,
      "reward": 0.4559151977300644,
      "reward_std": 0.16763852536678314,
      "rewards/accuracy_reward": 0.1183035783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.337611623108387,
      "step": 845
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1540.0514221191406,
      "epoch": 0.2527070420431633,
      "grad_norm": 11.999451637268066,
      "kl": 4.16015625,
      "learning_rate": 4.6532342534160444e-07,
      "loss": 0.3618,
      "reward": 0.4263393059372902,
      "reward_std": 0.22428957745432854,
      "rewards/accuracy_reward": 0.07142857601866126,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3549107238650322,
      "step": 846
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1566.72998046875,
      "epoch": 0.25300575013068477,
      "grad_norm": 10.131855964660645,
      "kl": 4.37109375,
      "learning_rate": 4.65190815995218e-07,
      "loss": 0.3703,
      "reward": 0.3643973395228386,
      "reward_std": 0.1941681168973446,
      "rewards/accuracy_reward": 0.04017857206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3242187649011612,
      "step": 847
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1625.2857666015625,
      "epoch": 0.25330445821820624,
      "grad_norm": 31.050949096679688,
      "kl": 3.9375,
      "learning_rate": 4.650579725423147e-07,
      "loss": 0.3689,
      "reward": 0.417410746216774,
      "reward_std": 0.18708643317222595,
      "rewards/accuracy_reward": 0.1004464328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3169643059372902,
      "step": 848
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1606.1965026855469,
      "epoch": 0.2536031663057277,
      "grad_norm": 28.903181076049805,
      "kl": 4.58984375,
      "learning_rate": 4.649248951274153e-07,
      "loss": 0.3544,
      "reward": 0.361607164144516,
      "reward_std": 0.20924041792750359,
      "rewards/accuracy_reward": 0.035714287078008056,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.325892873108387,
      "step": 849
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1578.0179443359375,
      "epoch": 0.2539018743932492,
      "grad_norm": 28.99771499633789,
      "kl": 3.41796875,
      "learning_rate": 4.647915838952948e-07,
      "loss": 0.3073,
      "reward": 0.4202009066939354,
      "reward_std": 0.19332637637853622,
      "rewards/accuracy_reward": 0.08928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3309151828289032,
      "step": 850
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1596.38623046875,
      "epoch": 0.25420058248077065,
      "grad_norm": 47.24482727050781,
      "kl": 3.6875,
      "learning_rate": 4.6465803899098287e-07,
      "loss": 0.379,
      "reward": 0.479910746216774,
      "reward_std": 0.21462104842066765,
      "rewards/accuracy_reward": 0.1428571492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3370535895228386,
      "step": 851
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1608.7567749023438,
      "epoch": 0.2544992905682921,
      "grad_norm": 45.97939682006836,
      "kl": 3.7578125,
      "learning_rate": 4.645242605597633e-07,
      "loss": 0.3593,
      "reward": 0.4369419738650322,
      "reward_std": 0.20487766340374947,
      "rewards/accuracy_reward": 0.10267857392318547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3342634066939354,
      "step": 852
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1534.0915832519531,
      "epoch": 0.2547979986558136,
      "grad_norm": 51.74606704711914,
      "kl": 3.6640625,
      "learning_rate": 4.6439024874717384e-07,
      "loss": 0.3804,
      "reward": 0.4854911044239998,
      "reward_std": 0.24209454283118248,
      "rewards/accuracy_reward": 0.14285714738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3426339402794838,
      "step": 853
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1588.6920471191406,
      "epoch": 0.25509670674333507,
      "grad_norm": 27.400096893310547,
      "kl": 4.8671875,
      "learning_rate": 4.642560036990063e-07,
      "loss": 0.3819,
      "reward": 0.376674123108387,
      "reward_std": 0.18695980310440063,
      "rewards/accuracy_reward": 0.05133928800933063,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3253348395228386,
      "step": 854
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1560.6741943359375,
      "epoch": 0.25539541483085654,
      "grad_norm": 14.810881614685059,
      "kl": 4.5859375,
      "learning_rate": 4.64121525561306e-07,
      "loss": 0.4016,
      "reward": 0.3878348469734192,
      "reward_std": 0.21267437189817429,
      "rewards/accuracy_reward": 0.05133928917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.336495541036129,
      "step": 855
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1642.3103637695312,
      "epoch": 0.255694122918378,
      "grad_norm": 35.78523635864258,
      "kl": 5.2734375,
      "learning_rate": 4.6398681448037213e-07,
      "loss": 0.4151,
      "reward": 0.3236607313156128,
      "reward_std": 0.1806083656847477,
      "rewards/accuracy_reward": 0.008928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3147321566939354,
      "step": 856
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1486.2032165527344,
      "epoch": 0.2559928310058995,
      "grad_norm": 9.741594314575195,
      "kl": 4.6640625,
      "learning_rate": 4.638518706027571e-07,
      "loss": 0.4249,
      "reward": 0.4285714477300644,
      "reward_std": 0.22246531024575233,
      "rewards/accuracy_reward": 0.07812500465661287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3504464402794838,
      "step": 857
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1597.0982971191406,
      "epoch": 0.25629153909342095,
      "grad_norm": 38.505401611328125,
      "kl": 5.3125,
      "learning_rate": 4.6371669407526655e-07,
      "loss": 0.4239,
      "reward": 0.3777901977300644,
      "reward_std": 0.2057284191250801,
      "rewards/accuracy_reward": 0.06026785937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.317522332072258,
      "step": 858
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1642.8661193847656,
      "epoch": 0.2565902471809424,
      "grad_norm": 4.1782379150390625,
      "kl": 4.48046875,
      "learning_rate": 4.635812850449593e-07,
      "loss": 0.384,
      "reward": 0.3727678656578064,
      "reward_std": 0.18589527904987335,
      "rewards/accuracy_reward": 0.04910714505240321,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3236607313156128,
      "step": 859
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1523.4353637695312,
      "epoch": 0.2568889552684639,
      "grad_norm": 9.808791160583496,
      "kl": 4.60546875,
      "learning_rate": 4.634456436591472e-07,
      "loss": 0.4167,
      "reward": 0.4179687649011612,
      "reward_std": 0.19432710483670235,
      "rewards/accuracy_reward": 0.08928571967408061,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3286830559372902,
      "step": 860
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1614.134033203125,
      "epoch": 0.25718766335598536,
      "grad_norm": 8.961434364318848,
      "kl": 4.58984375,
      "learning_rate": 4.6330977006539465e-07,
      "loss": 0.4032,
      "reward": 0.3973214477300644,
      "reward_std": 0.16868189722299576,
      "rewards/accuracy_reward": 0.07589285937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.321428582072258,
      "step": 861
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1535.04248046875,
      "epoch": 0.25748637144350683,
      "grad_norm": 13.457070350646973,
      "kl": 4.6640625,
      "learning_rate": 4.631736644115189e-07,
      "loss": 0.4222,
      "reward": 0.471540205180645,
      "reward_std": 0.2019900269806385,
      "rewards/accuracy_reward": 0.13392857648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3376116156578064,
      "step": 862
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1526.7433776855469,
      "epoch": 0.2577850795310283,
      "grad_norm": 103.91754913330078,
      "kl": 7.1796875,
      "learning_rate": 4.630373268455895e-07,
      "loss": 0.5891,
      "reward": 0.400669664144516,
      "reward_std": 0.21284732222557068,
      "rewards/accuracy_reward": 0.07589286006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3247767984867096,
      "step": 863
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1574.8840026855469,
      "epoch": 0.2580837876185498,
      "grad_norm": 143.74099731445312,
      "kl": 7.5859375,
      "learning_rate": 4.629007575159285e-07,
      "loss": 0.5454,
      "reward": 0.3978794813156128,
      "reward_std": 0.22335519641637802,
      "rewards/accuracy_reward": 0.07812500302679837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3197544813156128,
      "step": 864
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1615.5736999511719,
      "epoch": 0.25838249570607125,
      "grad_norm": 49.08283233642578,
      "kl": 5.359375,
      "learning_rate": 4.6276395657110974e-07,
      "loss": 0.3929,
      "reward": 0.3978794813156128,
      "reward_std": 0.19036248326301575,
      "rewards/accuracy_reward": 0.06473214481957257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3331473395228386,
      "step": 865
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1633.6027526855469,
      "epoch": 0.2586812037935927,
      "grad_norm": 92.3703384399414,
      "kl": 6.4375,
      "learning_rate": 4.626269241599594e-07,
      "loss": 0.4543,
      "reward": 0.3995535895228386,
      "reward_std": 0.2442575879395008,
      "rewards/accuracy_reward": 0.06919643003493547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3303571566939354,
      "step": 866
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1547.4844360351562,
      "epoch": 0.2589799118811142,
      "grad_norm": 67.17633056640625,
      "kl": 5.8671875,
      "learning_rate": 4.624896604315552e-07,
      "loss": 0.449,
      "reward": 0.509486623108387,
      "reward_std": 0.20164956897497177,
      "rewards/accuracy_reward": 0.16964286379516125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3398437649011612,
      "step": 867
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1594.7389221191406,
      "epoch": 0.25927861996863566,
      "grad_norm": 9.927915573120117,
      "kl": 4.63671875,
      "learning_rate": 4.623521655352267e-07,
      "loss": 0.3897,
      "reward": 0.3917410895228386,
      "reward_std": 0.22729899361729622,
      "rewards/accuracy_reward": 0.060267859138548374,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3314732313156128,
      "step": 868
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1534.8929443359375,
      "epoch": 0.25957732805615713,
      "grad_norm": 65.99718475341797,
      "kl": 3.15625,
      "learning_rate": 4.6221443962055483e-07,
      "loss": 0.347,
      "reward": 0.4620535895228386,
      "reward_std": 0.18996809795498848,
      "rewards/accuracy_reward": 0.10937500488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.352678582072258,
      "step": 869
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1532.1942749023438,
      "epoch": 0.2598760361436786,
      "grad_norm": 57.06461715698242,
      "kl": 3.23046875,
      "learning_rate": 4.6207648283737186e-07,
      "loss": 0.3392,
      "reward": 0.3565848395228386,
      "reward_std": 0.20425190404057503,
      "rewards/accuracy_reward": 0.02455357275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3320312649011612,
      "step": 870
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1577.2032165527344,
      "epoch": 0.2601747442312001,
      "grad_norm": 9.386937141418457,
      "kl": 4.19140625,
      "learning_rate": 4.6193829533576145e-07,
      "loss": 0.3738,
      "reward": 0.4620535969734192,
      "reward_std": 0.22620641440153122,
      "rewards/accuracy_reward": 0.11830357555299997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3437500149011612,
      "step": 871
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1563.1161193847656,
      "epoch": 0.26047345231872154,
      "grad_norm": 10.678582191467285,
      "kl": 4.2421875,
      "learning_rate": 4.6179987726605776e-07,
      "loss": 0.3757,
      "reward": 0.3616071566939354,
      "reward_std": 0.18918341398239136,
      "rewards/accuracy_reward": 0.024553571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3370535895228386,
      "step": 872
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1536.9107971191406,
      "epoch": 0.260772160406243,
      "grad_norm": 57.1290283203125,
      "kl": 3.24609375,
      "learning_rate": 4.616612287788463e-07,
      "loss": 0.3475,
      "reward": 0.4335937649011612,
      "reward_std": 0.17472489550709724,
      "rewards/accuracy_reward": 0.0848214328289032,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.3465401977300644,
      "step": 873
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1681.0692749023438,
      "epoch": 0.2610708684937645,
      "grad_norm": 17.58644676208496,
      "kl": 3.8046875,
      "learning_rate": 4.615223500249629e-07,
      "loss": 0.3113,
      "reward": 0.322544664144516,
      "reward_std": 0.17516427487134933,
      "rewards/accuracy_reward": 0.013392857741564512,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3091518059372902,
      "step": 874
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1536.540283203125,
      "epoch": 0.26136957658128596,
      "grad_norm": 15.848076820373535,
      "kl": 4.09765625,
      "learning_rate": 4.613832411554941e-07,
      "loss": 0.3865,
      "reward": 0.434709832072258,
      "reward_std": 0.18973276391625404,
      "rewards/accuracy_reward": 0.0937500074505806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.340959832072258,
      "step": 875
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1649.4331359863281,
      "epoch": 0.2616682846688074,
      "grad_norm": 26.35906219482422,
      "kl": 5.1484375,
      "learning_rate": 4.6124390232177674e-07,
      "loss": 0.4202,
      "reward": 0.4095982313156128,
      "reward_std": 0.2281733825802803,
      "rewards/accuracy_reward": 0.09598214738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.313616082072258,
      "step": 876
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1639.0670166015625,
      "epoch": 0.2619669927563289,
      "grad_norm": 49.567283630371094,
      "kl": 5.046875,
      "learning_rate": 4.611043336753978e-07,
      "loss": 0.3583,
      "reward": 0.4458705559372902,
      "reward_std": 0.2183237224817276,
      "rewards/accuracy_reward": 0.12053571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.325334832072258,
      "step": 877
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1454.5491638183594,
      "epoch": 0.26226570084385037,
      "grad_norm": 35.591224670410156,
      "kl": 3.9453125,
      "learning_rate": 4.6096453536819416e-07,
      "loss": 0.3849,
      "reward": 0.4542410895228386,
      "reward_std": 0.22669217735528946,
      "rewards/accuracy_reward": 0.0892857164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.364955373108387,
      "step": 878
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1553.9554138183594,
      "epoch": 0.26256440893137184,
      "grad_norm": 12.461967468261719,
      "kl": 4.625,
      "learning_rate": 4.608245075522529e-07,
      "loss": 0.4208,
      "reward": 0.498325914144516,
      "reward_std": 0.2063368298113346,
      "rewards/accuracy_reward": 0.1540178582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3443080559372902,
      "step": 879
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1671.4197082519531,
      "epoch": 0.2628631170188933,
      "grad_norm": 39.97450637817383,
      "kl": 5.4375,
      "learning_rate": 4.606842503799104e-07,
      "loss": 0.4136,
      "reward": 0.3733259066939354,
      "reward_std": 0.20198510587215424,
      "rewards/accuracy_reward": 0.05357142980210483,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3197544813156128,
      "step": 880
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1590.0759582519531,
      "epoch": 0.2631618251064148,
      "grad_norm": 29.42139434814453,
      "kl": 4.5859375,
      "learning_rate": 4.6054376400375274e-07,
      "loss": 0.3478,
      "reward": 0.4241071566939354,
      "reward_std": 0.19159143790602684,
      "rewards/accuracy_reward": 0.08258928847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.341517873108387,
      "step": 881
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1672.7232971191406,
      "epoch": 0.26346053319393625,
      "grad_norm": 46.092708587646484,
      "kl": 5.05078125,
      "learning_rate": 4.6040304857661536e-07,
      "loss": 0.3656,
      "reward": 0.3828125149011612,
      "reward_std": 0.17567341774702072,
      "rewards/accuracy_reward": 0.08035714644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.302455373108387,
      "step": 882
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1562.0715026855469,
      "epoch": 0.26375924128145767,
      "grad_norm": 13.574013710021973,
      "kl": 4.14453125,
      "learning_rate": 4.6026210425158284e-07,
      "loss": 0.379,
      "reward": 0.4185267984867096,
      "reward_std": 0.2103034295141697,
      "rewards/accuracy_reward": 0.0870535746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3314732238650322,
      "step": 883
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1569.2188110351562,
      "epoch": 0.26405794936897914,
      "grad_norm": 51.06179428100586,
      "kl": 3.78515625,
      "learning_rate": 4.6012093118198885e-07,
      "loss": 0.3718,
      "reward": 0.357142873108387,
      "reward_std": 0.2059067003428936,
      "rewards/accuracy_reward": 0.017857143888249993,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3392857238650322,
      "step": 884
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1576.3572082519531,
      "epoch": 0.2643566574565006,
      "grad_norm": 27.725021362304688,
      "kl": 4.0234375,
      "learning_rate": 4.5997952952141575e-07,
      "loss": 0.3647,
      "reward": 0.494977705180645,
      "reward_std": 0.2317986637353897,
      "rewards/accuracy_reward": 0.15401786379516125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3409598395228386,
      "step": 885
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1621.7411499023438,
      "epoch": 0.2646553655440221,
      "grad_norm": 19.939455032348633,
      "kl": 4.20703125,
      "learning_rate": 4.598378994236948e-07,
      "loss": 0.3771,
      "reward": 0.447544664144516,
      "reward_std": 0.20751018449664116,
      "rewards/accuracy_reward": 0.12946429220028222,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3180803656578064,
      "step": 886
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1546.3259887695312,
      "epoch": 0.26495407363154355,
      "grad_norm": 29.666196823120117,
      "kl": 4.23046875,
      "learning_rate": 4.5969604104290556e-07,
      "loss": 0.409,
      "reward": 0.4408482387661934,
      "reward_std": 0.1940433271229267,
      "rewards/accuracy_reward": 0.09375000209547579,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3470982313156128,
      "step": 887
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1641.8058776855469,
      "epoch": 0.265252781719065,
      "grad_norm": 21.547897338867188,
      "kl": 4.7578125,
      "learning_rate": 4.5955395453337607e-07,
      "loss": 0.3839,
      "reward": 0.4335937723517418,
      "reward_std": 0.23218850046396255,
      "rewards/accuracy_reward": 0.1093750037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3242187574505806,
      "step": 888
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1607.3616943359375,
      "epoch": 0.2655514898065865,
      "grad_norm": 14.991734504699707,
      "kl": 4.1953125,
      "learning_rate": 4.5941164004968253e-07,
      "loss": 0.3399,
      "reward": 0.3783482313156128,
      "reward_std": 0.21754376962780952,
      "rewards/accuracy_reward": 0.03794643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3404018059372902,
      "step": 889
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1503.9353637695312,
      "epoch": 0.26585019789410796,
      "grad_norm": 32.39583206176758,
      "kl": 5.28125,
      "learning_rate": 4.592690977466492e-07,
      "loss": 0.4288,
      "reward": 0.4631696715950966,
      "reward_std": 0.206858791410923,
      "rewards/accuracy_reward": 0.10937500232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.353794664144516,
      "step": 890
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1602.7545471191406,
      "epoch": 0.26614890598162944,
      "grad_norm": 28.859947204589844,
      "kl": 5.0703125,
      "learning_rate": 4.5912632777934805e-07,
      "loss": 0.4043,
      "reward": 0.4581473544239998,
      "reward_std": 0.18166309595108032,
      "rewards/accuracy_reward": 0.1361607201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.321986623108387,
      "step": 891
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1550.5603637695312,
      "epoch": 0.2664476140691509,
      "grad_norm": 16.414947509765625,
      "kl": 4.2734375,
      "learning_rate": 4.589833303030989e-07,
      "loss": 0.3909,
      "reward": 0.4871651977300644,
      "reward_std": 0.22048326209187508,
      "rewards/accuracy_reward": 0.1406250037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3465401902794838,
      "step": 892
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1603.2947082519531,
      "epoch": 0.2667463221566724,
      "grad_norm": 27.078393936157227,
      "kl": 4.734375,
      "learning_rate": 4.58840105473469e-07,
      "loss": 0.3638,
      "reward": 0.4637276977300644,
      "reward_std": 0.1959126926958561,
      "rewards/accuracy_reward": 0.12946429289877415,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.334263414144516,
      "step": 893
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1569.8438110351562,
      "epoch": 0.26704503024419385,
      "grad_norm": 27.020221710205078,
      "kl": 3.64453125,
      "learning_rate": 4.5869665344627273e-07,
      "loss": 0.3384,
      "reward": 0.3911830559372902,
      "reward_std": 0.1636234149336815,
      "rewards/accuracy_reward": 0.042410716181620955,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.348772332072258,
      "step": 894
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1563.2947082519531,
      "epoch": 0.2673437383317153,
      "grad_norm": 22.678203582763672,
      "kl": 4.40234375,
      "learning_rate": 4.585529743775721e-07,
      "loss": 0.4158,
      "reward": 0.3761160895228386,
      "reward_std": 0.2076995000243187,
      "rewards/accuracy_reward": 0.029017857974395156,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3470982313156128,
      "step": 895
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1473.7098999023438,
      "epoch": 0.2676424464192368,
      "grad_norm": 40.3590087890625,
      "kl": 3.9453125,
      "learning_rate": 4.5840906842367566e-07,
      "loss": 0.3781,
      "reward": 0.448660746216774,
      "reward_std": 0.2113383449614048,
      "rewards/accuracy_reward": 0.082589291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3660714477300644,
      "step": 896
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1625.2366943359375,
      "epoch": 0.26794115450675826,
      "grad_norm": 46.94992446899414,
      "kl": 5.31640625,
      "learning_rate": 4.582649357411391e-07,
      "loss": 0.3932,
      "reward": 0.4308035895228386,
      "reward_std": 0.17740712314844131,
      "rewards/accuracy_reward": 0.11383929220028222,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3169643059372902,
      "step": 897
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1626.8415832519531,
      "epoch": 0.26823986259427973,
      "grad_norm": 59.473388671875,
      "kl": 5.9453125,
      "learning_rate": 4.581205764867646e-07,
      "loss": 0.4386,
      "reward": 0.4302455484867096,
      "reward_std": 0.21160991489887238,
      "rewards/accuracy_reward": 0.09598214784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.334263414144516,
      "step": 898
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1446.087158203125,
      "epoch": 0.2685385706818012,
      "grad_norm": 9.765243530273438,
      "kl": 4.890625,
      "learning_rate": 4.57975990817601e-07,
      "loss": 0.4404,
      "reward": 0.4726562723517418,
      "reward_std": 0.19033697247505188,
      "rewards/accuracy_reward": 0.10714286309666932,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3655134066939354,
      "step": 899
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1605.4286193847656,
      "epoch": 0.2688372787693227,
      "grad_norm": 23.97719383239746,
      "kl": 4.8828125,
      "learning_rate": 4.578311788909431e-07,
      "loss": 0.4086,
      "reward": 0.3995535969734192,
      "reward_std": 0.2671196758747101,
      "rewards/accuracy_reward": 0.06919643096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3303571566939354,
      "step": 900
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1597.0313110351562,
      "epoch": 0.26913598685684414,
      "grad_norm": 5.647151470184326,
      "kl": 4.50390625,
      "learning_rate": 4.576861408643322e-07,
      "loss": 0.3944,
      "reward": 0.3984375223517418,
      "reward_std": 0.20241311565041542,
      "rewards/accuracy_reward": 0.06250000302679837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3359375149011612,
      "step": 901
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1469.3415832519531,
      "epoch": 0.2694346949443656,
      "grad_norm": 45.46286392211914,
      "kl": 3.4453125,
      "learning_rate": 4.5754087689555555e-07,
      "loss": 0.367,
      "reward": 0.3777901977300644,
      "reward_std": 0.18921225145459175,
      "rewards/accuracy_reward": 0.020089286379516125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.357700914144516,
      "step": 902
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1589.77685546875,
      "epoch": 0.2697334030318871,
      "grad_norm": 39.711246490478516,
      "kl": 3.41015625,
      "learning_rate": 4.5739538714264596e-07,
      "loss": 0.3196,
      "reward": 0.3934151977300644,
      "reward_std": 0.19973937422037125,
      "rewards/accuracy_reward": 0.051339286379516125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3420758992433548,
      "step": 903
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1503.4286193847656,
      "epoch": 0.27003211111940856,
      "grad_norm": 32.525455474853516,
      "kl": 4.7578125,
      "learning_rate": 4.572496717638821e-07,
      "loss": 0.4639,
      "reward": 0.4765625149011612,
      "reward_std": 0.18457292020320892,
      "rewards/accuracy_reward": 0.1250000074505806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625149011612,
      "step": 904
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1551.21435546875,
      "epoch": 0.27033081920693003,
      "grad_norm": 44.244903564453125,
      "kl": 4.0078125,
      "learning_rate": 4.571037309177881e-07,
      "loss": 0.3775,
      "reward": 0.3588169738650322,
      "reward_std": 0.17804919928312302,
      "rewards/accuracy_reward": 0.013392857974395156,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.345424123108387,
      "step": 905
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1622.2322387695312,
      "epoch": 0.2706295272944515,
      "grad_norm": 43.21623992919922,
      "kl": 6.0234375,
      "learning_rate": 4.5695756476313305e-07,
      "loss": 0.4662,
      "reward": 0.4045759215950966,
      "reward_std": 0.22162174433469772,
      "rewards/accuracy_reward": 0.0781250037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3264509066939354,
      "step": 906
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1575.3817749023438,
      "epoch": 0.27092823538197297,
      "grad_norm": 37.05295181274414,
      "kl": 5.8125,
      "learning_rate": 4.568111734589317e-07,
      "loss": 0.486,
      "reward": 0.4168526977300644,
      "reward_std": 0.19942563772201538,
      "rewards/accuracy_reward": 0.0825892873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.334263414144516,
      "step": 907
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1552.71435546875,
      "epoch": 0.27122694346949444,
      "grad_norm": 26.789337158203125,
      "kl": 5.3671875,
      "learning_rate": 4.5666455716444327e-07,
      "loss": 0.4578,
      "reward": 0.4285714477300644,
      "reward_std": 0.2162286713719368,
      "rewards/accuracy_reward": 0.082589291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3459821566939354,
      "step": 908
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1542.8907165527344,
      "epoch": 0.2715256515570159,
      "grad_norm": 20.8460636138916,
      "kl": 5.0703125,
      "learning_rate": 4.5651771603917187e-07,
      "loss": 0.4099,
      "reward": 0.3989955559372902,
      "reward_std": 0.20633529499173164,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3364955484867096,
      "step": 909
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1619.6987609863281,
      "epoch": 0.2718243596445374,
      "grad_norm": 33.90433120727539,
      "kl": 5.4921875,
      "learning_rate": 4.5637065024286645e-07,
      "loss": 0.4124,
      "reward": 0.3872767984867096,
      "reward_std": 0.20532633736729622,
      "rewards/accuracy_reward": 0.058035715483129025,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3292410895228386,
      "step": 910
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1465.5067443847656,
      "epoch": 0.27212306773205885,
      "grad_norm": 11.435953140258789,
      "kl": 4.171875,
      "learning_rate": 4.5622335993552e-07,
      "loss": 0.3541,
      "reward": 0.4375000223517418,
      "reward_std": 0.21598805859684944,
      "rewards/accuracy_reward": 0.06919643189758062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3683035895228386,
      "step": 911
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1635.4152526855469,
      "epoch": 0.2724217758195803,
      "grad_norm": 32.17258071899414,
      "kl": 4.8125,
      "learning_rate": 4.5607584527737e-07,
      "loss": 0.3754,
      "reward": 0.4146205559372902,
      "reward_std": 0.22238923609256744,
      "rewards/accuracy_reward": 0.08928571874275804,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.325334832072258,
      "step": 912
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1571.7232666015625,
      "epoch": 0.2727204839071018,
      "grad_norm": 7.548865795135498,
      "kl": 4.2109375,
      "learning_rate": 4.559281064288979e-07,
      "loss": 0.3535,
      "reward": 0.4062500149011612,
      "reward_std": 0.184102401137352,
      "rewards/accuracy_reward": 0.060267859837040305,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3459821566939354,
      "step": 913
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1542.5067749023438,
      "epoch": 0.27301919199462327,
      "grad_norm": 49.508846282958984,
      "kl": 3.5546875,
      "learning_rate": 4.557801435508292e-07,
      "loss": 0.3603,
      "reward": 0.4447544813156128,
      "reward_std": 0.18034479394555092,
      "rewards/accuracy_reward": 0.09375000488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3510044738650322,
      "step": 914
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1548.79248046875,
      "epoch": 0.27331790008214474,
      "grad_norm": 52.294124603271484,
      "kl": 3.66015625,
      "learning_rate": 4.556319568041329e-07,
      "loss": 0.3422,
      "reward": 0.399553582072258,
      "reward_std": 0.17013169452548027,
      "rewards/accuracy_reward": 0.0491071455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3504464477300644,
      "step": 915
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1589.1920471191406,
      "epoch": 0.2736166081696662,
      "grad_norm": 31.779356002807617,
      "kl": 3.73046875,
      "learning_rate": 4.5548354635002175e-07,
      "loss": 0.3366,
      "reward": 0.4224330559372902,
      "reward_std": 0.2041812315583229,
      "rewards/accuracy_reward": 0.07589285937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3465401902794838,
      "step": 916
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1648.5581359863281,
      "epoch": 0.2739153162571877,
      "grad_norm": 10.567296028137207,
      "kl": 4.5703125,
      "learning_rate": 4.553349123499517e-07,
      "loss": 0.3667,
      "reward": 0.408482164144516,
      "reward_std": 0.192521333694458,
      "rewards/accuracy_reward": 0.09151786286383867,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3169642984867096,
      "step": 917
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1586.5469360351562,
      "epoch": 0.27421402434470915,
      "grad_norm": 33.52521514892578,
      "kl": 3.65625,
      "learning_rate": 4.55186054965622e-07,
      "loss": 0.317,
      "reward": 0.3532366305589676,
      "reward_std": 0.18983694538474083,
      "rewards/accuracy_reward": 0.024553572526201606,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3286830559372902,
      "step": 918
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1386.97998046875,
      "epoch": 0.2745127324322306,
      "grad_norm": 72.76332092285156,
      "kl": 3.46484375,
      "learning_rate": 4.550369743589748e-07,
      "loss": 0.3988,
      "reward": 0.4960937723517418,
      "reward_std": 0.19807007908821106,
      "rewards/accuracy_reward": 0.11830357555299997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3777901977300644,
      "step": 919
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1574.7165832519531,
      "epoch": 0.2748114405197521,
      "grad_norm": 6.297797679901123,
      "kl": 4.5234375,
      "learning_rate": 4.5488767069219536e-07,
      "loss": 0.3779,
      "reward": 0.4821428805589676,
      "reward_std": 0.19668077304959297,
      "rewards/accuracy_reward": 0.129464291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3526785895228386,
      "step": 920
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1551.185302734375,
      "epoch": 0.27511014860727356,
      "grad_norm": 38.58011245727539,
      "kl": 4.43359375,
      "learning_rate": 4.547381441277114e-07,
      "loss": 0.4124,
      "reward": 0.4330357313156128,
      "reward_std": 0.17036039009690285,
      "rewards/accuracy_reward": 0.08035714528523386,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3526785895228386,
      "step": 921
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1682.8773193359375,
      "epoch": 0.27540885669479503,
      "grad_norm": 26.623886108398438,
      "kl": 5.04296875,
      "learning_rate": 4.54588394828193e-07,
      "loss": 0.3817,
      "reward": 0.3593750149011612,
      "reward_std": 0.18209074065089226,
      "rewards/accuracy_reward": 0.046875003492459655,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3125000149011612,
      "step": 922
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1448.2054138183594,
      "epoch": 0.2757075647823165,
      "grad_norm": 6.805716514587402,
      "kl": 4.765625,
      "learning_rate": 4.54438422956553e-07,
      "loss": 0.4422,
      "reward": 0.4447544887661934,
      "reward_std": 0.17976408079266548,
      "rewards/accuracy_reward": 0.0870535746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3577009066939354,
      "step": 923
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1560.5692749023438,
      "epoch": 0.276006272869838,
      "grad_norm": 30.562122344970703,
      "kl": 5.203125,
      "learning_rate": 4.5428822867594585e-07,
      "loss": 0.4148,
      "reward": 0.4324776977300644,
      "reward_std": 0.21806598454713821,
      "rewards/accuracy_reward": 0.10267857578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.329799123108387,
      "step": 924
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1539.1920166015625,
      "epoch": 0.27630498095735945,
      "grad_norm": 7.877579689025879,
      "kl": 4.8828125,
      "learning_rate": 4.5413781214976826e-07,
      "loss": 0.4179,
      "reward": 0.3582589402794838,
      "reward_std": 0.15835804492235184,
      "rewards/accuracy_reward": 0.006696428870782256,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625074505806,
      "step": 925
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1645.4130249023438,
      "epoch": 0.27660368904488086,
      "grad_norm": 42.64602279663086,
      "kl": 5.6875,
      "learning_rate": 4.539871735416588e-07,
      "loss": 0.4129,
      "reward": 0.3560267984867096,
      "reward_std": 0.19773318618535995,
      "rewards/accuracy_reward": 0.029017858672887087,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3270089402794838,
      "step": 926
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1555.7188110351562,
      "epoch": 0.27690239713240233,
      "grad_norm": 12.59932804107666,
      "kl": 5.4765625,
      "learning_rate": 4.5383631301549733e-07,
      "loss": 0.4738,
      "reward": 0.4129464402794838,
      "reward_std": 0.2063601128757,
      "rewards/accuracy_reward": 0.066964291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3459821566939354,
      "step": 927
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1579.6273193359375,
      "epoch": 0.2772011052199238,
      "grad_norm": 32.66411590576172,
      "kl": 5.34375,
      "learning_rate": 4.5368523073540543e-07,
      "loss": 0.4364,
      "reward": 0.4185268133878708,
      "reward_std": 0.2288465015590191,
      "rewards/accuracy_reward": 0.08258928940631449,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3359375149011612,
      "step": 928
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1590.2634582519531,
      "epoch": 0.2774998133074453,
      "grad_norm": 24.75836944580078,
      "kl": 4.1328125,
      "learning_rate": 4.535339268657459e-07,
      "loss": 0.3508,
      "reward": 0.3995535895228386,
      "reward_std": 0.17974304035305977,
      "rewards/accuracy_reward": 0.046875000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.352678582072258,
      "step": 929
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1547.9978637695312,
      "epoch": 0.27779852139496675,
      "grad_norm": 24.30841064453125,
      "kl": 3.9921875,
      "learning_rate": 4.5338240157112226e-07,
      "loss": 0.3684,
      "reward": 0.4983259066939354,
      "reward_std": 0.18460837379097939,
      "rewards/accuracy_reward": 0.1361607164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3621651977300644,
      "step": 930
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1500.9442749023438,
      "epoch": 0.2780972294824882,
      "grad_norm": 55.247737884521484,
      "kl": 3.66015625,
      "learning_rate": 4.532306550163794e-07,
      "loss": 0.378,
      "reward": 0.444196455180645,
      "reward_std": 0.17786908149719238,
      "rewards/accuracy_reward": 0.07142857578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.372767873108387,
      "step": 931
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1631.9889221191406,
      "epoch": 0.2783959375700097,
      "grad_norm": 23.401912689208984,
      "kl": 4.90625,
      "learning_rate": 4.530786873666026e-07,
      "loss": 0.3831,
      "reward": 0.3476562574505806,
      "reward_std": 0.1953258216381073,
      "rewards/accuracy_reward": 0.024553572526201606,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3231026902794838,
      "step": 932
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1534.9822387695312,
      "epoch": 0.27869464565753116,
      "grad_norm": 24.194305419921875,
      "kl": 4.47265625,
      "learning_rate": 4.5292649878711783e-07,
      "loss": 0.4093,
      "reward": 0.4375000149011612,
      "reward_std": 0.17405148595571518,
      "rewards/accuracy_reward": 0.08258928917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3549107313156128,
      "step": 933
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1578.6630249023438,
      "epoch": 0.27899335374505263,
      "grad_norm": 100.38318634033203,
      "kl": 7.1953125,
      "learning_rate": 4.5277408944349126e-07,
      "loss": 0.5469,
      "reward": 0.4693080633878708,
      "reward_std": 0.1900699958205223,
      "rewards/accuracy_reward": 0.13616072433069348,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3331473395228386,
      "step": 934
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1619.7567749023438,
      "epoch": 0.2792920618325741,
      "grad_norm": 11.667678833007812,
      "kl": 4.8828125,
      "learning_rate": 4.526214595015294e-07,
      "loss": 0.3957,
      "reward": 0.4056919738650322,
      "reward_std": 0.21412968263030052,
      "rewards/accuracy_reward": 0.06696428777649999,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3387276902794838,
      "step": 935
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1616.3861999511719,
      "epoch": 0.27959076992009557,
      "grad_norm": 12.179121017456055,
      "kl": 4.640625,
      "learning_rate": 4.524686091272787e-07,
      "loss": 0.3883,
      "reward": 0.4648437723517418,
      "reward_std": 0.22595339640974998,
      "rewards/accuracy_reward": 0.11830357927829027,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3465401902794838,
      "step": 936
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1521.7187805175781,
      "epoch": 0.27988947800761704,
      "grad_norm": 30.797372817993164,
      "kl": 4.44921875,
      "learning_rate": 4.523155384870252e-07,
      "loss": 0.4092,
      "reward": 0.4296875149011612,
      "reward_std": 0.1795186921954155,
      "rewards/accuracy_reward": 0.0647321455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.364955373108387,
      "step": 937
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1487.57373046875,
      "epoch": 0.2801881860951385,
      "grad_norm": 19.977609634399414,
      "kl": 3.85546875,
      "learning_rate": 4.5216224774729497e-07,
      "loss": 0.3328,
      "reward": 0.4107142984867096,
      "reward_std": 0.21190990880131721,
      "rewards/accuracy_reward": 0.03794643096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3727678805589676,
      "step": 938
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1599.6986999511719,
      "epoch": 0.28048689418266,
      "grad_norm": 97.52548217773438,
      "kl": 6.546875,
      "learning_rate": 4.520087370748532e-07,
      "loss": 0.4424,
      "reward": 0.3487723469734192,
      "reward_std": 0.16231832653284073,
      "rewards/accuracy_reward": 0.006696428870782256,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3420759066939354,
      "step": 939
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1566.5179138183594,
      "epoch": 0.28078560227018146,
      "grad_norm": 102.2385482788086,
      "kl": 6.9296875,
      "learning_rate": 4.518550066367045e-07,
      "loss": 0.5074,
      "reward": 0.4207589477300644,
      "reward_std": 0.20485138893127441,
      "rewards/accuracy_reward": 0.06473214528523386,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3560268059372902,
      "step": 940
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1472.9688110351562,
      "epoch": 0.2810843103577029,
      "grad_norm": 46.230628967285156,
      "kl": 6.359375,
      "learning_rate": 4.517010566000924e-07,
      "loss": 0.5199,
      "reward": 0.4514509215950966,
      "reward_std": 0.15559089556336403,
      "rewards/accuracy_reward": 0.0848214328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3666294813156128,
      "step": 941
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1545.6317443847656,
      "epoch": 0.2813830184452244,
      "grad_norm": 15.406947135925293,
      "kl": 5.5,
      "learning_rate": 4.5154688713249964e-07,
      "loss": 0.4416,
      "reward": 0.3917410969734192,
      "reward_std": 0.17182153835892677,
      "rewards/accuracy_reward": 0.02455357206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875074505806,
      "step": 942
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1576.47998046875,
      "epoch": 0.28168172653274587,
      "grad_norm": 19.918075561523438,
      "kl": 5.359375,
      "learning_rate": 4.513924984016474e-07,
      "loss": 0.4169,
      "reward": 0.403459832072258,
      "reward_std": 0.18727761134505272,
      "rewards/accuracy_reward": 0.05357143026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3498884066939354,
      "step": 943
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1630.6764221191406,
      "epoch": 0.28198043462026734,
      "grad_norm": 13.503260612487793,
      "kl": 4.34375,
      "learning_rate": 4.5123789057549544e-07,
      "loss": 0.3424,
      "reward": 0.3588169813156128,
      "reward_std": 0.1735735982656479,
      "rewards/accuracy_reward": 0.011160714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3476562723517418,
      "step": 944
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1497.5648193359375,
      "epoch": 0.2822791427077888,
      "grad_norm": 44.53622817993164,
      "kl": 4.21875,
      "learning_rate": 4.51083063822242e-07,
      "loss": 0.4182,
      "reward": 0.4017857387661934,
      "reward_std": 0.18243606202304363,
      "rewards/accuracy_reward": 0.0290178582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3727678805589676,
      "step": 945
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1498.1629943847656,
      "epoch": 0.2825778507953103,
      "grad_norm": 76.17205810546875,
      "kl": 2.95703125,
      "learning_rate": 4.509280183103234e-07,
      "loss": 0.2919,
      "reward": 0.3895089402794838,
      "reward_std": 0.16386397182941437,
      "rewards/accuracy_reward": 0.011160714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3783482313156128,
      "step": 946
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1507.0692749023438,
      "epoch": 0.28287655888283175,
      "grad_norm": 54.65409469604492,
      "kl": 3.671875,
      "learning_rate": 4.507727542084139e-07,
      "loss": 0.3581,
      "reward": 0.4425223469734192,
      "reward_std": 0.20599256455898285,
      "rewards/accuracy_reward": 0.07142857578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3710937649011612,
      "step": 947
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1624.0938415527344,
      "epoch": 0.2831752669703532,
      "grad_norm": 42.41265106201172,
      "kl": 4.7421875,
      "learning_rate": 4.506172716854256e-07,
      "loss": 0.3438,
      "reward": 0.4129464402794838,
      "reward_std": 0.19797534495592117,
      "rewards/accuracy_reward": 0.06919643236324191,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3437500223517418,
      "step": 948
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1641.9063110351562,
      "epoch": 0.2834739750578747,
      "grad_norm": 58.391300201416016,
      "kl": 3.50390625,
      "learning_rate": 4.5046157091050843e-07,
      "loss": 0.3174,
      "reward": 0.344866082072258,
      "reward_std": 0.15947942808270454,
      "rewards/accuracy_reward": 0.0022321429569274187,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3426339402794838,
      "step": 949
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1557.6050109863281,
      "epoch": 0.28377268314539617,
      "grad_norm": 75.96441650390625,
      "kl": 3.1875,
      "learning_rate": 4.503056520530494e-07,
      "loss": 0.318,
      "reward": 0.4492187723517418,
      "reward_std": 0.15650830045342445,
      "rewards/accuracy_reward": 0.07366071757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3755580484867096,
      "step": 950
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1515.0648193359375,
      "epoch": 0.28407139123291764,
      "grad_norm": 61.73088073730469,
      "kl": 3.171875,
      "learning_rate": 4.501495152826731e-07,
      "loss": 0.3248,
      "reward": 0.4587053805589676,
      "reward_std": 0.2060912624001503,
      "rewards/accuracy_reward": 0.0915178619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875149011612,
      "step": 951
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1574.7210693359375,
      "epoch": 0.2843700993204391,
      "grad_norm": 11.68893051147461,
      "kl": 4.92578125,
      "learning_rate": 4.4999316076924075e-07,
      "loss": 0.407,
      "reward": 0.419084832072258,
      "reward_std": 0.189629964530468,
      "rewards/accuracy_reward": 0.058035717345774174,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.361049123108387,
      "step": 952
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1473.0201416015625,
      "epoch": 0.2846688074079606,
      "grad_norm": 22.036500930786133,
      "kl": 4.734375,
      "learning_rate": 4.498365886828511e-07,
      "loss": 0.4481,
      "reward": 0.4520089477300644,
      "reward_std": 0.2107294723391533,
      "rewards/accuracy_reward": 0.0803571455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3716517984867096,
      "step": 953
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1486.7702026367188,
      "epoch": 0.28496751549548205,
      "grad_norm": 18.980680465698242,
      "kl": 4.44921875,
      "learning_rate": 4.4967979919383894e-07,
      "loss": 0.3634,
      "reward": 0.4302455559372902,
      "reward_std": 0.1791878640651703,
      "rewards/accuracy_reward": 0.04017857299186289,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3900669813156128,
      "step": 954
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1533.7411499023438,
      "epoch": 0.2852662235830035,
      "grad_norm": 54.8022575378418,
      "kl": 6.3828125,
      "learning_rate": 4.4952279247277603e-07,
      "loss": 0.493,
      "reward": 0.4107143059372902,
      "reward_std": 0.22598502039909363,
      "rewards/accuracy_reward": 0.04910714575089514,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.361607164144516,
      "step": 955
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1552.0580749511719,
      "epoch": 0.285564931670525,
      "grad_norm": 65.22401428222656,
      "kl": 5.83984375,
      "learning_rate": 4.493655686904702e-07,
      "loss": 0.415,
      "reward": 0.4436384066939354,
      "reward_std": 0.22857194393873215,
      "rewards/accuracy_reward": 0.0825892873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.361049123108387,
      "step": 956
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1624.6116638183594,
      "epoch": 0.28586363975804646,
      "grad_norm": 105.107666015625,
      "kl": 6.9609375,
      "learning_rate": 4.492081280179656e-07,
      "loss": 0.4641,
      "reward": 0.3911830484867096,
      "reward_std": 0.20640220120549202,
      "rewards/accuracy_reward": 0.04241071571595967,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3487723395228386,
      "step": 957
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1497.2634582519531,
      "epoch": 0.28616234784556793,
      "grad_norm": 10.020505905151367,
      "kl": 5.625,
      "learning_rate": 4.490504706265422e-07,
      "loss": 0.4824,
      "reward": 0.4882812723517418,
      "reward_std": 0.191363837569952,
      "rewards/accuracy_reward": 0.10937500488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3789062649011612,
      "step": 958
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1601.649658203125,
      "epoch": 0.2864610559330894,
      "grad_norm": 35.52611541748047,
      "kl": 5.37109375,
      "learning_rate": 4.488925966877158e-07,
      "loss": 0.4114,
      "reward": 0.478794664144516,
      "reward_std": 0.22123905643820763,
      "rewards/accuracy_reward": 0.11383929220028222,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.364955373108387,
      "step": 959
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1526.0022888183594,
      "epoch": 0.2867597640206109,
      "grad_norm": 30.565229415893555,
      "kl": 6.0078125,
      "learning_rate": 4.487345063732377e-07,
      "loss": 0.5076,
      "reward": 0.4280134066939354,
      "reward_std": 0.18954980745911598,
      "rewards/accuracy_reward": 0.06919643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3588169813156128,
      "step": 960
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1549.66748046875,
      "epoch": 0.28705847210813235,
      "grad_norm": 12.109847068786621,
      "kl": 4.96875,
      "learning_rate": 4.485761998550949e-07,
      "loss": 0.4055,
      "reward": 0.446986623108387,
      "reward_std": 0.17507700063288212,
      "rewards/accuracy_reward": 0.0691964328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3777901977300644,
      "step": 961
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1553.6228637695312,
      "epoch": 0.2873571801956538,
      "grad_norm": 13.964942932128906,
      "kl": 4.24609375,
      "learning_rate": 4.4841767730550904e-07,
      "loss": 0.3489,
      "reward": 0.4916294813156128,
      "reward_std": 0.22823909297585487,
      "rewards/accuracy_reward": 0.1339285783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3577009066939354,
      "step": 962
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1479.46435546875,
      "epoch": 0.2876558882831753,
      "grad_norm": 21.5960636138916,
      "kl": 4.5859375,
      "learning_rate": 4.4825893889693746e-07,
      "loss": 0.398,
      "reward": 0.4308035895228386,
      "reward_std": 0.1750134490430355,
      "rewards/accuracy_reward": 0.05357143236324191,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3772321566939354,
      "step": 963
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1552.9732666015625,
      "epoch": 0.28795459637069676,
      "grad_norm": 44.17295837402344,
      "kl": 3.70703125,
      "learning_rate": 4.4809998480207174e-07,
      "loss": 0.3348,
      "reward": 0.4112723395228386,
      "reward_std": 0.19608675688505173,
      "rewards/accuracy_reward": 0.03348214481957257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3777901977300644,
      "step": 964
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1489.4197082519531,
      "epoch": 0.28825330445821823,
      "grad_norm": 48.88083267211914,
      "kl": 4.015625,
      "learning_rate": 4.479408151938384e-07,
      "loss": 0.3753,
      "reward": 0.3761160895228386,
      "reward_std": 0.1502522975206375,
      "rewards/accuracy_reward": 0.004464285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3716518059372902,
      "step": 965
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1529.0513916015625,
      "epoch": 0.2885520125457397,
      "grad_norm": 28.302608489990234,
      "kl": 5.0546875,
      "learning_rate": 4.4778143024539847e-07,
      "loss": 0.3911,
      "reward": 0.5027902126312256,
      "reward_std": 0.21731223165988922,
      "rewards/accuracy_reward": 0.12053571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3822544813156128,
      "step": 966
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1562.0804138183594,
      "epoch": 0.28885072063326117,
      "grad_norm": 73.09947967529297,
      "kl": 6.359375,
      "learning_rate": 4.476218301301471e-07,
      "loss": 0.4723,
      "reward": 0.3532366305589676,
      "reward_std": 0.16485610231757164,
      "rewards/accuracy_reward": 0.004464285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3487723395228386,
      "step": 967
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1620.7634582519531,
      "epoch": 0.28914942872078264,
      "grad_norm": 33.76585006713867,
      "kl": 5.234375,
      "learning_rate": 4.4746201502171355e-07,
      "loss": 0.3977,
      "reward": 0.4179687649011612,
      "reward_std": 0.22551409900188446,
      "rewards/accuracy_reward": 0.055803571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3621651977300644,
      "step": 968
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1590.0357666015625,
      "epoch": 0.28944813680830406,
      "grad_norm": 9.169942855834961,
      "kl": 5.0703125,
      "learning_rate": 4.4730198509396104e-07,
      "loss": 0.4131,
      "reward": 0.4453125223517418,
      "reward_std": 0.23576795309782028,
      "rewards/accuracy_reward": 0.08705357508733869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3582589477300644,
      "step": 969
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1523.6763916015625,
      "epoch": 0.28974684489582553,
      "grad_norm": 41.497596740722656,
      "kl": 4.0,
      "learning_rate": 4.471417405209864e-07,
      "loss": 0.351,
      "reward": 0.4637276977300644,
      "reward_std": 0.23182670772075653,
      "rewards/accuracy_reward": 0.082589291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3811384066939354,
      "step": 970
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1561.7188110351562,
      "epoch": 0.290045552983347,
      "grad_norm": 14.74995231628418,
      "kl": 4.91015625,
      "learning_rate": 4.4698128147712004e-07,
      "loss": 0.4235,
      "reward": 0.5089285969734192,
      "reward_std": 0.2200334146618843,
      "rewards/accuracy_reward": 0.14732143888249993,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.361607164144516,
      "step": 971
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1591.1340026855469,
      "epoch": 0.29034426107086847,
      "grad_norm": 13.123516082763672,
      "kl": 5.078125,
      "learning_rate": 4.468206081369257e-07,
      "loss": 0.4161,
      "reward": 0.3660714477300644,
      "reward_std": 0.17188263684511185,
      "rewards/accuracy_reward": 0.008928572060540318,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3571428805589676,
      "step": 972
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1512.4866638183594,
      "epoch": 0.29064296915838994,
      "grad_norm": 35.884769439697266,
      "kl": 4.2421875,
      "learning_rate": 4.466597206752002e-07,
      "loss": 0.4024,
      "reward": 0.4453125298023224,
      "reward_std": 0.22344591841101646,
      "rewards/accuracy_reward": 0.08705357578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3582589477300644,
      "step": 973
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1608.1407165527344,
      "epoch": 0.2909416772459114,
      "grad_norm": 25.00407600402832,
      "kl": 4.23828125,
      "learning_rate": 4.464986192669733e-07,
      "loss": 0.3379,
      "reward": 0.4469866156578064,
      "reward_std": 0.1797037310898304,
      "rewards/accuracy_reward": 0.08705357578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3599330484867096,
      "step": 974
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1451.8259582519531,
      "epoch": 0.2912403853334329,
      "grad_norm": 42.25094985961914,
      "kl": 3.64453125,
      "learning_rate": 4.463373040875076e-07,
      "loss": 0.3223,
      "reward": 0.4804687723517418,
      "reward_std": 0.1670316755771637,
      "rewards/accuracy_reward": 0.09151786006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3889509066939354,
      "step": 975
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1466.7344665527344,
      "epoch": 0.29153909342095435,
      "grad_norm": 38.1767692565918,
      "kl": 4.31640625,
      "learning_rate": 4.461757753122981e-07,
      "loss": 0.4222,
      "reward": 0.4626116305589676,
      "reward_std": 0.18099632114171982,
      "rewards/accuracy_reward": 0.06696428963914514,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3956473395228386,
      "step": 976
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1517.8505249023438,
      "epoch": 0.2918378015084758,
      "grad_norm": 28.05716896057129,
      "kl": 4.59765625,
      "learning_rate": 4.460140331170724e-07,
      "loss": 0.4062,
      "reward": 0.446986623108387,
      "reward_std": 0.1984858587384224,
      "rewards/accuracy_reward": 0.06473214761354029,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3822544813156128,
      "step": 977
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1560.8929443359375,
      "epoch": 0.2921365095959973,
      "grad_norm": 14.244260787963867,
      "kl": 5.203125,
      "learning_rate": 4.4585207767779016e-07,
      "loss": 0.4359,
      "reward": 0.4693080633878708,
      "reward_std": 0.16065412759780884,
      "rewards/accuracy_reward": 0.098214291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3710937649011612,
      "step": 978
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1561.8661499023438,
      "epoch": 0.29243521768351877,
      "grad_norm": 71.85823822021484,
      "kl": 5.59375,
      "learning_rate": 4.4568990917064294e-07,
      "loss": 0.3834,
      "reward": 0.5167410969734192,
      "reward_std": 0.20071503147482872,
      "rewards/accuracy_reward": 0.13169643515720963,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3850446492433548,
      "step": 979
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1588.7679138183594,
      "epoch": 0.29273392577104024,
      "grad_norm": 24.071256637573242,
      "kl": 5.5546875,
      "learning_rate": 4.4552752777205414e-07,
      "loss": 0.4099,
      "reward": 0.4335937723517418,
      "reward_std": 0.1797768399119377,
      "rewards/accuracy_reward": 0.0602678619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3733259066939354,
      "step": 980
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1480.4375915527344,
      "epoch": 0.2930326338585617,
      "grad_norm": 75.88976287841797,
      "kl": 6.81640625,
      "learning_rate": 4.45364933658679e-07,
      "loss": 0.5181,
      "reward": 0.517857164144516,
      "reward_std": 0.1631302312016487,
      "rewards/accuracy_reward": 0.1473214365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3705357387661934,
      "step": 981
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1627.0335693359375,
      "epoch": 0.2933313419460832,
      "grad_norm": 103.74700164794922,
      "kl": 7.671875,
      "learning_rate": 4.452021270074038e-07,
      "loss": 0.5307,
      "reward": 0.3777901902794838,
      "reward_std": 0.15603306517004967,
      "rewards/accuracy_reward": 0.04017857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.337611623108387,
      "step": 982
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1560.0380249023438,
      "epoch": 0.29363005003360465,
      "grad_norm": 70.32150268554688,
      "kl": 6.328125,
      "learning_rate": 4.4503910799534626e-07,
      "loss": 0.4856,
      "reward": 0.4704241305589676,
      "reward_std": 0.19276686757802963,
      "rewards/accuracy_reward": 0.10267857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3677455559372902,
      "step": 983
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1486.7656860351562,
      "epoch": 0.2939287581211261,
      "grad_norm": 20.87571907043457,
      "kl": 4.15234375,
      "learning_rate": 4.448758767998552e-07,
      "loss": 0.3801,
      "reward": 0.4687500149011612,
      "reward_std": 0.1901467889547348,
      "rewards/accuracy_reward": 0.07812500605359674,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3906250223517418,
      "step": 984
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1568.76123046875,
      "epoch": 0.2942274662086476,
      "grad_norm": 39.32463455200195,
      "kl": 5.6796875,
      "learning_rate": 4.4471243359851e-07,
      "loss": 0.4367,
      "reward": 0.4430803805589676,
      "reward_std": 0.23091403022408485,
      "rewards/accuracy_reward": 0.0848214328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3582589402794838,
      "step": 985
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1467.9688110351562,
      "epoch": 0.29452617429616906,
      "grad_norm": 29.723173141479492,
      "kl": 3.82421875,
      "learning_rate": 4.4454877856912086e-07,
      "loss": 0.3649,
      "reward": 0.518415205180645,
      "reward_std": 0.20138828083872795,
      "rewards/accuracy_reward": 0.11383929289877415,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.404575914144516,
      "step": 986
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1500.9018249511719,
      "epoch": 0.29482488238369053,
      "grad_norm": 19.145767211914062,
      "kl": 5.0859375,
      "learning_rate": 4.4438491188972853e-07,
      "loss": 0.4107,
      "reward": 0.510044664144516,
      "reward_std": 0.1924300603568554,
      "rewards/accuracy_reward": 0.1361607238650322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3738839477300644,
      "step": 987
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1585.5246276855469,
      "epoch": 0.295123590471212,
      "grad_norm": 27.292633056640625,
      "kl": 4.046875,
      "learning_rate": 4.442208337386039e-07,
      "loss": 0.3557,
      "reward": 0.4414062649011612,
      "reward_std": 0.22749869152903557,
      "rewards/accuracy_reward": 0.08258928917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3588169813156128,
      "step": 988
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1594.3906860351562,
      "epoch": 0.2954222985587335,
      "grad_norm": 28.84008026123047,
      "kl": 4.2734375,
      "learning_rate": 4.4405654429424774e-07,
      "loss": 0.375,
      "reward": 0.4296875074505806,
      "reward_std": 0.18899821117520332,
      "rewards/accuracy_reward": 0.0870535746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3426339477300644,
      "step": 989
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1470.0580749511719,
      "epoch": 0.29572100664625495,
      "grad_norm": 8.223483085632324,
      "kl": 4.4609375,
      "learning_rate": 4.43892043735391e-07,
      "loss": 0.3745,
      "reward": 0.424665205180645,
      "reward_std": 0.17849614843726158,
      "rewards/accuracy_reward": 0.04241071757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3822544813156128,
      "step": 990
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1550.1898193359375,
      "epoch": 0.2960197147337764,
      "grad_norm": 17.552621841430664,
      "kl": 4.7734375,
      "learning_rate": 4.4372733224099413e-07,
      "loss": 0.4114,
      "reward": 0.4648437798023224,
      "reward_std": 0.20557358115911484,
      "rewards/accuracy_reward": 0.1093750037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3554687649011612,
      "step": 991
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1606.5022583007812,
      "epoch": 0.2963184228212979,
      "grad_norm": 63.84352111816406,
      "kl": 6.5546875,
      "learning_rate": 4.4356240999024715e-07,
      "loss": 0.4849,
      "reward": 0.4308035895228386,
      "reward_std": 0.1798197366297245,
      "rewards/accuracy_reward": 0.06696428847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3638392984867096,
      "step": 992
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1604.8348693847656,
      "epoch": 0.29661713090881936,
      "grad_norm": 21.80406379699707,
      "kl": 6.125,
      "learning_rate": 4.4339727716256927e-07,
      "loss": 0.4763,
      "reward": 0.4034598395228386,
      "reward_std": 0.16411307454109192,
      "rewards/accuracy_reward": 0.04687500186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3565848395228386,
      "step": 993
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1539.1942443847656,
      "epoch": 0.29691583899634083,
      "grad_norm": 51.44820022583008,
      "kl": 6.0703125,
      "learning_rate": 4.432319339376087e-07,
      "loss": 0.4495,
      "reward": 0.4799107313156128,
      "reward_std": 0.21330787986516953,
      "rewards/accuracy_reward": 0.10044643469154835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3794643059372902,
      "step": 994
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1639.6094360351562,
      "epoch": 0.2972145470838623,
      "grad_norm": 79.29431915283203,
      "kl": 6.3515625,
      "learning_rate": 4.4306638049524286e-07,
      "loss": 0.4432,
      "reward": 0.4425223395228386,
      "reward_std": 0.1793471798300743,
      "rewards/accuracy_reward": 0.09598214412108064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3465401902794838,
      "step": 995
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1640.0134582519531,
      "epoch": 0.2975132551713838,
      "grad_norm": 58.035011291503906,
      "kl": 6.1015625,
      "learning_rate": 4.4290061701557756e-07,
      "loss": 0.4487,
      "reward": 0.4983259215950966,
      "reward_std": 0.21006934344768524,
      "rewards/accuracy_reward": 0.14732143748551607,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3510044887661934,
      "step": 996
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1547.165283203125,
      "epoch": 0.29781196325890524,
      "grad_norm": 11.679484367370605,
      "kl": 4.67578125,
      "learning_rate": 4.427346436789472e-07,
      "loss": 0.3812,
      "reward": 0.4642857387661934,
      "reward_std": 0.20103354007005692,
      "rewards/accuracy_reward": 0.08705357578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.377232164144516,
      "step": 997
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1519.5759582519531,
      "epoch": 0.2981106713464267,
      "grad_norm": 14.813997268676758,
      "kl": 4.4140625,
      "learning_rate": 4.425684606659146e-07,
      "loss": 0.3565,
      "reward": 0.5150669813156128,
      "reward_std": 0.19538959488272667,
      "rewards/accuracy_reward": 0.13839286286383867,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.376674123108387,
      "step": 998
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1577.0045166015625,
      "epoch": 0.2984093794339482,
      "grad_norm": 34.87970733642578,
      "kl": 3.97265625,
      "learning_rate": 4.4240206815727054e-07,
      "loss": 0.3433,
      "reward": 0.4525669887661934,
      "reward_std": 0.20338879898190498,
      "rewards/accuracy_reward": 0.08705357648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.365513414144516,
      "step": 999
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1577.1027526855469,
      "epoch": 0.29870808752146966,
      "grad_norm": 8.51466178894043,
      "kl": 4.74609375,
      "learning_rate": 4.4223546633403373e-07,
      "loss": 0.3881,
      "reward": 0.4375000223517418,
      "reward_std": 0.16045677661895752,
      "rewards/accuracy_reward": 0.0781250037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3593750149011612,
      "step": 1000
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1469.6429443359375,
      "epoch": 0.2990067956089911,
      "grad_norm": 51.540889739990234,
      "kl": 3.95703125,
      "learning_rate": 4.420686553774506e-07,
      "loss": 0.3679,
      "reward": 0.4531250223517418,
      "reward_std": 0.19104110449552536,
      "rewards/accuracy_reward": 0.06473214668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.388392873108387,
      "step": 1001
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1485.7745971679688,
      "epoch": 0.2993055036965126,
      "grad_norm": 9.281465530395508,
      "kl": 4.4453125,
      "learning_rate": 4.4190163546899527e-07,
      "loss": 0.3859,
      "reward": 0.4676339477300644,
      "reward_std": 0.22507762908935547,
      "rewards/accuracy_reward": 0.09375000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3738839477300644,
      "step": 1002
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1518.8772888183594,
      "epoch": 0.29960421178403407,
      "grad_norm": 42.958457946777344,
      "kl": 4.28515625,
      "learning_rate": 4.4173440679036885e-07,
      "loss": 0.3902,
      "reward": 0.5251116305589676,
      "reward_std": 0.20143787190318108,
      "rewards/accuracy_reward": 0.1428571529686451,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3822544813156128,
      "step": 1003
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1593.94873046875,
      "epoch": 0.29990291987155554,
      "grad_norm": 65.59077453613281,
      "kl": 5.59375,
      "learning_rate": 4.415669695234998e-07,
      "loss": 0.3752,
      "reward": 0.4754464477300644,
      "reward_std": 0.20330213755369186,
      "rewards/accuracy_reward": 0.10491071944124997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3705357313156128,
      "step": 1004
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1592.5670166015625,
      "epoch": 0.300201627959077,
      "grad_norm": 16.7629451751709,
      "kl": 4.984375,
      "learning_rate": 4.413993238505436e-07,
      "loss": 0.4012,
      "reward": 0.4955357313156128,
      "reward_std": 0.16545912250876427,
      "rewards/accuracy_reward": 0.1183035746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.377232164144516,
      "step": 1005
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1588.5179443359375,
      "epoch": 0.3005003360465985,
      "grad_norm": 108.32943725585938,
      "kl": 6.8046875,
      "learning_rate": 4.4123146995388215e-07,
      "loss": 0.4744,
      "reward": 0.3710937723517418,
      "reward_std": 0.2016550935804844,
      "rewards/accuracy_reward": 0.022321430267766118,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3487723469734192,
      "step": 1006
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1499.3951721191406,
      "epoch": 0.30079904413411995,
      "grad_norm": 32.783966064453125,
      "kl": 5.46875,
      "learning_rate": 4.4106340801612405e-07,
      "loss": 0.4417,
      "reward": 0.4564732238650322,
      "reward_std": 0.21038522943854332,
      "rewards/accuracy_reward": 0.08035714481957257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3761160895228386,
      "step": 1007
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1608.5782165527344,
      "epoch": 0.3010977522216414,
      "grad_norm": 48.1810417175293,
      "kl": 5.8828125,
      "learning_rate": 4.4089513822010435e-07,
      "loss": 0.4162,
      "reward": 0.448660746216774,
      "reward_std": 0.15346087515354156,
      "rewards/accuracy_reward": 0.08035714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3683035895228386,
      "step": 1008
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1617.8125610351562,
      "epoch": 0.3013964603091629,
      "grad_norm": 26.268178939819336,
      "kl": 4.609375,
      "learning_rate": 4.407266607488839e-07,
      "loss": 0.3183,
      "reward": 0.426897332072258,
      "reward_std": 0.1750059649348259,
      "rewards/accuracy_reward": 0.04910714505240321,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3777901902794838,
      "step": 1009
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1541.1049499511719,
      "epoch": 0.30169516839668437,
      "grad_norm": 17.426828384399414,
      "kl": 4.125,
      "learning_rate": 4.4055797578574983e-07,
      "loss": 0.343,
      "reward": 0.4743303805589676,
      "reward_std": 0.1794714257121086,
      "rewards/accuracy_reward": 0.08928572107106447,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3850446566939354,
      "step": 1010
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1525.3795471191406,
      "epoch": 0.30199387648420584,
      "grad_norm": 50.402915954589844,
      "kl": 3.671875,
      "learning_rate": 4.403890835142148e-07,
      "loss": 0.3282,
      "reward": 0.4966518059372902,
      "reward_std": 0.16985390335321426,
      "rewards/accuracy_reward": 0.0959821492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4006696492433548,
      "step": 1011
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1591.0380249023438,
      "epoch": 0.30229258457172725,
      "grad_norm": 7.6185736656188965,
      "kl": 4.71875,
      "learning_rate": 4.4021998411801705e-07,
      "loss": 0.3687,
      "reward": 0.5139509215950966,
      "reward_std": 0.23822612315416336,
      "rewards/accuracy_reward": 0.1383928656578064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3755580484867096,
      "step": 1012
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1554.3215026855469,
      "epoch": 0.3025912926592487,
      "grad_norm": 53.159149169921875,
      "kl": 3.890625,
      "learning_rate": 4.400506777811201e-07,
      "loss": 0.3674,
      "reward": 0.446986623108387,
      "reward_std": 0.20431933924555779,
      "rewards/accuracy_reward": 0.07142857694998384,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3755580484867096,
      "step": 1013
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1580.9085388183594,
      "epoch": 0.3028900007467702,
      "grad_norm": 9.96097183227539,
      "kl": 4.517578125,
      "learning_rate": 4.3988116468771275e-07,
      "loss": 0.3683,
      "reward": 0.499441996216774,
      "reward_std": 0.18105580657720566,
      "rewards/accuracy_reward": 0.12946429033763707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3699776902794838,
      "step": 1014
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1578.1004943847656,
      "epoch": 0.30318870883429166,
      "grad_norm": 9.517431259155273,
      "kl": 4.7421875,
      "learning_rate": 4.397114450222085e-07,
      "loss": 0.3903,
      "reward": 0.4614955559372902,
      "reward_std": 0.20815807580947876,
      "rewards/accuracy_reward": 0.10044643096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.361049123108387,
      "step": 1015
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1455.5982666015625,
      "epoch": 0.30348741692181314,
      "grad_norm": 49.265785217285156,
      "kl": 3.33984375,
      "learning_rate": 4.3954151896924586e-07,
      "loss": 0.316,
      "reward": 0.5212053805589676,
      "reward_std": 0.19894807785749435,
      "rewards/accuracy_reward": 0.10937500651925802,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.411830373108387,
      "step": 1016
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1542.2009582519531,
      "epoch": 0.3037861250093346,
      "grad_norm": 14.753308296203613,
      "kl": 3.91796875,
      "learning_rate": 4.3937138671368756e-07,
      "loss": 0.3152,
      "reward": 0.4994419887661934,
      "reward_std": 0.21065231412649155,
      "rewards/accuracy_reward": 0.10267857578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.396763414144516,
      "step": 1017
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1526.35498046875,
      "epoch": 0.3040848330968561,
      "grad_norm": 6.20982027053833,
      "kl": 4.34375,
      "learning_rate": 4.39201048440621e-07,
      "loss": 0.3671,
      "reward": 0.5825893059372902,
      "reward_std": 0.20545059069991112,
      "rewards/accuracy_reward": 0.20312500977888703,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3794642984867096,
      "step": 1018
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1475.0581359863281,
      "epoch": 0.30438354118437755,
      "grad_norm": 35.25672149658203,
      "kl": 3.994140625,
      "learning_rate": 4.390305043353574e-07,
      "loss": 0.3371,
      "reward": 0.5440848469734192,
      "reward_std": 0.17119980603456497,
      "rewards/accuracy_reward": 0.1383928656578064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4056919813156128,
      "step": 1019
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1546.8438415527344,
      "epoch": 0.304682249271899,
      "grad_norm": 34.416229248046875,
      "kl": 5.8359375,
      "learning_rate": 4.38859754583432e-07,
      "loss": 0.4598,
      "reward": 0.5066964477300644,
      "reward_std": 0.1853574588894844,
      "rewards/accuracy_reward": 0.12276786426082253,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.383928582072258,
      "step": 1020
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1491.5625610351562,
      "epoch": 0.3049809573594205,
      "grad_norm": 86.2718505859375,
      "kl": 6.28125,
      "learning_rate": 4.386887993706039e-07,
      "loss": 0.4302,
      "reward": 0.459821455180645,
      "reward_std": 0.19159509614109993,
      "rewards/accuracy_reward": 0.07142857485450804,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.388392873108387,
      "step": 1021
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1563.9844665527344,
      "epoch": 0.30527966544694196,
      "grad_norm": 82.9708480834961,
      "kl": 6.203125,
      "learning_rate": 4.385176388828556e-07,
      "loss": 0.4386,
      "reward": 0.4068080484867096,
      "reward_std": 0.18860863521695137,
      "rewards/accuracy_reward": 0.029017858672887087,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3777901902794838,
      "step": 1022
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1546.7523193359375,
      "epoch": 0.30557837353446343,
      "grad_norm": 40.68043899536133,
      "kl": 5.1328125,
      "learning_rate": 4.383462733063929e-07,
      "loss": 0.3875,
      "reward": 0.553571455180645,
      "reward_std": 0.24302224814891815,
      "rewards/accuracy_reward": 0.15848215110599995,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3950893059372902,
      "step": 1023
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1539.2255249023438,
      "epoch": 0.3058770816219849,
      "grad_norm": 14.853148460388184,
      "kl": 5.0859375,
      "learning_rate": 4.381747028276448e-07,
      "loss": 0.3968,
      "reward": 0.4051339477300644,
      "reward_std": 0.1969018653035164,
      "rewards/accuracy_reward": 0.02455357206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.380580373108387,
      "step": 1024
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1591.21435546875,
      "epoch": 0.3061757897095064,
      "grad_norm": 19.180971145629883,
      "kl": 3.9375,
      "learning_rate": 4.3800292763326307e-07,
      "loss": 0.2954,
      "reward": 0.446986623108387,
      "reward_std": 0.23462994769215584,
      "rewards/accuracy_reward": 0.06473214784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3822544813156128,
      "step": 1025
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1544.1652526855469,
      "epoch": 0.30647449779702785,
      "grad_norm": 33.8619270324707,
      "kl": 4.109375,
      "learning_rate": 4.3783094791012233e-07,
      "loss": 0.36,
      "reward": 0.5513393133878708,
      "reward_std": 0.1862843707203865,
      "rewards/accuracy_reward": 0.1718750074505806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3794642984867096,
      "step": 1026
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1590.2054138183594,
      "epoch": 0.3067732058845493,
      "grad_norm": 9.645373344421387,
      "kl": 4.8984375,
      "learning_rate": 4.376587638453197e-07,
      "loss": 0.3721,
      "reward": 0.4084821566939354,
      "reward_std": 0.20559799298644066,
      "rewards/accuracy_reward": 0.03125000116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3772321566939354,
      "step": 1027
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1626.8415832519531,
      "epoch": 0.3070719139720708,
      "grad_norm": 8.181839942932129,
      "kl": 4.46484375,
      "learning_rate": 4.374863756261745e-07,
      "loss": 0.3375,
      "reward": 0.4218750223517418,
      "reward_std": 0.18572933599352837,
      "rewards/accuracy_reward": 0.05580357299186289,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3660714477300644,
      "step": 1028
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1596.3348999023438,
      "epoch": 0.30737062205959226,
      "grad_norm": 17.298545837402344,
      "kl": 4.2265625,
      "learning_rate": 4.373137834402283e-07,
      "loss": 0.3378,
      "reward": 0.4235491380095482,
      "reward_std": 0.18014146015048027,
      "rewards/accuracy_reward": 0.0513392873108387,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.369977705180645,
      "step": 1029
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1569.7500610351562,
      "epoch": 0.30766933014711373,
      "grad_norm": 58.50421142578125,
      "kl": 3.69140625,
      "learning_rate": 4.3714098747524445e-07,
      "loss": 0.3306,
      "reward": 0.4391741305589676,
      "reward_std": 0.18023748695850372,
      "rewards/accuracy_reward": 0.05357143119908869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3856026977300644,
      "step": 1030
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1501.5871276855469,
      "epoch": 0.3079680382346352,
      "grad_norm": 46.79623031616211,
      "kl": 3.875,
      "learning_rate": 4.36967987919208e-07,
      "loss": 0.3609,
      "reward": 0.4425223469734192,
      "reward_std": 0.23066024482250214,
      "rewards/accuracy_reward": 0.0602678582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3822544738650322,
      "step": 1031
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1528.0558776855469,
      "epoch": 0.30826674632215667,
      "grad_norm": 9.182872772216797,
      "kl": 4.796875,
      "learning_rate": 4.367947849603256e-07,
      "loss": 0.388,
      "reward": 0.5251116380095482,
      "reward_std": 0.250782061368227,
      "rewards/accuracy_reward": 0.1428571492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3822544738650322,
      "step": 1032
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1529.5156860351562,
      "epoch": 0.30856545440967814,
      "grad_norm": 33.41720962524414,
      "kl": 5.34375,
      "learning_rate": 4.366213787870249e-07,
      "loss": 0.3879,
      "reward": 0.4268973395228386,
      "reward_std": 0.18928997591137886,
      "rewards/accuracy_reward": 0.04017857275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3867187574505806,
      "step": 1033
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1481.4219360351562,
      "epoch": 0.3088641624971996,
      "grad_norm": 9.278277397155762,
      "kl": 5.171875,
      "learning_rate": 4.3644776958795503e-07,
      "loss": 0.4309,
      "reward": 0.5200893208384514,
      "reward_std": 0.17495165392756462,
      "rewards/accuracy_reward": 0.11607143585570157,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.404017873108387,
      "step": 1034
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1496.4464721679688,
      "epoch": 0.3091628705847211,
      "grad_norm": 47.507598876953125,
      "kl": 5.421875,
      "learning_rate": 4.362739575519856e-07,
      "loss": 0.4096,
      "reward": 0.5005580559372902,
      "reward_std": 0.179439514875412,
      "rewards/accuracy_reward": 0.10044643329456449,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.400111623108387,
      "step": 1035
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1551.5491943359375,
      "epoch": 0.30946157867224255,
      "grad_norm": 228.46095275878906,
      "kl": 6.5,
      "learning_rate": 4.360999428682073e-07,
      "loss": 0.4317,
      "reward": 0.4966518059372902,
      "reward_std": 0.18391728959977627,
      "rewards/accuracy_reward": 0.0892857164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4073660895228386,
      "step": 1036
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1454.8192443847656,
      "epoch": 0.309760286759764,
      "grad_norm": 32.96681213378906,
      "kl": 3.765625,
      "learning_rate": 4.3592572572593076e-07,
      "loss": 0.3312,
      "reward": 0.5318080633878708,
      "reward_std": 0.19193265959620476,
      "rewards/accuracy_reward": 0.12946429220028222,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4023437723517418,
      "step": 1037
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1468.8505249023438,
      "epoch": 0.3100589948472855,
      "grad_norm": 28.990102767944336,
      "kl": 3.87109375,
      "learning_rate": 4.357513063146874e-07,
      "loss": 0.3639,
      "reward": 0.5318080559372902,
      "reward_std": 0.22349325940012932,
      "rewards/accuracy_reward": 0.13169643329456449,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.400111623108387,
      "step": 1038
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1481.2679138183594,
      "epoch": 0.31035770293480697,
      "grad_norm": 68.97732543945312,
      "kl": 2.6484375,
      "learning_rate": 4.3557668482422836e-07,
      "loss": 0.2548,
      "reward": 0.5106027126312256,
      "reward_std": 0.19009055942296982,
      "rewards/accuracy_reward": 0.0959821455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205559372902,
      "step": 1039
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1477.6116638183594,
      "epoch": 0.31065641102232844,
      "grad_norm": 13.56062126159668,
      "kl": 5.13671875,
      "learning_rate": 4.3540186144452466e-07,
      "loss": 0.4411,
      "reward": 0.5050223395228386,
      "reward_std": 0.1911173053085804,
      "rewards/accuracy_reward": 0.10044643632136285,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4045759066939354,
      "step": 1040
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1573.7277526855469,
      "epoch": 0.3109551191098499,
      "grad_norm": 53.514617919921875,
      "kl": 5.609375,
      "learning_rate": 4.3522683636576706e-07,
      "loss": 0.4143,
      "reward": 0.4034598395228386,
      "reward_std": 0.1875198446214199,
      "rewards/accuracy_reward": 0.03794643119908869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.365513414144516,
      "step": 1041
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1413.1362609863281,
      "epoch": 0.3112538271973714,
      "grad_norm": 31.889387130737305,
      "kl": 4.9140625,
      "learning_rate": 4.3505160977836566e-07,
      "loss": 0.402,
      "reward": 0.5172991380095482,
      "reward_std": 0.18907925486564636,
      "rewards/accuracy_reward": 0.11383928870782256,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4034598469734192,
      "step": 1042
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1490.4844665527344,
      "epoch": 0.31155253528489285,
      "grad_norm": 9.542855262756348,
      "kl": 4.94140625,
      "learning_rate": 4.3487618187294983e-07,
      "loss": 0.4084,
      "reward": 0.4369419813156128,
      "reward_std": 0.2156931683421135,
      "rewards/accuracy_reward": 0.04910714412108064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.387834832072258,
      "step": 1043
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1484.6384582519531,
      "epoch": 0.3118512433724143,
      "grad_norm": 30.693645477294922,
      "kl": 5.2421875,
      "learning_rate": 4.34700552840368e-07,
      "loss": 0.4237,
      "reward": 0.4492187723517418,
      "reward_std": 0.2037702091038227,
      "rewards/accuracy_reward": 0.06696428824216127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3822544813156128,
      "step": 1044
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1456.7255249023438,
      "epoch": 0.3121499514599358,
      "grad_norm": 10.81494140625,
      "kl": 4.84765625,
      "learning_rate": 4.345247228716872e-07,
      "loss": 0.4383,
      "reward": 0.4492187649011612,
      "reward_std": 0.1630881167948246,
      "rewards/accuracy_reward": 0.04910714412108064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.400111623108387,
      "step": 1045
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1531.1161499023438,
      "epoch": 0.31244865954745726,
      "grad_norm": 52.00796127319336,
      "kl": 3.421875,
      "learning_rate": 4.3434869215819334e-07,
      "loss": 0.3163,
      "reward": 0.4626116380095482,
      "reward_std": 0.1817893274128437,
      "rewards/accuracy_reward": 0.06473214644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3978794738650322,
      "step": 1046
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1573.1473999023438,
      "epoch": 0.31274736763497873,
      "grad_norm": 56.746829986572266,
      "kl": 3.2421875,
      "learning_rate": 4.341724608913905e-07,
      "loss": 0.2804,
      "reward": 0.4983259215950966,
      "reward_std": 0.191475048661232,
      "rewards/accuracy_reward": 0.1026785783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.395647332072258,
      "step": 1047
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1457.8148193359375,
      "epoch": 0.3130460757225002,
      "grad_norm": 14.838501930236816,
      "kl": 4.3515625,
      "learning_rate": 4.3399602926300107e-07,
      "loss": 0.3507,
      "reward": 0.4944196715950966,
      "reward_std": 0.1919793225824833,
      "rewards/accuracy_reward": 0.0937500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.400669664144516,
      "step": 1048
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1465.8259582519531,
      "epoch": 0.3133447838100217,
      "grad_norm": 19.73334312438965,
      "kl": 3.8984375,
      "learning_rate": 4.3381939746496533e-07,
      "loss": 0.3351,
      "reward": 0.4938616305589676,
      "reward_std": 0.19539830833673477,
      "rewards/accuracy_reward": 0.08035714738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4135044738650322,
      "step": 1049
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1590.7969360351562,
      "epoch": 0.31364349189754315,
      "grad_norm": 12.36715316772461,
      "kl": 4.7734375,
      "learning_rate": 4.3364256568944135e-07,
      "loss": 0.3699,
      "reward": 0.4441964477300644,
      "reward_std": 0.19625502079725266,
      "rewards/accuracy_reward": 0.058035716880112886,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3861607387661934,
      "step": 1050
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1440.2210693359375,
      "epoch": 0.3139421999850646,
      "grad_norm": 7.537537574768066,
      "kl": 3.87109375,
      "learning_rate": 4.3346553412880483e-07,
      "loss": 0.3235,
      "reward": 0.5072544813156128,
      "reward_std": 0.16967600211501122,
      "rewards/accuracy_reward": 0.098214291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4090401977300644,
      "step": 1051
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1629.1272888183594,
      "epoch": 0.3142409080725861,
      "grad_norm": 60.840091705322266,
      "kl": 6.2578125,
      "learning_rate": 4.332883029756488e-07,
      "loss": 0.4206,
      "reward": 0.420200914144516,
      "reward_std": 0.18977253884077072,
      "rewards/accuracy_reward": 0.05357142956927419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3666294738650322,
      "step": 1052
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1557.0335388183594,
      "epoch": 0.31453961616010756,
      "grad_norm": 33.55450439453125,
      "kl": 4.13671875,
      "learning_rate": 4.3311087242278324e-07,
      "loss": 0.3377,
      "reward": 0.5825893133878708,
      "reward_std": 0.17647244408726692,
      "rewards/accuracy_reward": 0.1941964365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.388392873108387,
      "step": 1053
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1622.3594360351562,
      "epoch": 0.31483832424762903,
      "grad_norm": 10.01598834991455,
      "kl": 5.515625,
      "learning_rate": 4.329332426632353e-07,
      "loss": 0.4227,
      "reward": 0.4592634066939354,
      "reward_std": 0.19166633114218712,
      "rewards/accuracy_reward": 0.06696428963914514,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3922991305589676,
      "step": 1054
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1486.3170471191406,
      "epoch": 0.31513703233515045,
      "grad_norm": 19.091739654541016,
      "kl": 5.171875,
      "learning_rate": 4.3275541389024873e-07,
      "loss": 0.4047,
      "reward": 0.430803582072258,
      "reward_std": 0.18778089806437492,
      "rewards/accuracy_reward": 0.0334821455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3973214402794838,
      "step": 1055
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1499.7076721191406,
      "epoch": 0.3154357404226719,
      "grad_norm": 28.23240852355957,
      "kl": 5.2890625,
      "learning_rate": 4.325773862972838e-07,
      "loss": 0.4085,
      "reward": 0.4296875149011612,
      "reward_std": 0.19161390513181686,
      "rewards/accuracy_reward": 0.03125000209547579,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375149011612,
      "step": 1056
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1469.9688110351562,
      "epoch": 0.3157344485101934,
      "grad_norm": 54.29639434814453,
      "kl": 3.6171875,
      "learning_rate": 4.323991600780171e-07,
      "loss": 0.3544,
      "reward": 0.467075914144516,
      "reward_std": 0.17298756167292595,
      "rewards/accuracy_reward": 0.06250000325962901,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4045759066939354,
      "step": 1057
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1573.2656860351562,
      "epoch": 0.31603315659771486,
      "grad_norm": 40.57893753051758,
      "kl": 3.8046875,
      "learning_rate": 4.322207354263412e-07,
      "loss": 0.3293,
      "reward": 0.4146205559372902,
      "reward_std": 0.16566652804613113,
      "rewards/accuracy_reward": 0.013392857741564512,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4012276902794838,
      "step": 1058
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1496.7857971191406,
      "epoch": 0.31633186468523633,
      "grad_norm": 37.15719223022461,
      "kl": 4.11328125,
      "learning_rate": 4.320421125363648e-07,
      "loss": 0.3689,
      "reward": 0.5167410969734192,
      "reward_std": 0.1951860785484314,
      "rewards/accuracy_reward": 0.12276786309666932,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3939732387661934,
      "step": 1059
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1522.79248046875,
      "epoch": 0.3166305727727578,
      "grad_norm": 19.989177703857422,
      "kl": 4.04296875,
      "learning_rate": 4.31863291602412e-07,
      "loss": 0.3258,
      "reward": 0.4296875298023224,
      "reward_std": 0.18419376760721207,
      "rewards/accuracy_reward": 0.026785715715959668,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4029018059372902,
      "step": 1060
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1619.35498046875,
      "epoch": 0.31692928086027927,
      "grad_norm": 91.51507568359375,
      "kl": 6.6484375,
      "learning_rate": 4.316842728190224e-07,
      "loss": 0.4395,
      "reward": 0.440290205180645,
      "reward_std": 0.1580643653869629,
      "rewards/accuracy_reward": 0.0491071455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3911830559372902,
      "step": 1061
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1551.9129943847656,
      "epoch": 0.31722798894780074,
      "grad_norm": 91.06034088134766,
      "kl": 6.40625,
      "learning_rate": 4.31505056380951e-07,
      "loss": 0.457,
      "reward": 0.4475446715950966,
      "reward_std": 0.17632845789194107,
      "rewards/accuracy_reward": 0.05580357392318547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.391741082072258,
      "step": 1062
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1572.274658203125,
      "epoch": 0.3175266970353222,
      "grad_norm": 85.88459014892578,
      "kl": 6.6953125,
      "learning_rate": 4.313256424831676e-07,
      "loss": 0.4709,
      "reward": 0.5011160895228386,
      "reward_std": 0.1591567024588585,
      "rewards/accuracy_reward": 0.11607143748551607,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3850446566939354,
      "step": 1063
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1470.0781860351562,
      "epoch": 0.3178254051228437,
      "grad_norm": 26.231706619262695,
      "kl": 4.125,
      "learning_rate": 4.3114603132085703e-07,
      "loss": 0.3578,
      "reward": 0.486049123108387,
      "reward_std": 0.17275992408394814,
      "rewards/accuracy_reward": 0.07812500465661287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.407924123108387,
      "step": 1064
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1497.01123046875,
      "epoch": 0.31812411321036516,
      "grad_norm": 9.794968605041504,
      "kl": 4.78125,
      "learning_rate": 4.3096622308941863e-07,
      "loss": 0.3651,
      "reward": 0.4726562723517418,
      "reward_std": 0.15365530364215374,
      "rewards/accuracy_reward": 0.0758928582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.396763414144516,
      "step": 1065
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1451.2656860351562,
      "epoch": 0.3184228212978866,
      "grad_norm": 19.662031173706055,
      "kl": 4.23046875,
      "learning_rate": 4.3078621798446605e-07,
      "loss": 0.3343,
      "reward": 0.4888393133878708,
      "reward_std": 0.22691982984542847,
      "rewards/accuracy_reward": 0.06696428917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 1066
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1494.1897888183594,
      "epoch": 0.3187215293854081,
      "grad_norm": 47.1057014465332,
      "kl": 5.2109375,
      "learning_rate": 4.306060162018274e-07,
      "loss": 0.3819,
      "reward": 0.5022321566939354,
      "reward_std": 0.14717273227870464,
      "rewards/accuracy_reward": 0.08928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4129464477300644,
      "step": 1067
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1570.8974304199219,
      "epoch": 0.31902023747292957,
      "grad_norm": 27.620433807373047,
      "kl": 5.33984375,
      "learning_rate": 4.3042561793754425e-07,
      "loss": 0.4005,
      "reward": 0.4905134215950966,
      "reward_std": 0.18510233610868454,
      "rewards/accuracy_reward": 0.10491072130389512,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3856026902794838,
      "step": 1068
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1556.3415832519531,
      "epoch": 0.31931894556045104,
      "grad_norm": 13.912795066833496,
      "kl": 4.30859375,
      "learning_rate": 4.3024502338787246e-07,
      "loss": 0.3389,
      "reward": 0.5440848469734192,
      "reward_std": 0.2520570158958435,
      "rewards/accuracy_reward": 0.1450892947614193,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3989955559372902,
      "step": 1069
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1454.8170471191406,
      "epoch": 0.3196176536479725,
      "grad_norm": 93.93225860595703,
      "kl": 2.39453125,
      "learning_rate": 4.3006423274928103e-07,
      "loss": 0.2753,
      "reward": 0.5641741305589676,
      "reward_std": 0.19079000689089298,
      "rewards/accuracy_reward": 0.1473214365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4168526977300644,
      "step": 1070
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1446.9129943847656,
      "epoch": 0.319916361735494,
      "grad_norm": 79.51519775390625,
      "kl": 2.365234375,
      "learning_rate": 4.298832462184527e-07,
      "loss": 0.2427,
      "reward": 0.5357143133878708,
      "reward_std": 0.1665214877575636,
      "rewards/accuracy_reward": 0.11607143469154835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4196428805589676,
      "step": 1071
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1529.8661193847656,
      "epoch": 0.32021506982301545,
      "grad_norm": 85.4726791381836,
      "kl": 2.80859375,
      "learning_rate": 4.297020639922829e-07,
      "loss": 0.288,
      "reward": 0.4492187798023224,
      "reward_std": 0.1789465695619583,
      "rewards/accuracy_reward": 0.05357143026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3956473469734192,
      "step": 1072
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1594.0201416015625,
      "epoch": 0.3205137779105369,
      "grad_norm": 103.33338165283203,
      "kl": 2.62109375,
      "learning_rate": 4.295206862678802e-07,
      "loss": 0.2648,
      "reward": 0.4771205484867096,
      "reward_std": 0.17957113310694695,
      "rewards/accuracy_reward": 0.08482143119908869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.392299123108387,
      "step": 1073
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1518.8371276855469,
      "epoch": 0.3208124859980584,
      "grad_norm": 81.19915008544922,
      "kl": 2.78515625,
      "learning_rate": 4.293391132425657e-07,
      "loss": 0.2653,
      "reward": 0.470982164144516,
      "reward_std": 0.19119353592395782,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3995535969734192,
      "step": 1074
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1513.6897888183594,
      "epoch": 0.32111119408557987,
      "grad_norm": 75.15235900878906,
      "kl": 3.078125,
      "learning_rate": 4.291573451138731e-07,
      "loss": 0.2963,
      "reward": 0.506138414144516,
      "reward_std": 0.15147174708545208,
      "rewards/accuracy_reward": 0.08482143399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4213169813156128,
      "step": 1075
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1552.9599304199219,
      "epoch": 0.32140990217310134,
      "grad_norm": 60.64521026611328,
      "kl": 4.14453125,
      "learning_rate": 4.289753820795484e-07,
      "loss": 0.3601,
      "reward": 0.483258955180645,
      "reward_std": 0.15666737407445908,
      "rewards/accuracy_reward": 0.08035714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4029018059372902,
      "step": 1076
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1544.1942749023438,
      "epoch": 0.3217086102606228,
      "grad_norm": 29.060739517211914,
      "kl": 4.23828125,
      "learning_rate": 4.287932243375494e-07,
      "loss": 0.2942,
      "reward": 0.4436384066939354,
      "reward_std": 0.17921943590044975,
      "rewards/accuracy_reward": 0.035714288242161274,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.407924123108387,
      "step": 1077
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1404.9598999023438,
      "epoch": 0.3220073183481443,
      "grad_norm": 56.80302047729492,
      "kl": 3.1171875,
      "learning_rate": 4.28610872086046e-07,
      "loss": 0.3162,
      "reward": 0.5753348469734192,
      "reward_std": 0.2275884449481964,
      "rewards/accuracy_reward": 0.14285714970901608,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776977300644,
      "step": 1078
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1480.1183776855469,
      "epoch": 0.32230602643566575,
      "grad_norm": 27.2441349029541,
      "kl": 3.95703125,
      "learning_rate": 4.284283255234195e-07,
      "loss": 0.3146,
      "reward": 0.5625000298023224,
      "reward_std": 0.21537933684885502,
      "rewards/accuracy_reward": 0.14732143399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4151785895228386,
      "step": 1079
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1493.6853332519531,
      "epoch": 0.3226047345231872,
      "grad_norm": 15.151577949523926,
      "kl": 4.671875,
      "learning_rate": 4.282455848482627e-07,
      "loss": 0.3867,
      "reward": 0.5351562649011612,
      "reward_std": 0.17754266411066055,
      "rewards/accuracy_reward": 0.12500000465661287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4101562723517418,
      "step": 1080
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1497.3348999023438,
      "epoch": 0.3229034426107087,
      "grad_norm": 11.79818344116211,
      "kl": 4.625,
      "learning_rate": 4.280626502593795e-07,
      "loss": 0.38,
      "reward": 0.4581473469734192,
      "reward_std": 0.16819462552666664,
      "rewards/accuracy_reward": 0.058035717345774174,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4001116305589676,
      "step": 1081
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1486.8259582519531,
      "epoch": 0.32320215069823016,
      "grad_norm": 24.63823890686035,
      "kl": 5.1796875,
      "learning_rate": 4.2787952195578483e-07,
      "loss": 0.4114,
      "reward": 0.5937500223517418,
      "reward_std": 0.21099060028791428,
      "rewards/accuracy_reward": 0.18080358020961285,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4129464477300644,
      "step": 1082
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1541.3192443847656,
      "epoch": 0.32350085878575163,
      "grad_norm": 10.474635124206543,
      "kl": 4.8359375,
      "learning_rate": 4.2769620013670437e-07,
      "loss": 0.389,
      "reward": 0.5401786044239998,
      "reward_std": 0.16725080832839012,
      "rewards/accuracy_reward": 0.13392857578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500223517418,
      "step": 1083
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1537.6340026855469,
      "epoch": 0.3237995668732731,
      "grad_norm": 25.016260147094727,
      "kl": 4.49609375,
      "learning_rate": 4.275126850015742e-07,
      "loss": 0.328,
      "reward": 0.510602705180645,
      "reward_std": 0.19061278365552425,
      "rewards/accuracy_reward": 0.10044643376022577,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4101562723517418,
      "step": 1084
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1492.9107666015625,
      "epoch": 0.3240982749607946,
      "grad_norm": 11.41002082824707,
      "kl": 4.15234375,
      "learning_rate": 4.273289767500409e-07,
      "loss": 0.3133,
      "reward": 0.5106026977300644,
      "reward_std": 0.19239627942442894,
      "rewards/accuracy_reward": 0.08258928917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428013414144516,
      "step": 1085
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1530.9085388183594,
      "epoch": 0.32439698304831605,
      "grad_norm": 20.341691970825195,
      "kl": 4.25390625,
      "learning_rate": 4.27145075581961e-07,
      "loss": 0.3675,
      "reward": 0.4799107387661934,
      "reward_std": 0.20326539874076843,
      "rewards/accuracy_reward": 0.06919643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4107143059372902,
      "step": 1086
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1578.0491638183594,
      "epoch": 0.3246956911358375,
      "grad_norm": 110.20577239990234,
      "kl": 6.8203125,
      "learning_rate": 4.2696098169740094e-07,
      "loss": 0.434,
      "reward": 0.474888414144516,
      "reward_std": 0.19509606808423996,
      "rewards/accuracy_reward": 0.08482143585570157,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3900669738650322,
      "step": 1087
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1482.8014221191406,
      "epoch": 0.324994399223359,
      "grad_norm": 58.631919860839844,
      "kl": 3.69921875,
      "learning_rate": 4.2677669529663686e-07,
      "loss": 0.3422,
      "reward": 0.538504496216774,
      "reward_std": 0.20462054014205933,
      "rewards/accuracy_reward": 0.11160714668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4268973469734192,
      "step": 1088
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1557.1161193847656,
      "epoch": 0.32529310731088046,
      "grad_norm": 25.2952938079834,
      "kl": 4.94140625,
      "learning_rate": 4.2659221658015426e-07,
      "loss": 0.3885,
      "reward": 0.5463170036673546,
      "reward_std": 0.23365453258156776,
      "rewards/accuracy_reward": 0.15178571827709675,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.3922991305589676,
      "step": 1089
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1581.4085388183594,
      "epoch": 0.32559181539840193,
      "grad_norm": 65.10674285888672,
      "kl": 5.890625,
      "learning_rate": 4.264075457486479e-07,
      "loss": 0.3865,
      "reward": 0.4330357313156128,
      "reward_std": 0.18515907786786556,
      "rewards/accuracy_reward": 0.03125000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4017857313156128,
      "step": 1090
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1451.9531555175781,
      "epoch": 0.3258905234859234,
      "grad_norm": 25.68756103515625,
      "kl": 4.87109375,
      "learning_rate": 4.262226830030216e-07,
      "loss": 0.3932,
      "reward": 0.497767873108387,
      "reward_std": 0.18824592605233192,
      "rewards/accuracy_reward": 0.08035714458674192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4174107313156128,
      "step": 1091
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1511.1897888183594,
      "epoch": 0.32618923157344487,
      "grad_norm": 49.686153411865234,
      "kl": 5.42578125,
      "learning_rate": 4.2603762854438786e-07,
      "loss": 0.4052,
      "reward": 0.466517873108387,
      "reward_std": 0.18446096777915955,
      "rewards/accuracy_reward": 0.06026785937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500223517418,
      "step": 1092
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1557.26123046875,
      "epoch": 0.32648793966096634,
      "grad_norm": 43.23857498168945,
      "kl": 4.65234375,
      "learning_rate": 4.258523825740678e-07,
      "loss": 0.3074,
      "reward": 0.4218750149011612,
      "reward_std": 0.1619756855070591,
      "rewards/accuracy_reward": 0.020089287078008056,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4017857313156128,
      "step": 1093
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1458.2455749511719,
      "epoch": 0.3267866477484878,
      "grad_norm": 43.935081481933594,
      "kl": 4.125,
      "learning_rate": 4.2566694529359095e-07,
      "loss": 0.3686,
      "reward": 0.5825893208384514,
      "reward_std": 0.13865990564227104,
      "rewards/accuracy_reward": 0.1629464365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4196428880095482,
      "step": 1094
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1507.2188110351562,
      "epoch": 0.3270853558360093,
      "grad_norm": 35.38380813598633,
      "kl": 4.3125,
      "learning_rate": 4.254813169046947e-07,
      "loss": 0.3804,
      "reward": 0.5066964402794838,
      "reward_std": 0.17994016036391258,
      "rewards/accuracy_reward": 0.1049107201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4017857313156128,
      "step": 1095
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1453.6116638183594,
      "epoch": 0.32738406392353075,
      "grad_norm": 55.5343017578125,
      "kl": 2.96875,
      "learning_rate": 4.2529549760932474e-07,
      "loss": 0.3138,
      "reward": 0.6054687649011612,
      "reward_std": 0.19688594713807106,
      "rewards/accuracy_reward": 0.1763392947614193,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294887661934,
      "step": 1096
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1479.1005554199219,
      "epoch": 0.3276827720110522,
      "grad_norm": 56.875335693359375,
      "kl": 3.3515625,
      "learning_rate": 4.2510948760963425e-07,
      "loss": 0.3147,
      "reward": 0.4776785969734192,
      "reward_std": 0.20819284953176975,
      "rewards/accuracy_reward": 0.06026786006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4174107313156128,
      "step": 1097
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1459.97998046875,
      "epoch": 0.32798148009857364,
      "grad_norm": 36.21147918701172,
      "kl": 3.5390625,
      "learning_rate": 4.249232871079837e-07,
      "loss": 0.303,
      "reward": 0.5697544813156128,
      "reward_std": 0.18278021924197674,
      "rewards/accuracy_reward": 0.15401786006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.415736623108387,
      "step": 1098
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1549.1786193847656,
      "epoch": 0.3282801881860951,
      "grad_norm": 30.499515533447266,
      "kl": 4.00390625,
      "learning_rate": 4.2473689630694123e-07,
      "loss": 0.3306,
      "reward": 0.4972098469734192,
      "reward_std": 0.19302776083350182,
      "rewards/accuracy_reward": 0.08035714738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4168526977300644,
      "step": 1099
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1534.2456359863281,
      "epoch": 0.3285788962736166,
      "grad_norm": 11.760320663452148,
      "kl": 4.4140625,
      "learning_rate": 4.245503154092815e-07,
      "loss": 0.348,
      "reward": 0.5000000149011612,
      "reward_std": 0.19340495020151138,
      "rewards/accuracy_reward": 0.08705357694998384,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4129464477300644,
      "step": 1100
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1579.08935546875,
      "epoch": 0.32887760436113805,
      "grad_norm": 26.37672233581543,
      "kl": 5.640625,
      "learning_rate": 4.243635446179864e-07,
      "loss": 0.4297,
      "reward": 0.5078125298023224,
      "reward_std": 0.17703606933355331,
      "rewards/accuracy_reward": 0.1026785783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4051339477300644,
      "step": 1101
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1499.8103332519531,
      "epoch": 0.3291763124486595,
      "grad_norm": 28.15887451171875,
      "kl": 5.05859375,
      "learning_rate": 4.241765841362443e-07,
      "loss": 0.3777,
      "reward": 0.4782366305589676,
      "reward_std": 0.1633016001433134,
      "rewards/accuracy_reward": 0.06026786006987095,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.4157366305589676,
      "step": 1102
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1543.3371276855469,
      "epoch": 0.329475020536181,
      "grad_norm": 47.15367126464844,
      "kl": 5.3359375,
      "learning_rate": 4.239894341674497e-07,
      "loss": 0.3869,
      "reward": 0.4726562798023224,
      "reward_std": 0.17321716621518135,
      "rewards/accuracy_reward": 0.06696429033763707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4056919813156128,
      "step": 1103
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1500.0000610351562,
      "epoch": 0.32977372862370247,
      "grad_norm": 41.22899627685547,
      "kl": 3.56640625,
      "learning_rate": 4.238020949152036e-07,
      "loss": 0.3066,
      "reward": 0.4871651902794838,
      "reward_std": 0.16500382870435715,
      "rewards/accuracy_reward": 0.0736607147846371,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4135044813156128,
      "step": 1104
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1533.9263916015625,
      "epoch": 0.33007243671122394,
      "grad_norm": 38.69378662109375,
      "kl": 4.35546875,
      "learning_rate": 4.2361456658331264e-07,
      "loss": 0.3985,
      "reward": 0.5396205633878708,
      "reward_std": 0.1611441820859909,
      "rewards/accuracy_reward": 0.13169643469154835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4079241305589676,
      "step": 1105
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1478.3281860351562,
      "epoch": 0.3303711447987454,
      "grad_norm": 53.9987907409668,
      "kl": 5.9375,
      "learning_rate": 4.234268493757893e-07,
      "loss": 0.4773,
      "reward": 0.5251116380095482,
      "reward_std": 0.15964259207248688,
      "rewards/accuracy_reward": 0.12500000977888703,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4001116305589676,
      "step": 1106
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1437.2969360351562,
      "epoch": 0.3306698528862669,
      "grad_norm": 19.226041793823242,
      "kl": 5.1484375,
      "learning_rate": 4.2323894349685153e-07,
      "loss": 0.458,
      "reward": 0.5731026977300644,
      "reward_std": 0.1622866876423359,
      "rewards/accuracy_reward": 0.1584821492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205559372902,
      "step": 1107
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1406.2299499511719,
      "epoch": 0.33096856097378835,
      "grad_norm": 50.84612274169922,
      "kl": 5.609375,
      "learning_rate": 4.2305084915092245e-07,
      "loss": 0.4667,
      "reward": 0.545200914144516,
      "reward_std": 0.18520469777286053,
      "rewards/accuracy_reward": 0.1272321492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4179687649011612,
      "step": 1108
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1539.8728637695312,
      "epoch": 0.3312672690613098,
      "grad_norm": 63.42246627807617,
      "kl": 5.91796875,
      "learning_rate": 4.2286256654263033e-07,
      "loss": 0.4136,
      "reward": 0.5758928805589676,
      "reward_std": 0.15752670541405678,
      "rewards/accuracy_reward": 0.1651785783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4107142984867096,
      "step": 1109
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1523.2589721679688,
      "epoch": 0.3315659771488313,
      "grad_norm": 28.401241302490234,
      "kl": 5.0859375,
      "learning_rate": 4.226740958768081e-07,
      "loss": 0.3984,
      "reward": 0.472098246216774,
      "reward_std": 0.21684274449944496,
      "rewards/accuracy_reward": 0.06696429033763707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4051339402794838,
      "step": 1110
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1555.7077026367188,
      "epoch": 0.33186468523635276,
      "grad_norm": 6.435114860534668,
      "kl": 4.953125,
      "learning_rate": 4.224854373584934e-07,
      "loss": 0.3701,
      "reward": 0.5385044887661934,
      "reward_std": 0.17384998872876167,
      "rewards/accuracy_reward": 0.13169643399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4068080559372902,
      "step": 1111
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1518.6920471191406,
      "epoch": 0.33216339332387423,
      "grad_norm": 24.98603057861328,
      "kl": 4.96875,
      "learning_rate": 4.222965911929283e-07,
      "loss": 0.3725,
      "reward": 0.569754496216774,
      "reward_std": 0.23836610466241837,
      "rewards/accuracy_reward": 0.1718750074505806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3978794813156128,
      "step": 1112
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1557.9129943847656,
      "epoch": 0.3324621014113957,
      "grad_norm": 27.3829402923584,
      "kl": 5.4921875,
      "learning_rate": 4.2210755758555895e-07,
      "loss": 0.4142,
      "reward": 0.455357164144516,
      "reward_std": 0.18140070885419846,
      "rewards/accuracy_reward": 0.058035717345774174,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3973214477300644,
      "step": 1113
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1478.5938110351562,
      "epoch": 0.3327608094989172,
      "grad_norm": 76.59188079833984,
      "kl": 3.6328125,
      "learning_rate": 4.219183367420351e-07,
      "loss": 0.3486,
      "reward": 0.4296875149011612,
      "reward_std": 0.1655359622091055,
      "rewards/accuracy_reward": 0.0223214291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4073660895228386,
      "step": 1114
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1502.5804443359375,
      "epoch": 0.33305951758643865,
      "grad_norm": 40.41933059692383,
      "kl": 3.53515625,
      "learning_rate": 4.2172892886821087e-07,
      "loss": 0.3171,
      "reward": 0.5390625223517418,
      "reward_std": 0.2093126941472292,
      "rewards/accuracy_reward": 0.1272321492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4118303805589676,
      "step": 1115
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1447.3014221191406,
      "epoch": 0.3333582256739601,
      "grad_norm": 66.87886047363281,
      "kl": 3.01171875,
      "learning_rate": 4.215393341701432e-07,
      "loss": 0.2825,
      "reward": 0.6065848469734192,
      "reward_std": 0.19747721403837204,
      "rewards/accuracy_reward": 0.1852678656578064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4213169738650322,
      "step": 1116
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1484.9085388183594,
      "epoch": 0.3336569337614816,
      "grad_norm": 48.29243850708008,
      "kl": 3.4921875,
      "learning_rate": 4.2134955285409266e-07,
      "loss": 0.3337,
      "reward": 0.435267873108387,
      "reward_std": 0.16089589893817902,
      "rewards/accuracy_reward": 0.022321430267766118,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4129464477300644,
      "step": 1117
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1443.3371276855469,
      "epoch": 0.33395564184900306,
      "grad_norm": 33.04225158691406,
      "kl": 3.8203125,
      "learning_rate": 4.211595851265226e-07,
      "loss": 0.3577,
      "reward": 0.5089286044239998,
      "reward_std": 0.17562073655426502,
      "rewards/accuracy_reward": 0.10267857648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500223517418,
      "step": 1118
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1464.3550109863281,
      "epoch": 0.33425434993652453,
      "grad_norm": 55.92015838623047,
      "kl": 3.90234375,
      "learning_rate": 4.2096943119409944e-07,
      "loss": 0.354,
      "reward": 0.470982164144516,
      "reward_std": 0.15139034762978554,
      "rewards/accuracy_reward": 0.05133928940631449,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.419642873108387,
      "step": 1119
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1537.2098693847656,
      "epoch": 0.334553058024046,
      "grad_norm": 55.06420135498047,
      "kl": 6.5,
      "learning_rate": 4.207790912636919e-07,
      "loss": 0.4892,
      "reward": 0.4419643059372902,
      "reward_std": 0.16076616197824478,
      "rewards/accuracy_reward": 0.044642859138548374,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3973214477300644,
      "step": 1120
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1429.3639221191406,
      "epoch": 0.3348517661115675,
      "grad_norm": 12.442398071289062,
      "kl": 4.58984375,
      "learning_rate": 4.205885655423712e-07,
      "loss": 0.3962,
      "reward": 0.4765625149011612,
      "reward_std": 0.154171384871006,
      "rewards/accuracy_reward": 0.0513392873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232313156128,
      "step": 1121
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1464.7656555175781,
      "epoch": 0.33515047419908894,
      "grad_norm": 27.795188903808594,
      "kl": 4.36328125,
      "learning_rate": 4.203978542374107e-07,
      "loss": 0.3934,
      "reward": 0.4771205633878708,
      "reward_std": 0.15596584975719452,
      "rewards/accuracy_reward": 0.05357143213041127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4235491305589676,
      "step": 1122
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1507.69873046875,
      "epoch": 0.3354491822866104,
      "grad_norm": 20.383512496948242,
      "kl": 5.375,
      "learning_rate": 4.2020695755628555e-07,
      "loss": 0.4356,
      "reward": 0.4587053805589676,
      "reward_std": 0.17846173979341984,
      "rewards/accuracy_reward": 0.05580357415601611,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4029018059372902,
      "step": 1123
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1483.3259582519531,
      "epoch": 0.3357478903741319,
      "grad_norm": 15.075225830078125,
      "kl": 5.125,
      "learning_rate": 4.200158757066727e-07,
      "loss": 0.4314,
      "reward": 0.5027901977300644,
      "reward_std": 0.15048566833138466,
      "rewards/accuracy_reward": 0.09598214481957257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4068080484867096,
      "step": 1124
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1404.8460388183594,
      "epoch": 0.33604659846165336,
      "grad_norm": 36.89778137207031,
      "kl": 3.1484375,
      "learning_rate": 4.198246088964504e-07,
      "loss": 0.2715,
      "reward": 0.5167410969734192,
      "reward_std": 0.16084223426878452,
      "rewards/accuracy_reward": 0.08482143026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.431919664144516,
      "step": 1125
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1464.2679138183594,
      "epoch": 0.3363453065491748,
      "grad_norm": 50.02904510498047,
      "kl": 3.58984375,
      "learning_rate": 4.196331573336982e-07,
      "loss": 0.3321,
      "reward": 0.5061384066939354,
      "reward_std": 0.18709429539740086,
      "rewards/accuracy_reward": 0.0870535729918629,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4190848395228386,
      "step": 1126
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1525.8572082519531,
      "epoch": 0.3366440146366963,
      "grad_norm": 28.44256019592285,
      "kl": 5.03125,
      "learning_rate": 4.194415212266967e-07,
      "loss": 0.4236,
      "reward": 0.4190848469734192,
      "reward_std": 0.17601820826530457,
      "rewards/accuracy_reward": 0.020089286845177412,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3989955559372902,
      "step": 1127
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1533.6295471191406,
      "epoch": 0.33694272272421777,
      "grad_norm": 29.089014053344727,
      "kl": 5.36328125,
      "learning_rate": 4.1924970078392706e-07,
      "loss": 0.4133,
      "reward": 0.4854911044239998,
      "reward_std": 0.18045857921242714,
      "rewards/accuracy_reward": 0.07812500302679837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4073660969734192,
      "step": 1128
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1517.16748046875,
      "epoch": 0.33724143081173924,
      "grad_norm": 35.516754150390625,
      "kl": 5.1484375,
      "learning_rate": 4.190576962140714e-07,
      "loss": 0.3678,
      "reward": 0.4799107387661934,
      "reward_std": 0.15518346428871155,
      "rewards/accuracy_reward": 0.06919643399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4107143059372902,
      "step": 1129
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1480.3572082519531,
      "epoch": 0.3375401388992607,
      "grad_norm": 18.417436599731445,
      "kl": 4.83203125,
      "learning_rate": 4.188655077260115e-07,
      "loss": 0.3841,
      "reward": 0.4877232313156128,
      "reward_std": 0.17442062497138977,
      "rewards/accuracy_reward": 0.06919643376022577,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268059372902,
      "step": 1130
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1562.82373046875,
      "epoch": 0.3378388469867822,
      "grad_norm": 25.589128494262695,
      "kl": 5.7734375,
      "learning_rate": 4.186731355288299e-07,
      "loss": 0.4549,
      "reward": 0.4609375223517418,
      "reward_std": 0.21183444187045097,
      "rewards/accuracy_reward": 0.0691964328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3917410895228386,
      "step": 1131
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1411.1473999023438,
      "epoch": 0.33813755507430365,
      "grad_norm": 44.12651062011719,
      "kl": 3.7109375,
      "learning_rate": 4.184805798318086e-07,
      "loss": 0.3563,
      "reward": 0.450892873108387,
      "reward_std": 0.1629181131720543,
      "rewards/accuracy_reward": 0.022321430034935474,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428571455180645,
      "step": 1132
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1453.66748046875,
      "epoch": 0.3384362631618251,
      "grad_norm": 51.030975341796875,
      "kl": 4.0078125,
      "learning_rate": 4.1828784084442924e-07,
      "loss": 0.3604,
      "reward": 0.5641741305589676,
      "reward_std": 0.1582561917603016,
      "rewards/accuracy_reward": 0.1428571455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4213169887661934,
      "step": 1133
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1491.3281860351562,
      "epoch": 0.3387349712493466,
      "grad_norm": 29.74655532836914,
      "kl": 4.03125,
      "learning_rate": 4.180949187763731e-07,
      "loss": 0.3415,
      "reward": 0.431919664144516,
      "reward_std": 0.18150590732693672,
      "rewards/accuracy_reward": 0.02455357206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4073660969734192,
      "step": 1134
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1490.7009582519531,
      "epoch": 0.33903367933686807,
      "grad_norm": 42.432926177978516,
      "kl": 4.43359375,
      "learning_rate": 4.179018138375203e-07,
      "loss": 0.3659,
      "reward": 0.4972098395228386,
      "reward_std": 0.18259546533226967,
      "rewards/accuracy_reward": 0.08258928963914514,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205633878708,
      "step": 1135
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1412.4576416015625,
      "epoch": 0.33933238742438954,
      "grad_norm": 38.338382720947266,
      "kl": 3.7578125,
      "learning_rate": 4.1770852623795035e-07,
      "loss": 0.3601,
      "reward": 0.4704241305589676,
      "reward_std": 0.17673451639711857,
      "rewards/accuracy_reward": 0.05133928917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4190848395228386,
      "step": 1136
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1443.5335388183594,
      "epoch": 0.339631095511911,
      "grad_norm": 40.047447204589844,
      "kl": 3.453125,
      "learning_rate": 4.175150561879409e-07,
      "loss": 0.3142,
      "reward": 0.4776785895228386,
      "reward_std": 0.14084906689822674,
      "rewards/accuracy_reward": 0.04687500209547579,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035895228386,
      "step": 1137
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1521.6295166015625,
      "epoch": 0.3399298035994325,
      "grad_norm": 22.019664764404297,
      "kl": 5.01171875,
      "learning_rate": 4.173214038979687e-07,
      "loss": 0.4024,
      "reward": 0.4347098395228386,
      "reward_std": 0.16105489805340767,
      "rewards/accuracy_reward": 0.01562500069849193,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4190848395228386,
      "step": 1138
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1450.7032165527344,
      "epoch": 0.34022851168695395,
      "grad_norm": 22.877798080444336,
      "kl": 4.78125,
      "learning_rate": 4.171275695787082e-07,
      "loss": 0.4292,
      "reward": 0.4966518059372902,
      "reward_std": 0.16517476364970207,
      "rewards/accuracy_reward": 0.07366071757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 1139
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1540.3728332519531,
      "epoch": 0.3405272197744754,
      "grad_norm": 65.56777954101562,
      "kl": 5.859375,
      "learning_rate": 4.169335534410323e-07,
      "loss": 0.4206,
      "reward": 0.4877232313156128,
      "reward_std": 0.18596554920077324,
      "rewards/accuracy_reward": 0.07812500093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4095982387661934,
      "step": 1140
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1456.4286193847656,
      "epoch": 0.3408259278619969,
      "grad_norm": 76.34263610839844,
      "kl": 5.91796875,
      "learning_rate": 4.167393556960115e-07,
      "loss": 0.4556,
      "reward": 0.5585937947034836,
      "reward_std": 0.21691688522696495,
      "rewards/accuracy_reward": 0.13839286379516125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.420200914144516,
      "step": 1141
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1431.2723999023438,
      "epoch": 0.3411246359495183,
      "grad_norm": 14.116023063659668,
      "kl": 4.3125,
      "learning_rate": 4.1654497655491394e-07,
      "loss": 0.3793,
      "reward": 0.4425223395228386,
      "reward_std": 0.13572319224476814,
      "rewards/accuracy_reward": 0.008928572060540318,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937723517418,
      "step": 1142
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1477.3147888183594,
      "epoch": 0.3414233440370398,
      "grad_norm": 52.86299514770508,
      "kl": 5.0078125,
      "learning_rate": 4.16350416229205e-07,
      "loss": 0.3734,
      "reward": 0.4497768059372902,
      "reward_std": 0.1685454361140728,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268133878708,
      "step": 1143
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1418.93310546875,
      "epoch": 0.34172205212456125,
      "grad_norm": 23.434572219848633,
      "kl": 4.0703125,
      "learning_rate": 4.161556749305474e-07,
      "loss": 0.3248,
      "reward": 0.489955373108387,
      "reward_std": 0.18199655413627625,
      "rewards/accuracy_reward": 0.053571431431919336,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4363839402794838,
      "step": 1144
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1552.60498046875,
      "epoch": 0.3420207602120827,
      "grad_norm": 13.298458099365234,
      "kl": 4.72265625,
      "learning_rate": 4.159607528708003e-07,
      "loss": 0.3844,
      "reward": 0.5636161044239998,
      "reward_std": 0.2067689411342144,
      "rewards/accuracy_reward": 0.15178572130389512,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.411830373108387,
      "step": 1145
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1583.9442443847656,
      "epoch": 0.3423194682996042,
      "grad_norm": 11.088444709777832,
      "kl": 3.8515625,
      "learning_rate": 4.1576565026202013e-07,
      "loss": 0.2959,
      "reward": 0.4486607313156128,
      "reward_std": 0.14514323510229588,
      "rewards/accuracy_reward": 0.0468750037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4017857387661934,
      "step": 1146
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1477.7590026855469,
      "epoch": 0.34261817638712566,
      "grad_norm": 68.97903442382812,
      "kl": 3.453125,
      "learning_rate": 4.155703673164592e-07,
      "loss": 0.36,
      "reward": 0.5775669887661934,
      "reward_std": 0.15102426148951054,
      "rewards/accuracy_reward": 0.15178572619333863,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812723517418,
      "step": 1147
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1423.2790832519531,
      "epoch": 0.34291688447464713,
      "grad_norm": 41.52378463745117,
      "kl": 2.966796875,
      "learning_rate": 4.153749042465663e-07,
      "loss": 0.2801,
      "reward": 0.6099330633878708,
      "reward_std": 0.23177003860473633,
      "rewards/accuracy_reward": 0.17410715040750802,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435825914144516,
      "step": 1148
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1475.1786499023438,
      "epoch": 0.3432155925621686,
      "grad_norm": 26.453310012817383,
      "kl": 4.2265625,
      "learning_rate": 4.1517926126498606e-07,
      "loss": 0.3444,
      "reward": 0.5195312723517418,
      "reward_std": 0.19151940196752548,
      "rewards/accuracy_reward": 0.10267857555299997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4168526977300644,
      "step": 1149
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1494.6072082519531,
      "epoch": 0.3435143006496901,
      "grad_norm": 53.48565673828125,
      "kl": 4.9765625,
      "learning_rate": 4.149834385845589e-07,
      "loss": 0.3549,
      "reward": 0.526785746216774,
      "reward_std": 0.22198162972927094,
      "rewards/accuracy_reward": 0.10937500419095159,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4174107313156128,
      "step": 1150
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1507.8259582519531,
      "epoch": 0.34381300873721155,
      "grad_norm": 20.78907585144043,
      "kl": 4.46484375,
      "learning_rate": 4.147874364183206e-07,
      "loss": 0.3543,
      "reward": 0.4888393133878708,
      "reward_std": 0.15104317292571068,
      "rewards/accuracy_reward": 0.06696428917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750149011612,
      "step": 1151
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1523.01123046875,
      "epoch": 0.344111716824733,
      "grad_norm": 46.41920471191406,
      "kl": 6.01171875,
      "learning_rate": 4.1459125497950245e-07,
      "loss": 0.47,
      "reward": 0.4497768059372902,
      "reward_std": 0.12864741683006287,
      "rewards/accuracy_reward": 0.03794643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.411830373108387,
      "step": 1152
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1510.2879943847656,
      "epoch": 0.3444104249122545,
      "grad_norm": 15.101097106933594,
      "kl": 4.75,
      "learning_rate": 4.1439489448153046e-07,
      "loss": 0.396,
      "reward": 0.5904018133878708,
      "reward_std": 0.16178684681653976,
      "rewards/accuracy_reward": 0.16517857811413705,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232313156128,
      "step": 1153
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1538.6429138183594,
      "epoch": 0.34470913299977596,
      "grad_norm": 46.85988235473633,
      "kl": 5.8984375,
      "learning_rate": 4.141983551380258e-07,
      "loss": 0.4232,
      "reward": 0.5636160969734192,
      "reward_std": 0.22882236912846565,
      "rewards/accuracy_reward": 0.15848214644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4051339477300644,
      "step": 1154
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1399.3973999023438,
      "epoch": 0.34500784108729743,
      "grad_norm": 35.17140197753906,
      "kl": 5.21875,
      "learning_rate": 4.140016371628039e-07,
      "loss": 0.4356,
      "reward": 0.5251116380095482,
      "reward_std": 0.19577893614768982,
      "rewards/accuracy_reward": 0.09598214458674192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.429129496216774,
      "step": 1155
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1500.9063110351562,
      "epoch": 0.3453065491748189,
      "grad_norm": 14.357109069824219,
      "kl": 4.765625,
      "learning_rate": 4.138047407698746e-07,
      "loss": 0.381,
      "reward": 0.5100446566939354,
      "reward_std": 0.13611139729619026,
      "rewards/accuracy_reward": 0.082589291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.427455373108387,
      "step": 1156
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1404.4063110351562,
      "epoch": 0.34560525726234037,
      "grad_norm": 6.704031467437744,
      "kl": 3.9140625,
      "learning_rate": 4.13607666173442e-07,
      "loss": 0.3405,
      "reward": 0.4453125223517418,
      "reward_std": 0.13210083171725273,
      "rewards/accuracy_reward": 0.01785714365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4274553880095482,
      "step": 1157
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1422.1585083007812,
      "epoch": 0.34590396534986184,
      "grad_norm": 21.669687271118164,
      "kl": 4.4140625,
      "learning_rate": 4.1341041358790387e-07,
      "loss": 0.3787,
      "reward": 0.5295759215950966,
      "reward_std": 0.16853434592485428,
      "rewards/accuracy_reward": 0.10714286239817739,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330484867096,
      "step": 1158
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1417.33935546875,
      "epoch": 0.3462026734373833,
      "grad_norm": 57.74095153808594,
      "kl": 3.734375,
      "learning_rate": 4.132129832278517e-07,
      "loss": 0.3731,
      "reward": 0.5083705559372902,
      "reward_std": 0.1640004850924015,
      "rewards/accuracy_reward": 0.07812500046566129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455559372902,
      "step": 1159
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1436.3103332519531,
      "epoch": 0.3465013815249048,
      "grad_norm": 37.90242385864258,
      "kl": 4.08984375,
      "learning_rate": 4.130153753080703e-07,
      "loss": 0.3872,
      "reward": 0.5329241305589676,
      "reward_std": 0.1283276081085205,
      "rewards/accuracy_reward": 0.1093750037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.423549123108387,
      "step": 1160
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1557.3504943847656,
      "epoch": 0.34680008961242625,
      "grad_norm": 44.708621978759766,
      "kl": 3.89453125,
      "learning_rate": 4.1281759004353787e-07,
      "loss": 0.3215,
      "reward": 0.4430803805589676,
      "reward_std": 0.1616559438407421,
      "rewards/accuracy_reward": 0.024553572293370962,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268059372902,
      "step": 1161
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1408.9554138183594,
      "epoch": 0.3470987976999477,
      "grad_norm": 71.8008041381836,
      "kl": 3.390625,
      "learning_rate": 4.1261962764942526e-07,
      "loss": 0.3446,
      "reward": 0.635044664144516,
      "reward_std": 0.15830965526401997,
      "rewards/accuracy_reward": 0.191964291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443080373108387,
      "step": 1162
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1476.4063415527344,
      "epoch": 0.3473975057874692,
      "grad_norm": 34.387786865234375,
      "kl": 3.95703125,
      "learning_rate": 4.1242148834109623e-07,
      "loss": 0.3382,
      "reward": 0.5111607238650322,
      "reward_std": 0.16255823522806168,
      "rewards/accuracy_reward": 0.08705357694998384,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.424107164144516,
      "step": 1163
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1499.9732666015625,
      "epoch": 0.34769621387499067,
      "grad_norm": 15.148104667663574,
      "kl": 5.1171875,
      "learning_rate": 4.1222317233410684e-07,
      "loss": 0.4481,
      "reward": 0.4642857313156128,
      "reward_std": 0.19027116894721985,
      "rewards/accuracy_reward": 0.06696428847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3973214402794838,
      "step": 1164
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1432.9241638183594,
      "epoch": 0.34799492196251214,
      "grad_norm": 15.272122383117676,
      "kl": 4.203125,
      "learning_rate": 4.1202467984420564e-07,
      "loss": 0.321,
      "reward": 0.5066964626312256,
      "reward_std": 0.15943104773759842,
      "rewards/accuracy_reward": 0.08482143376022577,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 1165
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1473.8527526855469,
      "epoch": 0.3482936300500336,
      "grad_norm": 38.41749954223633,
      "kl": 4.4140625,
      "learning_rate": 4.1182601108733284e-07,
      "loss": 0.3305,
      "reward": 0.6010044813156128,
      "reward_std": 0.2197759486734867,
      "rewards/accuracy_reward": 0.1808035783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.420200914144516,
      "step": 1166
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1575.5871276855469,
      "epoch": 0.3485923381375551,
      "grad_norm": 27.68985939025879,
      "kl": 3.5390625,
      "learning_rate": 4.1162716627962064e-07,
      "loss": 0.2855,
      "reward": 0.5033482387661934,
      "reward_std": 0.1654346026480198,
      "rewards/accuracy_reward": 0.07142857508733869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4319196566939354,
      "step": 1167
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1522.13623046875,
      "epoch": 0.34889104622507655,
      "grad_norm": 9.421676635742188,
      "kl": 4.98828125,
      "learning_rate": 4.114281456373927e-07,
      "loss": 0.4147,
      "reward": 0.4363839477300644,
      "reward_std": 0.17174207419157028,
      "rewards/accuracy_reward": 0.026785714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4095982238650322,
      "step": 1168
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1430.0536499023438,
      "epoch": 0.349189754312598,
      "grad_norm": 42.35009002685547,
      "kl": 5.41015625,
      "learning_rate": 4.1122894937716393e-07,
      "loss": 0.4413,
      "reward": 0.4525669813156128,
      "reward_std": 0.1388267558068037,
      "rewards/accuracy_reward": 0.029017859371379018,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4235491305589676,
      "step": 1169
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1458.8728332519531,
      "epoch": 0.3494884624001195,
      "grad_norm": 21.947912216186523,
      "kl": 4.65234375,
      "learning_rate": 4.110295777156404e-07,
      "loss": 0.3893,
      "reward": 0.5401785969734192,
      "reward_std": 0.16548151895403862,
      "rewards/accuracy_reward": 0.1071428619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357238650322,
      "step": 1170
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1509.3795166015625,
      "epoch": 0.34978717048764096,
      "grad_norm": 7.804295063018799,
      "kl": 4.29296875,
      "learning_rate": 4.108300308697189e-07,
      "loss": 0.3397,
      "reward": 0.4899553805589676,
      "reward_std": 0.1958666853606701,
      "rewards/accuracy_reward": 0.07142857648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268133878708,
      "step": 1171
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1479.15185546875,
      "epoch": 0.35008587857516243,
      "grad_norm": 14.75503158569336,
      "kl": 4.0546875,
      "learning_rate": 4.106303090564868e-07,
      "loss": 0.3257,
      "reward": 0.5390625298023224,
      "reward_std": 0.1663689874112606,
      "rewards/accuracy_reward": 0.12276786146685481,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.416294664144516,
      "step": 1172
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1601.1741638183594,
      "epoch": 0.3503845866626839,
      "grad_norm": 28.54844093322754,
      "kl": 4.15234375,
      "learning_rate": 4.1043041249322195e-07,
      "loss": 0.3287,
      "reward": 0.4726562723517418,
      "reward_std": 0.16040240973234177,
      "rewards/accuracy_reward": 0.0691964328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4034598395228386,
      "step": 1173
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1510.2969665527344,
      "epoch": 0.3506832947502054,
      "grad_norm": 27.811861038208008,
      "kl": 3.90625,
      "learning_rate": 4.102303413973922e-07,
      "loss": 0.3353,
      "reward": 0.4799107387661934,
      "reward_std": 0.19103499501943588,
      "rewards/accuracy_reward": 0.06250000419095159,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4174107387661934,
      "step": 1174
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1541.1741943359375,
      "epoch": 0.35098200283772685,
      "grad_norm": 24.54498291015625,
      "kl": 4.48046875,
      "learning_rate": 4.1003009598665526e-07,
      "loss": 0.3259,
      "reward": 0.5937500298023224,
      "reward_std": 0.22726193442940712,
      "rewards/accuracy_reward": 0.1696428656578064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4241071566939354,
      "step": 1175
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1499.8616638183594,
      "epoch": 0.3512807109252483,
      "grad_norm": 50.237117767333984,
      "kl": 4.27734375,
      "learning_rate": 4.098296764788586e-07,
      "loss": 0.3786,
      "reward": 0.4938616380095482,
      "reward_std": 0.15878462046384811,
      "rewards/accuracy_reward": 0.06919643096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4246651977300644,
      "step": 1176
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1474.5714721679688,
      "epoch": 0.3515794190127698,
      "grad_norm": 14.401457786560059,
      "kl": 4.0703125,
      "learning_rate": 4.096290830920388e-07,
      "loss": 0.3511,
      "reward": 0.5189732313156128,
      "reward_std": 0.16647867485880852,
      "rewards/accuracy_reward": 0.08928571734577417,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875223517418,
      "step": 1177
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1411.8013916015625,
      "epoch": 0.35187812710029126,
      "grad_norm": 42.403282165527344,
      "kl": 4.68359375,
      "learning_rate": 4.0942831604442197e-07,
      "loss": 0.3737,
      "reward": 0.5708705633878708,
      "reward_std": 0.15510273538529873,
      "rewards/accuracy_reward": 0.14508929220028222,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812723517418,
      "step": 1178
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1529.0067443847656,
      "epoch": 0.35217683518781273,
      "grad_norm": 21.990720748901367,
      "kl": 4.5859375,
      "learning_rate": 4.0922737555442295e-07,
      "loss": 0.3652,
      "reward": 0.463169664144516,
      "reward_std": 0.14694657735526562,
      "rewards/accuracy_reward": 0.0491071455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625149011612,
      "step": 1179
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1501.8929138183594,
      "epoch": 0.3524755432753342,
      "grad_norm": 31.601350784301758,
      "kl": 4.27734375,
      "learning_rate": 4.0902626184064525e-07,
      "loss": 0.3647,
      "reward": 0.5920759215950966,
      "reward_std": 0.16835422441363335,
      "rewards/accuracy_reward": 0.1741071529686451,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4179687649011612,
      "step": 1180
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1522.8192443847656,
      "epoch": 0.3527742513628557,
      "grad_norm": 88.24003601074219,
      "kl": 5.9375,
      "learning_rate": 4.0882497512188097e-07,
      "loss": 0.4166,
      "reward": 0.5228794887661934,
      "reward_std": 0.2252089325338602,
      "rewards/accuracy_reward": 0.1026785783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.420200914144516,
      "step": 1181
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1479.9531555175781,
      "epoch": 0.35307295945037714,
      "grad_norm": 25.299148559570312,
      "kl": 4.296875,
      "learning_rate": 4.0862351561711026e-07,
      "loss": 0.3294,
      "reward": 0.5859375447034836,
      "reward_std": 0.14981507323682308,
      "rewards/accuracy_reward": 0.1517857201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341518059372902,
      "step": 1182
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1468.3795471191406,
      "epoch": 0.3533716675378986,
      "grad_norm": 31.78482437133789,
      "kl": 4.8203125,
      "learning_rate": 4.0842188354550124e-07,
      "loss": 0.3514,
      "reward": 0.4620535895228386,
      "reward_std": 0.14866582490503788,
      "rewards/accuracy_reward": 0.03794643119908869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4241071566939354,
      "step": 1183
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1450.7813415527344,
      "epoch": 0.3536703756254201,
      "grad_norm": 24.50768280029297,
      "kl": 4.8203125,
      "learning_rate": 4.0822007912641004e-07,
      "loss": 0.3612,
      "reward": 0.5066964477300644,
      "reward_std": 0.1542123407125473,
      "rewards/accuracy_reward": 0.08482143003493547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 1184
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1614.9554138183594,
      "epoch": 0.3539690837129415,
      "grad_norm": 72.21038055419922,
      "kl": 6.33984375,
      "learning_rate": 4.0801810257938e-07,
      "loss": 0.4325,
      "reward": 0.4095982387661934,
      "reward_std": 0.1483864150941372,
      "rewards/accuracy_reward": 0.013392857974395156,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.396205373108387,
      "step": 1185
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1570.0514221191406,
      "epoch": 0.35426779180046297,
      "grad_norm": 18.263654708862305,
      "kl": 3.7578125,
      "learning_rate": 4.0781595412414173e-07,
      "loss": 0.2738,
      "reward": 0.4330357387661934,
      "reward_std": 0.14903313852846622,
      "rewards/accuracy_reward": 0.013392857974395156,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4196428805589676,
      "step": 1186
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1558.1451721191406,
      "epoch": 0.35456649988798444,
      "grad_norm": 39.52939987182617,
      "kl": 2.59765625,
      "learning_rate": 4.07613633980613e-07,
      "loss": 0.2128,
      "reward": 0.4810268133878708,
      "reward_std": 0.18253256753087044,
      "rewards/accuracy_reward": 0.04464285937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.436383955180645,
      "step": 1187
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1450.3884887695312,
      "epoch": 0.3548652079755059,
      "grad_norm": 66.02269744873047,
      "kl": 3.04296875,
      "learning_rate": 4.074111423688984e-07,
      "loss": 0.3098,
      "reward": 0.4469866305589676,
      "reward_std": 0.14859113842248917,
      "rewards/accuracy_reward": 0.01562500069849193,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4313616305589676,
      "step": 1188
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1482.7813110351562,
      "epoch": 0.3551639160630274,
      "grad_norm": 67.85047912597656,
      "kl": 2.7578125,
      "learning_rate": 4.072084795092887e-07,
      "loss": 0.2597,
      "reward": 0.5647321715950966,
      "reward_std": 0.1490248292684555,
      "rewards/accuracy_reward": 0.1294642905704677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435267873108387,
      "step": 1189
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1441.26123046875,
      "epoch": 0.35546262415054886,
      "grad_norm": 60.97877502441406,
      "kl": 2.87109375,
      "learning_rate": 4.070056456222615e-07,
      "loss": 0.2629,
      "reward": 0.5217634290456772,
      "reward_std": 0.18236533366143703,
      "rewards/accuracy_reward": 0.10044643515720963,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4213169887661934,
      "step": 1190
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1522.1830749511719,
      "epoch": 0.3557613322380703,
      "grad_norm": 40.4552116394043,
      "kl": 3.34375,
      "learning_rate": 4.0680264092848005e-07,
      "loss": 0.2655,
      "reward": 0.5468750223517418,
      "reward_std": 0.15329928323626518,
      "rewards/accuracy_reward": 0.1183035783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 1191
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1563.6496276855469,
      "epoch": 0.3560600403255918,
      "grad_norm": 13.308736801147461,
      "kl": 4.7265625,
      "learning_rate": 4.065994656487935e-07,
      "loss": 0.3668,
      "reward": 0.5329241454601288,
      "reward_std": 0.20560182258486748,
      "rewards/accuracy_reward": 0.1183035783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205559372902,
      "step": 1192
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1519.2701416015625,
      "epoch": 0.35635874841311327,
      "grad_norm": 11.208361625671387,
      "kl": 4.0859375,
      "learning_rate": 4.063961200042368e-07,
      "loss": 0.3282,
      "reward": 0.544084832072258,
      "reward_std": 0.17383621633052826,
      "rewards/accuracy_reward": 0.11607143469154835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4280134066939354,
      "step": 1193
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1451.466552734375,
      "epoch": 0.35665745650063474,
      "grad_norm": 35.108638763427734,
      "kl": 5.3515625,
      "learning_rate": 4.0619260421602997e-07,
      "loss": 0.432,
      "reward": 0.5323660969734192,
      "reward_std": 0.18533695861697197,
      "rewards/accuracy_reward": 0.11383929220028222,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185267984867096,
      "step": 1194
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1482.6630249023438,
      "epoch": 0.3569561645881562,
      "grad_norm": 132.93759155273438,
      "kl": 6.890625,
      "learning_rate": 4.0598891850557823e-07,
      "loss": 0.4819,
      "reward": 0.6065848469734192,
      "reward_std": 0.14587992243468761,
      "rewards/accuracy_reward": 0.19642858300358057,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4101562574505806,
      "step": 1195
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1621.5781860351562,
      "epoch": 0.3572548726756777,
      "grad_norm": 179.8414764404297,
      "kl": 9.0078125,
      "learning_rate": 4.0578506309447183e-07,
      "loss": 0.5792,
      "reward": 0.4570312723517418,
      "reward_std": 0.18259981274604797,
      "rewards/accuracy_reward": 0.06026785844005644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3967634066939354,
      "step": 1196
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1391.9554138183594,
      "epoch": 0.35755358076319915,
      "grad_norm": 89.42304229736328,
      "kl": 6.3203125,
      "learning_rate": 4.055810382044853e-07,
      "loss": 0.482,
      "reward": 0.5239955484867096,
      "reward_std": 0.17241018265485764,
      "rewards/accuracy_reward": 0.10714286426082253,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4168526977300644,
      "step": 1197
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1384.3973693847656,
      "epoch": 0.3578522888507206,
      "grad_norm": 27.747486114501953,
      "kl": 4.84375,
      "learning_rate": 4.053768440575778e-07,
      "loss": 0.4131,
      "reward": 0.6517857387661934,
      "reward_std": 0.20018428191542625,
      "rewards/accuracy_reward": 0.21875001210719347,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357313156128,
      "step": 1198
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1433.4442749023438,
      "epoch": 0.3581509969382421,
      "grad_norm": 29.561710357666016,
      "kl": 4.7734375,
      "learning_rate": 4.051724808758925e-07,
      "loss": 0.3898,
      "reward": 0.4871652126312256,
      "reward_std": 0.149665467441082,
      "rewards/accuracy_reward": 0.05357143236324191,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937649011612,
      "step": 1199
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1465.4844360351562,
      "epoch": 0.35844970502576357,
      "grad_norm": 37.81698989868164,
      "kl": 4.43359375,
      "learning_rate": 4.0496794888175656e-07,
      "loss": 0.3727,
      "reward": 0.5156250223517418,
      "reward_std": 0.1488787792623043,
      "rewards/accuracy_reward": 0.09151785937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.424107164144516,
      "step": 1200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1473.9219360351562,
      "epoch": 0.35874841311328504,
      "grad_norm": 27.308847427368164,
      "kl": 3.91796875,
      "learning_rate": 4.0476324829768074e-07,
      "loss": 0.338,
      "reward": 0.574776828289032,
      "reward_std": 0.17400255054235458,
      "rewards/accuracy_reward": 0.1428571529686451,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.431919664144516,
      "step": 1201
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1443.3951721191406,
      "epoch": 0.3590471212008065,
      "grad_norm": 16.00550079345703,
      "kl": 4.5,
      "learning_rate": 4.0455837934635914e-07,
      "loss": 0.3553,
      "reward": 0.5094866380095482,
      "reward_std": 0.12299532815814018,
      "rewards/accuracy_reward": 0.07812500488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4313616305589676,
      "step": 1202
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1448.2232971191406,
      "epoch": 0.359345829288328,
      "grad_norm": 52.6285400390625,
      "kl": 5.4140625,
      "learning_rate": 4.043533422506692e-07,
      "loss": 0.3888,
      "reward": 0.4380580633878708,
      "reward_std": 0.1409877184778452,
      "rewards/accuracy_reward": 0.017857144121080637,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.420200914144516,
      "step": 1203
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1508.9978332519531,
      "epoch": 0.35964453737584945,
      "grad_norm": 39.36411666870117,
      "kl": 4.0859375,
      "learning_rate": 4.041481372336712e-07,
      "loss": 0.3583,
      "reward": 0.4737723395228386,
      "reward_std": 0.16968788392841816,
      "rewards/accuracy_reward": 0.04910714365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4246651977300644,
      "step": 1204
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1582.37060546875,
      "epoch": 0.3599432454633709,
      "grad_norm": 67.00851440429688,
      "kl": 5.99609375,
      "learning_rate": 4.0394276451860796e-07,
      "loss": 0.4066,
      "reward": 0.4614955633878708,
      "reward_std": 0.19210495799779892,
      "rewards/accuracy_reward": 0.06250000302679837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3989955559372902,
      "step": 1205
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1442.5870971679688,
      "epoch": 0.3602419535508924,
      "grad_norm": 36.35771179199219,
      "kl": 4.171875,
      "learning_rate": 4.0373722432890493e-07,
      "loss": 0.3554,
      "reward": 0.5044643208384514,
      "reward_std": 0.17856038361787796,
      "rewards/accuracy_reward": 0.07142857275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357387661934,
      "step": 1206
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1521.3795471191406,
      "epoch": 0.36054066163841386,
      "grad_norm": 25.4012508392334,
      "kl": 4.70703125,
      "learning_rate": 4.035315168881698e-07,
      "loss": 0.3385,
      "reward": 0.467633955180645,
      "reward_std": 0.18183300644159317,
      "rewards/accuracy_reward": 0.051339286379516125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.416294664144516,
      "step": 1207
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1449.4040832519531,
      "epoch": 0.36083936972593533,
      "grad_norm": 50.16354751586914,
      "kl": 3.171875,
      "learning_rate": 4.0332564242019184e-07,
      "loss": 0.3005,
      "reward": 0.498325914144516,
      "reward_std": 0.16363093256950378,
      "rewards/accuracy_reward": 0.06250000232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435825914144516,
      "step": 1208
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1492.0826721191406,
      "epoch": 0.3611380778134568,
      "grad_norm": 10.722467422485352,
      "kl": 3.78515625,
      "learning_rate": 4.031196011489426e-07,
      "loss": 0.3004,
      "reward": 0.5284598395228386,
      "reward_std": 0.18046438694000244,
      "rewards/accuracy_reward": 0.1026785746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812723517418,
      "step": 1209
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1451.1317443847656,
      "epoch": 0.3614367859009783,
      "grad_norm": 50.4427604675293,
      "kl": 2.6640625,
      "learning_rate": 4.0291339329857453e-07,
      "loss": 0.255,
      "reward": 0.4765625223517418,
      "reward_std": 0.15627948567271233,
      "rewards/accuracy_reward": 0.037946430733427405,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4386160895228386,
      "step": 1210
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1566.4353332519531,
      "epoch": 0.36173549398849975,
      "grad_norm": 13.415938377380371,
      "kl": 4.5859375,
      "learning_rate": 4.027070190934218e-07,
      "loss": 0.3322,
      "reward": 0.4631696566939354,
      "reward_std": 0.14856291934847832,
      "rewards/accuracy_reward": 0.04687500186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4162946566939354,
      "step": 1211
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1508.0804138183594,
      "epoch": 0.3620342020760212,
      "grad_norm": 7.612099647521973,
      "kl": 4.21484375,
      "learning_rate": 4.025004787579991e-07,
      "loss": 0.3088,
      "reward": 0.5351562798023224,
      "reward_std": 0.1676801424473524,
      "rewards/accuracy_reward": 0.1205357201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205559372902,
      "step": 1212
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1501.0469360351562,
      "epoch": 0.3623329101635427,
      "grad_norm": 67.95134735107422,
      "kl": 2.94921875,
      "learning_rate": 4.0229377251700213e-07,
      "loss": 0.2738,
      "reward": 0.5725446790456772,
      "reward_std": 0.19888300448656082,
      "rewards/accuracy_reward": 0.14285714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875149011612,
      "step": 1213
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1409.7366943359375,
      "epoch": 0.36263161825106416,
      "grad_norm": 68.59136199951172,
      "kl": 2.70703125,
      "learning_rate": 4.0208690059530714e-07,
      "loss": 0.2687,
      "reward": 0.505580373108387,
      "reward_std": 0.13024732656776905,
      "rewards/accuracy_reward": 0.0580357164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.447544664144516,
      "step": 1214
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1284.0245971679688,
      "epoch": 0.36293032633858563,
      "grad_norm": 13.629560470581055,
      "kl": 3.06640625,
      "learning_rate": 4.018798632179704e-07,
      "loss": 0.267,
      "reward": 0.5842634215950966,
      "reward_std": 0.18835154175758362,
      "rewards/accuracy_reward": 0.1294642877765,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.454799123108387,
      "step": 1215
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1492.5156860351562,
      "epoch": 0.3632290344261071,
      "grad_norm": 24.961658477783203,
      "kl": 4.046875,
      "learning_rate": 4.016726606102283e-07,
      "loss": 0.3393,
      "reward": 0.4927455559372902,
      "reward_std": 0.17969315871596336,
      "rewards/accuracy_reward": 0.06473214668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428013414144516,
      "step": 1216
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1528.6406860351562,
      "epoch": 0.36352774251362857,
      "grad_norm": 5.946451187133789,
      "kl": 4.3125,
      "learning_rate": 4.01465292997497e-07,
      "loss": 0.3174,
      "reward": 0.4570312649011612,
      "reward_std": 0.1583476047962904,
      "rewards/accuracy_reward": 0.03348214481957257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4235491156578064,
      "step": 1217
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1441.76123046875,
      "epoch": 0.36382645060115004,
      "grad_norm": 23.61194610595703,
      "kl": 4.47265625,
      "learning_rate": 4.0125776060537207e-07,
      "loss": 0.3737,
      "reward": 0.5251116305589676,
      "reward_std": 0.18760451301932335,
      "rewards/accuracy_reward": 0.0937500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4313616305589676,
      "step": 1218
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1532.9241638183594,
      "epoch": 0.3641251586886715,
      "grad_norm": 46.158592224121094,
      "kl": 5.25390625,
      "learning_rate": 4.0105006365962847e-07,
      "loss": 0.3629,
      "reward": 0.4944196715950966,
      "reward_std": 0.18677605129778385,
      "rewards/accuracy_reward": 0.07589286239817739,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268059372902,
      "step": 1219
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1374.5871276855469,
      "epoch": 0.364423866776193,
      "grad_norm": 35.42365646362305,
      "kl": 4.57421875,
      "learning_rate": 4.008422023862201e-07,
      "loss": 0.3679,
      "reward": 0.6004464477300644,
      "reward_std": 0.18019600957632065,
      "rewards/accuracy_reward": 0.16517858020961285,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435267873108387,
      "step": 1220
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1625.3192749023438,
      "epoch": 0.36472257486371445,
      "grad_norm": 145.6487579345703,
      "kl": 7.875,
      "learning_rate": 4.0063417701127964e-07,
      "loss": 0.5064,
      "reward": 0.4414062649011612,
      "reward_std": 0.19841773808002472,
      "rewards/accuracy_reward": 0.03794642956927419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4034598395228386,
      "step": 1221
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1608.1205749511719,
      "epoch": 0.3650212829512359,
      "grad_norm": 127.34754180908203,
      "kl": 6.890625,
      "learning_rate": 4.004259877611184e-07,
      "loss": 0.4276,
      "reward": 0.4854911044239998,
      "reward_std": 0.21125541254878044,
      "rewards/accuracy_reward": 0.07366071850992739,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.4095982238650322,
      "step": 1222
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1460.8326721191406,
      "epoch": 0.3653199910387574,
      "grad_norm": 71.03634643554688,
      "kl": 5.7421875,
      "learning_rate": 4.0021763486222577e-07,
      "loss": 0.4341,
      "reward": 0.513392873108387,
      "reward_std": 0.17122315801680088,
      "rewards/accuracy_reward": 0.09151786309666932,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750149011612,
      "step": 1223
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1556.290283203125,
      "epoch": 0.36561869912627887,
      "grad_norm": 100.50299072265625,
      "kl": 6.1796875,
      "learning_rate": 4.0000911854126927e-07,
      "loss": 0.4169,
      "reward": 0.4960937649011612,
      "reward_std": 0.19401892088353634,
      "rewards/accuracy_reward": 0.10267857648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3934151977300644,
      "step": 1224
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1467.2567749023438,
      "epoch": 0.36591740721380034,
      "grad_norm": 12.638955116271973,
      "kl": 3.9375,
      "learning_rate": 3.998004390250944e-07,
      "loss": 0.3276,
      "reward": 0.4419643059372902,
      "reward_std": 0.12977353297173977,
      "rewards/accuracy_reward": 0.0133928582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 1225
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1380.0067443847656,
      "epoch": 0.3662161153013218,
      "grad_norm": 67.33377838134766,
      "kl": 2.72265625,
      "learning_rate": 3.995915965407237e-07,
      "loss": 0.2842,
      "reward": 0.5078125223517418,
      "reward_std": 0.13513562828302383,
      "rewards/accuracy_reward": 0.0580357164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4497768059372902,
      "step": 1226
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1481.3683471679688,
      "epoch": 0.3665148233888433,
      "grad_norm": 43.77455520629883,
      "kl": 2.84375,
      "learning_rate": 3.9938259131535767e-07,
      "loss": 0.2419,
      "reward": 0.5937500149011612,
      "reward_std": 0.20719616673886776,
      "rewards/accuracy_reward": 0.16071428847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357313156128,
      "step": 1227
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1419.0402526855469,
      "epoch": 0.3668135314763647,
      "grad_norm": 50.266544342041016,
      "kl": 3.62109375,
      "learning_rate": 3.991734235763733e-07,
      "loss": 0.3232,
      "reward": 0.545758955180645,
      "reward_std": 0.13034122064709663,
      "rewards/accuracy_reward": 0.11383928963914514,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4319196715950966,
      "step": 1228
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1449.4375610351562,
      "epoch": 0.36711223956388617,
      "grad_norm": 36.54620361328125,
      "kl": 2.87890625,
      "learning_rate": 3.9896409355132463e-07,
      "loss": 0.2679,
      "reward": 0.5251116380095482,
      "reward_std": 0.22271152585744858,
      "rewards/accuracy_reward": 0.08258928917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4425223469734192,
      "step": 1229
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1471.602783203125,
      "epoch": 0.36741094765140764,
      "grad_norm": 33.91981506347656,
      "kl": 3.78125,
      "learning_rate": 3.9875460146794225e-07,
      "loss": 0.338,
      "reward": 0.5279017984867096,
      "reward_std": 0.19705335423350334,
      "rewards/accuracy_reward": 0.09375000302679837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341518059372902,
      "step": 1230
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1458.9933776855469,
      "epoch": 0.3677096557389291,
      "grad_norm": 12.131470680236816,
      "kl": 4.5078125,
      "learning_rate": 3.9854494755413305e-07,
      "loss": 0.3636,
      "reward": 0.5139509215950966,
      "reward_std": 0.13212929107248783,
      "rewards/accuracy_reward": 0.08258928824216127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4313616305589676,
      "step": 1231
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1521.7812805175781,
      "epoch": 0.3680083638264506,
      "grad_norm": 20.1412353515625,
      "kl": 4.38671875,
      "learning_rate": 3.983351320379798e-07,
      "loss": 0.3303,
      "reward": 0.4827009215950966,
      "reward_std": 0.16790449246764183,
      "rewards/accuracy_reward": 0.06250000419095159,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4202009066939354,
      "step": 1232
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1450.8214721679688,
      "epoch": 0.36830707191397205,
      "grad_norm": 16.944995880126953,
      "kl": 4.376953125,
      "learning_rate": 3.981251551477413e-07,
      "loss": 0.3506,
      "reward": 0.5909598544239998,
      "reward_std": 0.18590334989130497,
      "rewards/accuracy_reward": 0.16294643888249993,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428013414144516,
      "step": 1233
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1468.1808776855469,
      "epoch": 0.3686057800014935,
      "grad_norm": 17.270156860351562,
      "kl": 3.91796875,
      "learning_rate": 3.9791501711185185e-07,
      "loss": 0.2959,
      "reward": 0.5357143133878708,
      "reward_std": 0.18748386204242706,
      "rewards/accuracy_reward": 0.09375000558793545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4419643059372902,
      "step": 1234
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1475.4308776855469,
      "epoch": 0.368904488089015,
      "grad_norm": 18.383817672729492,
      "kl": 5.15234375,
      "learning_rate": 3.9770471815892105e-07,
      "loss": 0.4113,
      "reward": 0.518415205180645,
      "reward_std": 0.1484800148755312,
      "rewards/accuracy_reward": 0.09375000488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4246651977300644,
      "step": 1235
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1603.9264221191406,
      "epoch": 0.36920319617653646,
      "grad_norm": 11.350081443786621,
      "kl": 4.29296875,
      "learning_rate": 3.974942585177334e-07,
      "loss": 0.3153,
      "reward": 0.4654017984867096,
      "reward_std": 0.1712711751461029,
      "rewards/accuracy_reward": 0.042410716880112886,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 1236
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1532.0246276855469,
      "epoch": 0.36950190426405793,
      "grad_norm": 11.328507423400879,
      "kl": 4.71875,
      "learning_rate": 3.9728363841724857e-07,
      "loss": 0.3603,
      "reward": 0.5128348469734192,
      "reward_std": 0.16434019804000854,
      "rewards/accuracy_reward": 0.09151786030270159,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4213169738650322,
      "step": 1237
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1501.7723693847656,
      "epoch": 0.3698006123515794,
      "grad_norm": 14.709986686706543,
      "kl": 3.3046875,
      "learning_rate": 3.970728580866005e-07,
      "loss": 0.2546,
      "reward": 0.5619419664144516,
      "reward_std": 0.15384064614772797,
      "rewards/accuracy_reward": 0.12946429289877415,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.432477705180645,
      "step": 1238
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1449.1964721679688,
      "epoch": 0.3700993204391009,
      "grad_norm": 14.81948471069336,
      "kl": 3.88671875,
      "learning_rate": 3.9686191775509746e-07,
      "loss": 0.3489,
      "reward": 0.5485491380095482,
      "reward_std": 0.19902825728058815,
      "rewards/accuracy_reward": 0.1227678619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812649011612,
      "step": 1239
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1436.9687805175781,
      "epoch": 0.37039802852662235,
      "grad_norm": 14.535292625427246,
      "kl": 2.60546875,
      "learning_rate": 3.9665081765222193e-07,
      "loss": 0.2204,
      "reward": 0.5998884215950966,
      "reward_std": 0.17028621025383472,
      "rewards/accuracy_reward": 0.15178572479635477,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4481026977300644,
      "step": 1240
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1476.9241638183594,
      "epoch": 0.3706967366141438,
      "grad_norm": 16.168439865112305,
      "kl": 4.03515625,
      "learning_rate": 3.9643955800763013e-07,
      "loss": 0.3348,
      "reward": 0.4648437649011612,
      "reward_std": 0.13842986337840557,
      "rewards/accuracy_reward": 0.04687500116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4179687723517418,
      "step": 1241
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1551.8951721191406,
      "epoch": 0.3709954447016653,
      "grad_norm": 114.10850524902344,
      "kl": 6.7109375,
      "learning_rate": 3.962281390511518e-07,
      "loss": 0.4803,
      "reward": 0.4458705559372902,
      "reward_std": 0.16462809219956398,
      "rewards/accuracy_reward": 0.024553573224693537,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4213169887661934,
      "step": 1242
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1610.2232971191406,
      "epoch": 0.37129415278918676,
      "grad_norm": 47.2646598815918,
      "kl": 4.77734375,
      "learning_rate": 3.9601656101279013e-07,
      "loss": 0.3125,
      "reward": 0.4620535895228386,
      "reward_std": 0.16126078367233276,
      "rewards/accuracy_reward": 0.05133928847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4107142984867096,
      "step": 1243
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1541.47998046875,
      "epoch": 0.37159286087670823,
      "grad_norm": 19.098623275756836,
      "kl": 3.8515625,
      "learning_rate": 3.958048241227212e-07,
      "loss": 0.2727,
      "reward": 0.4854910895228386,
      "reward_std": 0.17604700475931168,
      "rewards/accuracy_reward": 0.058035718742758036,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4274553805589676,
      "step": 1244
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1433.0090026855469,
      "epoch": 0.3718915689642297,
      "grad_norm": 91.07913970947266,
      "kl": 4.94921875,
      "learning_rate": 3.9559292861129414e-07,
      "loss": 0.3587,
      "reward": 0.510602705180645,
      "reward_std": 0.1472528986632824,
      "rewards/accuracy_reward": 0.06250000232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.448102705180645,
      "step": 1245
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1494.8773193359375,
      "epoch": 0.3721902770517512,
      "grad_norm": 19.430967330932617,
      "kl": 3.171875,
      "learning_rate": 3.9538087470903024e-07,
      "loss": 0.2301,
      "reward": 0.589285746216774,
      "reward_std": 0.16713102720677853,
      "rewards/accuracy_reward": 0.14285714854486287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4464285895228386,
      "step": 1246
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1608.5090026855469,
      "epoch": 0.37248898513927264,
      "grad_norm": 10.652753829956055,
      "kl": 4.29296875,
      "learning_rate": 3.951686626466236e-07,
      "loss": 0.3012,
      "reward": 0.4938616305589676,
      "reward_std": 0.1974820401519537,
      "rewards/accuracy_reward": 0.06696428940631449,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4268973469734192,
      "step": 1247
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1397.2701721191406,
      "epoch": 0.3727876932267941,
      "grad_norm": 20.05364418029785,
      "kl": 3.93359375,
      "learning_rate": 3.949562926549401e-07,
      "loss": 0.3347,
      "reward": 0.5664062723517418,
      "reward_std": 0.1871238425374031,
      "rewards/accuracy_reward": 0.1205357201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4458705633878708,
      "step": 1248
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1523.6920471191406,
      "epoch": 0.3730864013143156,
      "grad_norm": 17.155492782592773,
      "kl": 4.89453125,
      "learning_rate": 3.947437649650173e-07,
      "loss": 0.3885,
      "reward": 0.5530134215950966,
      "reward_std": 0.1734784133732319,
      "rewards/accuracy_reward": 0.12723214738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812723517418,
      "step": 1249
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1447.9197082519531,
      "epoch": 0.37338510940183706,
      "grad_norm": 47.11534881591797,
      "kl": 3.232421875,
      "learning_rate": 3.945310798080647e-07,
      "loss": 0.301,
      "reward": 0.5094866305589676,
      "reward_std": 0.15938874520361423,
      "rewards/accuracy_reward": 0.07812500232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.431361623108387,
      "step": 1250
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1574.5915832519531,
      "epoch": 0.3736838174893585,
      "grad_norm": 24.14457130432129,
      "kl": 4.72265625,
      "learning_rate": 3.943182374154629e-07,
      "loss": 0.3417,
      "reward": 0.4508928805589676,
      "reward_std": 0.17686554789543152,
      "rewards/accuracy_reward": 0.024553572293370962,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263393059372902,
      "step": 1251
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1467.1853332519531,
      "epoch": 0.37398252557688,
      "grad_norm": 46.41028594970703,
      "kl": 4.42578125,
      "learning_rate": 3.941052380187634e-07,
      "loss": 0.3044,
      "reward": 0.5429687798023224,
      "reward_std": 0.2112753801047802,
      "rewards/accuracy_reward": 0.1004464328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4425223395228386,
      "step": 1252
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1503.4532165527344,
      "epoch": 0.37428123366440147,
      "grad_norm": 44.662269592285156,
      "kl": 5.07421875,
      "learning_rate": 3.938920818496888e-07,
      "loss": 0.3604,
      "reward": 0.4910714477300644,
      "reward_std": 0.11502698808908463,
      "rewards/accuracy_reward": 0.05357143096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000223517418,
      "step": 1253
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1472.4152221679688,
      "epoch": 0.37457994175192294,
      "grad_norm": 88.24928283691406,
      "kl": 2.556640625,
      "learning_rate": 3.9367876914013207e-07,
      "loss": 0.2725,
      "reward": 0.5167410969734192,
      "reward_std": 0.17018435895442963,
      "rewards/accuracy_reward": 0.07142857555299997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125223517418,
      "step": 1254
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1597.9598999023438,
      "epoch": 0.3748786498394444,
      "grad_norm": 38.169376373291016,
      "kl": 3.01953125,
      "learning_rate": 3.9346530012215664e-07,
      "loss": 0.2234,
      "reward": 0.4302455484867096,
      "reward_std": 0.13925860822200775,
      "rewards/accuracy_reward": 0.0066964291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.423549123108387,
      "step": 1255
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1451.62060546875,
      "epoch": 0.3751773579269659,
      "grad_norm": 90.10745239257812,
      "kl": 2.1328125,
      "learning_rate": 3.932516750279957e-07,
      "loss": 0.211,
      "reward": 0.5234375149011612,
      "reward_std": 0.10717580281198025,
      "rewards/accuracy_reward": 0.0803571455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4430803805589676,
      "step": 1256
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1538.10498046875,
      "epoch": 0.37547606601448735,
      "grad_norm": 59.14175796508789,
      "kl": 2.65234375,
      "learning_rate": 3.9303789409005253e-07,
      "loss": 0.2273,
      "reward": 0.4475446566939354,
      "reward_std": 0.16204673424363136,
      "rewards/accuracy_reward": 0.02455357275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 1257
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1445.5045166015625,
      "epoch": 0.3757747741020088,
      "grad_norm": 67.37281036376953,
      "kl": 3.2734375,
      "learning_rate": 3.928239575408998e-07,
      "loss": 0.3127,
      "reward": 0.521763414144516,
      "reward_std": 0.13154826872050762,
      "rewards/accuracy_reward": 0.0915178619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455559372902,
      "step": 1258
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1522.935302734375,
      "epoch": 0.3760734821895303,
      "grad_norm": 15.712522506713867,
      "kl": 3.89453125,
      "learning_rate": 3.926098656132796e-07,
      "loss": 0.3025,
      "reward": 0.5000000298023224,
      "reward_std": 0.16033565811812878,
      "rewards/accuracy_reward": 0.07589285937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.424107164144516,
      "step": 1259
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1493.9643859863281,
      "epoch": 0.37637219027705177,
      "grad_norm": 66.83240509033203,
      "kl": 3.4296875,
      "learning_rate": 3.9239561854010276e-07,
      "loss": 0.2679,
      "reward": 0.5323660969734192,
      "reward_std": 0.15324660763144493,
      "rewards/accuracy_reward": 0.10491071920841932,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4274553805589676,
      "step": 1260
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1459.47998046875,
      "epoch": 0.37667089836457324,
      "grad_norm": 78.82701110839844,
      "kl": 5.6328125,
      "learning_rate": 3.9218121655444926e-07,
      "loss": 0.3833,
      "reward": 0.5262276977300644,
      "reward_std": 0.15068095549941063,
      "rewards/accuracy_reward": 0.1004464328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812723517418,
      "step": 1261
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1479.1630249023438,
      "epoch": 0.3769696064520947,
      "grad_norm": 19.32759666442871,
      "kl": 4.3125,
      "learning_rate": 3.9196665988956743e-07,
      "loss": 0.3278,
      "reward": 0.5725446715950966,
      "reward_std": 0.14285107888281345,
      "rewards/accuracy_reward": 0.1383928656578064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341518059372902,
      "step": 1262
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1568.6763916015625,
      "epoch": 0.3772683145396162,
      "grad_norm": 40.979087829589844,
      "kl": 5.5703125,
      "learning_rate": 3.9175194877887387e-07,
      "loss": 0.408,
      "reward": 0.5585937723517418,
      "reward_std": 0.22559519112110138,
      "rewards/accuracy_reward": 0.14508928963914514,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4135044813156128,
      "step": 1263
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1419.2366638183594,
      "epoch": 0.37756702262713765,
      "grad_norm": 54.73875427246094,
      "kl": 5.2421875,
      "learning_rate": 3.9153708345595315e-07,
      "loss": 0.4203,
      "reward": 0.5078125298023224,
      "reward_std": 0.14354369789361954,
      "rewards/accuracy_reward": 0.06696428963914514,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4408482387661934,
      "step": 1264
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1534.3929138183594,
      "epoch": 0.3778657307146591,
      "grad_norm": 73.04139709472656,
      "kl": 5.8671875,
      "learning_rate": 3.913220641545578e-07,
      "loss": 0.4089,
      "reward": 0.4815848469734192,
      "reward_std": 0.13637750782072544,
      "rewards/accuracy_reward": 0.05357143096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4280134215950966,
      "step": 1265
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1517.2165832519531,
      "epoch": 0.3781644388021806,
      "grad_norm": 55.29218292236328,
      "kl": 4.83984375,
      "learning_rate": 3.9110689110860774e-07,
      "loss": 0.3252,
      "reward": 0.4994419887661934,
      "reward_std": 0.16542514227330685,
      "rewards/accuracy_reward": 0.05133928754366934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.448102705180645,
      "step": 1266
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1499.93310546875,
      "epoch": 0.37846314688970206,
      "grad_norm": 102.9538803100586,
      "kl": 6.4140625,
      "learning_rate": 3.908915645521901e-07,
      "loss": 0.4136,
      "reward": 0.5089285895228386,
      "reward_std": 0.14185668528079987,
      "rewards/accuracy_reward": 0.08705357694998384,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750149011612,
      "step": 1267
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1454.9152526855469,
      "epoch": 0.37876185497722353,
      "grad_norm": 48.31977462768555,
      "kl": 3.23828125,
      "learning_rate": 3.90676084719559e-07,
      "loss": 0.2689,
      "reward": 0.5803571790456772,
      "reward_std": 0.16422819904983044,
      "rewards/accuracy_reward": 0.13616072130389512,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4441964477300644,
      "step": 1268
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1561.9822082519531,
      "epoch": 0.379060563064745,
      "grad_norm": 38.79275131225586,
      "kl": 4.8203125,
      "learning_rate": 3.904604518451356e-07,
      "loss": 0.333,
      "reward": 0.4570312723517418,
      "reward_std": 0.17782100848853588,
      "rewards/accuracy_reward": 0.0513392873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4056919813156128,
      "step": 1269
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1553.1005249023438,
      "epoch": 0.3793592711522665,
      "grad_norm": 18.94418716430664,
      "kl": 4.66796875,
      "learning_rate": 3.9024466616350713e-07,
      "loss": 0.3554,
      "reward": 0.537388414144516,
      "reward_std": 0.19001031666994095,
      "rewards/accuracy_reward": 0.1339285783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.403459832072258,
      "step": 1270
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1413.15185546875,
      "epoch": 0.3796579792397879,
      "grad_norm": 37.71917724609375,
      "kl": 3.4765625,
      "learning_rate": 3.900287279094273e-07,
      "loss": 0.3311,
      "reward": 0.5334821715950966,
      "reward_std": 0.18773821741342545,
      "rewards/accuracy_reward": 0.09375000465661287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.439732164144516,
      "step": 1271
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1471.1250305175781,
      "epoch": 0.37995668732730936,
      "grad_norm": 81.22847747802734,
      "kl": 2.0859375,
      "learning_rate": 3.898126373178158e-07,
      "loss": 0.2341,
      "reward": 0.5061384066939354,
      "reward_std": 0.17557835765182972,
      "rewards/accuracy_reward": 0.07142857648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098469734192,
      "step": 1272
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1611.2857971191406,
      "epoch": 0.38025539541483083,
      "grad_norm": 12.227075576782227,
      "kl": 4.15625,
      "learning_rate": 3.89596394623758e-07,
      "loss": 0.2844,
      "reward": 0.5613839477300644,
      "reward_std": 0.2170797437429428,
      "rewards/accuracy_reward": 0.15178572200238705,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4095982387661934,
      "step": 1273
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1466.0781860351562,
      "epoch": 0.3805541035023523,
      "grad_norm": 61.89448547363281,
      "kl": 2.96875,
      "learning_rate": 3.893800000625049e-07,
      "loss": 0.2724,
      "reward": 0.525111623108387,
      "reward_std": 0.17459117993712425,
      "rewards/accuracy_reward": 0.0848214328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4402901902794838,
      "step": 1274
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1517.3192749023438,
      "epoch": 0.3808528115898738,
      "grad_norm": 50.783721923828125,
      "kl": 2.7744140625,
      "learning_rate": 3.891634538694723e-07,
      "loss": 0.225,
      "reward": 0.5731026902794838,
      "reward_std": 0.1678936406970024,
      "rewards/accuracy_reward": 0.1361607164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419813156128,
      "step": 1275
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1455.9197082519531,
      "epoch": 0.38115151967739525,
      "grad_norm": 67.63020324707031,
      "kl": 3.287109375,
      "learning_rate": 3.889467562802414e-07,
      "loss": 0.3179,
      "reward": 0.5066964477300644,
      "reward_std": 0.16733341291546822,
      "rewards/accuracy_reward": 0.07366071757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357313156128,
      "step": 1276
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1532.4465026855469,
      "epoch": 0.3814502277649167,
      "grad_norm": 27.781240463256836,
      "kl": 5.14453125,
      "learning_rate": 3.88729907530558e-07,
      "loss": 0.3658,
      "reward": 0.5212053805589676,
      "reward_std": 0.1363147608935833,
      "rewards/accuracy_reward": 0.1116071455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4095982313156128,
      "step": 1277
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1495.6942749023438,
      "epoch": 0.3817489358524382,
      "grad_norm": 36.19876480102539,
      "kl": 4.7734375,
      "learning_rate": 3.885129078563321e-07,
      "loss": 0.3575,
      "reward": 0.5797991380095482,
      "reward_std": 0.21181683987379074,
      "rewards/accuracy_reward": 0.16741072060540318,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.412388414144516,
      "step": 1278
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1540.8482971191406,
      "epoch": 0.38204764393995966,
      "grad_norm": 9.210381507873535,
      "kl": 4.48046875,
      "learning_rate": 3.8829575749363817e-07,
      "loss": 0.3254,
      "reward": 0.5178571715950966,
      "reward_std": 0.14574753865599632,
      "rewards/accuracy_reward": 0.08482143399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357387661934,
      "step": 1279
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1431.0379943847656,
      "epoch": 0.38234635202748113,
      "grad_norm": 22.136837005615234,
      "kl": 4.04296875,
      "learning_rate": 3.880784566787145e-07,
      "loss": 0.3627,
      "reward": 0.4709821715950966,
      "reward_std": 0.1648912914097309,
      "rewards/accuracy_reward": 0.03794642956927419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357313156128,
      "step": 1280
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1555.7723693847656,
      "epoch": 0.3826450601150026,
      "grad_norm": 9.385151863098145,
      "kl": 3.6484375,
      "learning_rate": 3.878610056479632e-07,
      "loss": 0.2766,
      "reward": 0.4665178805589676,
      "reward_std": 0.18152549117803574,
      "rewards/accuracy_reward": 0.04017857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263393059372902,
      "step": 1281
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1572.9688110351562,
      "epoch": 0.38294376820252407,
      "grad_norm": 46.929527282714844,
      "kl": 4.375,
      "learning_rate": 3.8764340463794954e-07,
      "loss": 0.2764,
      "reward": 0.5518973469734192,
      "reward_std": 0.20453695766627789,
      "rewards/accuracy_reward": 0.1183035746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937723517418,
      "step": 1282
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1501.2009887695312,
      "epoch": 0.38324247629004554,
      "grad_norm": 34.31199645996094,
      "kl": 3.296875,
      "learning_rate": 3.8742565388540214e-07,
      "loss": 0.2697,
      "reward": 0.5468750223517418,
      "reward_std": 0.1539290864020586,
      "rewards/accuracy_reward": 0.09598214761354029,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4508928805589676,
      "step": 1283
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1583.1697082519531,
      "epoch": 0.383541184377567,
      "grad_norm": 60.0711784362793,
      "kl": 6.140625,
      "learning_rate": 3.8720775362721253e-07,
      "loss": 0.4217,
      "reward": 0.5284598395228386,
      "reward_std": 0.13499336317181587,
      "rewards/accuracy_reward": 0.11160714668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4168526977300644,
      "step": 1284
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1478.0625915527344,
      "epoch": 0.3838398924650885,
      "grad_norm": 19.3468017578125,
      "kl": 4.53125,
      "learning_rate": 3.8698970410043475e-07,
      "loss": 0.3328,
      "reward": 0.5195312723517418,
      "reward_std": 0.14610961265861988,
      "rewards/accuracy_reward": 0.09151786053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428013414144516,
      "step": 1285
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1456.8125610351562,
      "epoch": 0.38413860055260995,
      "grad_norm": 30.484533309936523,
      "kl": 4.55859375,
      "learning_rate": 3.8677150554228534e-07,
      "loss": 0.3279,
      "reward": 0.455357164144516,
      "reward_std": 0.1288006417453289,
      "rewards/accuracy_reward": 0.020089285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4352678805589676,
      "step": 1286
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1487.9085388183594,
      "epoch": 0.3844373086401314,
      "grad_norm": 50.648948669433594,
      "kl": 2.96484375,
      "learning_rate": 3.865531581901429e-07,
      "loss": 0.27,
      "reward": 0.4888393133878708,
      "reward_std": 0.120752127841115,
      "rewards/accuracy_reward": 0.044642860535532236,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.444196455180645,
      "step": 1287
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1526.22998046875,
      "epoch": 0.3847360167276529,
      "grad_norm": 24.656667709350586,
      "kl": 4.09375,
      "learning_rate": 3.8633466228154805e-07,
      "loss": 0.3221,
      "reward": 0.5786830708384514,
      "reward_std": 0.1672380492091179,
      "rewards/accuracy_reward": 0.1428571529686451,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4358259215950966,
      "step": 1288
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1463.0536499023438,
      "epoch": 0.38503472481517437,
      "grad_norm": 63.84490966796875,
      "kl": 2.583984375,
      "learning_rate": 3.861160180542028e-07,
      "loss": 0.2289,
      "reward": 0.5541294738650322,
      "reward_std": 0.18520366586744785,
      "rewards/accuracy_reward": 0.10937500232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4447544887661934,
      "step": 1289
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1388.7745971679688,
      "epoch": 0.38533343290269584,
      "grad_norm": 63.11453628540039,
      "kl": 1.92578125,
      "learning_rate": 3.8589722574597077e-07,
      "loss": 0.1756,
      "reward": 0.4877232313156128,
      "reward_std": 0.11110446229577065,
      "rewards/accuracy_reward": 0.0290178582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4587053805589676,
      "step": 1290
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1527.0447082519531,
      "epoch": 0.3856321409902173,
      "grad_norm": 18.445737838745117,
      "kl": 3.53125,
      "learning_rate": 3.8567828559487645e-07,
      "loss": 0.2742,
      "reward": 0.482142873108387,
      "reward_std": 0.18583733588457108,
      "rewards/accuracy_reward": 0.051339288242161274,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035895228386,
      "step": 1291
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1402.5715026855469,
      "epoch": 0.3859308490777388,
      "grad_norm": 14.82524585723877,
      "kl": 3.35546875,
      "learning_rate": 3.854591978391054e-07,
      "loss": 0.2731,
      "reward": 0.6166294813156128,
      "reward_std": 0.16885140351951122,
      "rewards/accuracy_reward": 0.1718750111758709,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4447544813156128,
      "step": 1292
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1451.9576721191406,
      "epoch": 0.38622955716526025,
      "grad_norm": 27.912349700927734,
      "kl": 4.10546875,
      "learning_rate": 3.8523996271700343e-07,
      "loss": 0.3641,
      "reward": 0.4815848469734192,
      "reward_std": 0.14909899793565273,
      "rewards/accuracy_reward": 0.051339289639145136,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455484867096,
      "step": 1293
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1482.2344665527344,
      "epoch": 0.3865282652527817,
      "grad_norm": 13.441256523132324,
      "kl": 3.7265625,
      "learning_rate": 3.85020580467077e-07,
      "loss": 0.3085,
      "reward": 0.5396205633878708,
      "reward_std": 0.17871698923408985,
      "rewards/accuracy_reward": 0.0915178619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.448102705180645,
      "step": 1294
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1462.2857666015625,
      "epoch": 0.3868269733403032,
      "grad_norm": 63.02687072753906,
      "kl": 4.83984375,
      "learning_rate": 3.8480105132799244e-07,
      "loss": 0.3327,
      "reward": 0.5770089477300644,
      "reward_std": 0.150366535410285,
      "rewards/accuracy_reward": 0.145089291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.431919664144516,
      "step": 1295
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1485.4621276855469,
      "epoch": 0.38712568142782466,
      "grad_norm": 110.61550903320312,
      "kl": 6.109375,
      "learning_rate": 3.84581375538576e-07,
      "loss": 0.4225,
      "reward": 0.5212053656578064,
      "reward_std": 0.18514585867524147,
      "rewards/accuracy_reward": 0.08928571944124997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.431919664144516,
      "step": 1296
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1547.7522888183594,
      "epoch": 0.38742438951534613,
      "grad_norm": 111.26005554199219,
      "kl": 6.796875,
      "learning_rate": 3.843615533378133e-07,
      "loss": 0.4716,
      "reward": 0.4659598395228386,
      "reward_std": 0.2050965502858162,
      "rewards/accuracy_reward": 0.053571431431919336,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.412388414144516,
      "step": 1297
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1688.2835388183594,
      "epoch": 0.3877230976028676,
      "grad_norm": 147.81222534179688,
      "kl": 7.84375,
      "learning_rate": 3.841415849648495e-07,
      "loss": 0.4872,
      "reward": 0.4693080559372902,
      "reward_std": 0.22909464314579964,
      "rewards/accuracy_reward": 0.07589286053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3934151977300644,
      "step": 1298
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1473.0291137695312,
      "epoch": 0.3880218056903891,
      "grad_norm": 116.06525421142578,
      "kl": 6.6953125,
      "learning_rate": 3.839214706589885e-07,
      "loss": 0.4722,
      "reward": 0.5340402126312256,
      "reward_std": 0.18341074883937836,
      "rewards/accuracy_reward": 0.10044643469154835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937723517418,
      "step": 1299
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1519.5000915527344,
      "epoch": 0.38832051377791055,
      "grad_norm": 167.61610412597656,
      "kl": 7.484375,
      "learning_rate": 3.837012106596932e-07,
      "loss": 0.4546,
      "reward": 0.491071455180645,
      "reward_std": 0.1913202926516533,
      "rewards/accuracy_reward": 0.06696428917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4241071566939354,
      "step": 1300
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1519.1451721191406,
      "epoch": 0.388619221865432,
      "grad_norm": 60.138526916503906,
      "kl": 4.75,
      "learning_rate": 3.8348080520658477e-07,
      "loss": 0.3322,
      "reward": 0.5647321715950966,
      "reward_std": 0.15331684984266758,
      "rewards/accuracy_reward": 0.11830357648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4464285895228386,
      "step": 1301
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1553.0067749023438,
      "epoch": 0.3889179299529535,
      "grad_norm": 23.569028854370117,
      "kl": 3.9140625,
      "learning_rate": 3.832602545394427e-07,
      "loss": 0.3211,
      "reward": 0.5552455708384514,
      "reward_std": 0.20371198281645775,
      "rewards/accuracy_reward": 0.12946428917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812723517418,
      "step": 1302
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1464.9643249511719,
      "epoch": 0.38921663804047496,
      "grad_norm": 10.51165771484375,
      "kl": 3.84375,
      "learning_rate": 3.830395588982047e-07,
      "loss": 0.3108,
      "reward": 0.4603794813156128,
      "reward_std": 0.13618669658899307,
      "rewards/accuracy_reward": 0.024553573224693537,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435825914144516,
      "step": 1303
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1502.5715026855469,
      "epoch": 0.38951534612799643,
      "grad_norm": 41.0344123840332,
      "kl": 3.6484375,
      "learning_rate": 3.8281871852296597e-07,
      "loss": 0.3125,
      "reward": 0.4693080484867096,
      "reward_std": 0.1203591600060463,
      "rewards/accuracy_reward": 0.0424107164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.426897332072258,
      "step": 1304
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1508.6495971679688,
      "epoch": 0.3898140542155179,
      "grad_norm": 8.734365463256836,
      "kl": 3.5078125,
      "learning_rate": 3.825977336539791e-07,
      "loss": 0.2661,
      "reward": 0.4955357313156128,
      "reward_std": 0.17815385200083256,
      "rewards/accuracy_reward": 0.0602678619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4352678805589676,
      "step": 1305
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1537.33935546875,
      "epoch": 0.3901127623030394,
      "grad_norm": 33.38508224487305,
      "kl": 2.86328125,
      "learning_rate": 3.8237660453165386e-07,
      "loss": 0.2541,
      "reward": 0.5323660969734192,
      "reward_std": 0.15103640407323837,
      "rewards/accuracy_reward": 0.09821428963914514,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341518059372902,
      "step": 1306
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1440.5402221679688,
      "epoch": 0.39041147039056084,
      "grad_norm": 52.93671417236328,
      "kl": 3.154296875,
      "learning_rate": 3.821553313965573e-07,
      "loss": 0.2862,
      "reward": 0.5546875223517418,
      "reward_std": 0.12212117575109005,
      "rewards/accuracy_reward": 0.11607143399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4386160895228386,
      "step": 1307
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1544.071533203125,
      "epoch": 0.3907101784780823,
      "grad_norm": 9.903542518615723,
      "kl": 4.45703125,
      "learning_rate": 3.8193391448941277e-07,
      "loss": 0.3392,
      "reward": 0.4709821715950966,
      "reward_std": 0.20191789418458939,
      "rewards/accuracy_reward": 0.05357143213041127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4174107387661934,
      "step": 1308
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1392.1496276855469,
      "epoch": 0.3910088865656038,
      "grad_norm": 68.98812866210938,
      "kl": 2.068359375,
      "learning_rate": 3.8171235405110013e-07,
      "loss": 0.2012,
      "reward": 0.5167411044239998,
      "reward_std": 0.1570369489490986,
      "rewards/accuracy_reward": 0.07366071781143546,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4430803805589676,
      "step": 1309
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1513.3840026855469,
      "epoch": 0.39130759465312526,
      "grad_norm": 72.37940979003906,
      "kl": 2.47265625,
      "learning_rate": 3.814906503226556e-07,
      "loss": 0.2274,
      "reward": 0.5239955633878708,
      "reward_std": 0.182341530919075,
      "rewards/accuracy_reward": 0.08258929150179029,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4414062723517418,
      "step": 1310
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1474.1183776855469,
      "epoch": 0.39160630274064673,
      "grad_norm": 40.21659851074219,
      "kl": 3.302734375,
      "learning_rate": 3.812688035452709e-07,
      "loss": 0.2926,
      "reward": 0.4866071715950966,
      "reward_std": 0.14452451653778553,
      "rewards/accuracy_reward": 0.05357143026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357313156128,
      "step": 1311
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1515.60498046875,
      "epoch": 0.3919050108281682,
      "grad_norm": 25.677051544189453,
      "kl": 3.69921875,
      "learning_rate": 3.810468139602937e-07,
      "loss": 0.3181,
      "reward": 0.5200893133878708,
      "reward_std": 0.19294268265366554,
      "rewards/accuracy_reward": 0.09375000465661287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263393133878708,
      "step": 1312
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1462.0313110351562,
      "epoch": 0.39220371891568967,
      "grad_norm": 17.955053329467773,
      "kl": 3.08203125,
      "learning_rate": 3.8082468180922683e-07,
      "loss": 0.1982,
      "reward": 0.6417410969734192,
      "reward_std": 0.20492911711335182,
      "rewards/accuracy_reward": 0.1941964365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.447544664144516,
      "step": 1313
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1393.0804443359375,
      "epoch": 0.3925024270032111,
      "grad_norm": 60.0616569519043,
      "kl": 4.91796875,
      "learning_rate": 3.806024073337284e-07,
      "loss": 0.3678,
      "reward": 0.5362723395228386,
      "reward_std": 0.1712651439011097,
      "rewards/accuracy_reward": 0.09375000488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4425223395228386,
      "step": 1314
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1588.6965026855469,
      "epoch": 0.39280113509073256,
      "grad_norm": 35.023162841796875,
      "kl": 4.66015625,
      "learning_rate": 3.803799907756112e-07,
      "loss": 0.3264,
      "reward": 0.5786830633878708,
      "reward_std": 0.16471973434090614,
      "rewards/accuracy_reward": 0.14062500488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4380580559372902,
      "step": 1315
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1522.5313110351562,
      "epoch": 0.393099843178254,
      "grad_norm": 21.160337448120117,
      "kl": 3.59765625,
      "learning_rate": 3.801574323768426e-07,
      "loss": 0.2876,
      "reward": 0.5401785895228386,
      "reward_std": 0.15324592031538486,
      "rewards/accuracy_reward": 0.10491072107106447,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4352678805589676,
      "step": 1316
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1490.04248046875,
      "epoch": 0.3933985512657755,
      "grad_norm": 18.046106338500977,
      "kl": 3.025390625,
      "learning_rate": 3.7993473237954445e-07,
      "loss": 0.2205,
      "reward": 0.4799107313156128,
      "reward_std": 0.16653244942426682,
      "rewards/accuracy_reward": 0.042410715483129025,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000223517418,
      "step": 1317
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1505.1652526855469,
      "epoch": 0.39369725935329697,
      "grad_norm": 10.065442085266113,
      "kl": 3.38671875,
      "learning_rate": 3.7971189102599246e-07,
      "loss": 0.2522,
      "reward": 0.5279018059372902,
      "reward_std": 0.19211949035525322,
      "rewards/accuracy_reward": 0.0848214328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4430803805589676,
      "step": 1318
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1517.68310546875,
      "epoch": 0.39399596744081844,
      "grad_norm": 35.18148422241211,
      "kl": 3.41796875,
      "learning_rate": 3.7948890855861603e-07,
      "loss": 0.2611,
      "reward": 0.5613839477300644,
      "reward_std": 0.11361411958932877,
      "rewards/accuracy_reward": 0.11830357648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4430803805589676,
      "step": 1319
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1495.7523193359375,
      "epoch": 0.3942946755283399,
      "grad_norm": 16.243579864501953,
      "kl": 3.59375,
      "learning_rate": 3.7926578521999843e-07,
      "loss": 0.2723,
      "reward": 0.517857164144516,
      "reward_std": 0.15928495675325394,
      "rewards/accuracy_reward": 0.07812500116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.439732164144516,
      "step": 1320
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1458.6585388183594,
      "epoch": 0.3945933836158614,
      "grad_norm": 29.111801147460938,
      "kl": 4.40234375,
      "learning_rate": 3.7904252125287594e-07,
      "loss": 0.319,
      "reward": 0.4737723395228386,
      "reward_std": 0.15724255610257387,
      "rewards/accuracy_reward": 0.0379464291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4358259066939354,
      "step": 1321
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1520.7433776855469,
      "epoch": 0.39489209170338285,
      "grad_norm": 24.655000686645508,
      "kl": 3.7109375,
      "learning_rate": 3.788191169001379e-07,
      "loss": 0.311,
      "reward": 0.4564732387661934,
      "reward_std": 0.17004377208650112,
      "rewards/accuracy_reward": 0.02455357275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4319196566939354,
      "step": 1322
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1447.1384582519531,
      "epoch": 0.3951907997909043,
      "grad_norm": 11.153548240661621,
      "kl": 3.95703125,
      "learning_rate": 3.785955724048262e-07,
      "loss": 0.3263,
      "reward": 0.5016741305589676,
      "reward_std": 0.13410876877605915,
      "rewards/accuracy_reward": 0.055803574388846755,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4458705633878708,
      "step": 1323
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1419.1942749023438,
      "epoch": 0.3954895078784258,
      "grad_norm": 41.64998245239258,
      "kl": 5.015625,
      "learning_rate": 3.7837188801013553e-07,
      "loss": 0.3784,
      "reward": 0.5256696715950966,
      "reward_std": 0.14723791368305683,
      "rewards/accuracy_reward": 0.09598214644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875223517418,
      "step": 1324
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1480.8438110351562,
      "epoch": 0.39578821596594727,
      "grad_norm": 26.610553741455078,
      "kl": 4.4453125,
      "learning_rate": 3.7814806395941246e-07,
      "loss": 0.3203,
      "reward": 0.5608259290456772,
      "reward_std": 0.1297959890216589,
      "rewards/accuracy_reward": 0.1227678582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4380580559372902,
      "step": 1325
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1444.3884582519531,
      "epoch": 0.39608692405346874,
      "grad_norm": 19.633434295654297,
      "kl": 4.21484375,
      "learning_rate": 3.7792410049615567e-07,
      "loss": 0.3692,
      "reward": 0.5680803805589676,
      "reward_std": 0.16257998906075954,
      "rewards/accuracy_reward": 0.129464291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4386160969734192,
      "step": 1326
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1493.0290832519531,
      "epoch": 0.3963856321409902,
      "grad_norm": 37.3018684387207,
      "kl": 4.82421875,
      "learning_rate": 3.776999978640154e-07,
      "loss": 0.3433,
      "reward": 0.5429687649011612,
      "reward_std": 0.1649802289903164,
      "rewards/accuracy_reward": 0.1160714328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4268973395228386,
      "step": 1327
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1547.6496276855469,
      "epoch": 0.3966843402285117,
      "grad_norm": 12.610398292541504,
      "kl": 4.40234375,
      "learning_rate": 3.774757563067935e-07,
      "loss": 0.3254,
      "reward": 0.5585937723517418,
      "reward_std": 0.14634745381772518,
      "rewards/accuracy_reward": 0.12946429196745157,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294813156128,
      "step": 1328
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1561.5960693359375,
      "epoch": 0.39698304831603315,
      "grad_norm": 25.606586456298828,
      "kl": 5.0078125,
      "learning_rate": 3.7725137606844273e-07,
      "loss": 0.3586,
      "reward": 0.5066964477300644,
      "reward_std": 0.19275490753352642,
      "rewards/accuracy_reward": 0.08482143189758062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750074505806,
      "step": 1329
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1482.3572082519531,
      "epoch": 0.3972817564035546,
      "grad_norm": 11.971052169799805,
      "kl": 4.35546875,
      "learning_rate": 3.770268573930666e-07,
      "loss": 0.3745,
      "reward": 0.486607164144516,
      "reward_std": 0.19278451800346375,
      "rewards/accuracy_reward": 0.05803571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 1330
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1507.8996276855469,
      "epoch": 0.3975804644910761,
      "grad_norm": 58.936798095703125,
      "kl": 3.23046875,
      "learning_rate": 3.7680220052491974e-07,
      "loss": 0.2903,
      "reward": 0.588727705180645,
      "reward_std": 0.120096854865551,
      "rewards/accuracy_reward": 0.1540178656578064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098395228386,
      "step": 1331
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1419.8728637695312,
      "epoch": 0.39787917257859756,
      "grad_norm": 46.11073684692383,
      "kl": 2.7265625,
      "learning_rate": 3.765774057084066e-07,
      "loss": 0.2468,
      "reward": 0.6121652200818062,
      "reward_std": 0.1577361524105072,
      "rewards/accuracy_reward": 0.16517857951112092,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4469866305589676,
      "step": 1332
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1562.2165832519531,
      "epoch": 0.39817788066611903,
      "grad_norm": 92.28487396240234,
      "kl": 5.5859375,
      "learning_rate": 3.76352473188082e-07,
      "loss": 0.354,
      "reward": 0.495535746216774,
      "reward_std": 0.14340483769774437,
      "rewards/accuracy_reward": 0.06696428777649999,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 1333
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1492.5603332519531,
      "epoch": 0.3984765887536405,
      "grad_norm": 15.35764217376709,
      "kl": 3.9375,
      "learning_rate": 3.761274032086504e-07,
      "loss": 0.3135,
      "reward": 0.5345982238650322,
      "reward_std": 0.1641866471618414,
      "rewards/accuracy_reward": 0.09598214970901608,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4386160895228386,
      "step": 1334
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1543.8304138183594,
      "epoch": 0.398775296841162,
      "grad_norm": 17.701705932617188,
      "kl": 3.646484375,
      "learning_rate": 3.7590219601496596e-07,
      "loss": 0.2776,
      "reward": 0.5714285969734192,
      "reward_std": 0.17822470515966415,
      "rewards/accuracy_reward": 0.1272321492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4441964477300644,
      "step": 1335
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1545.1808776855469,
      "epoch": 0.39907400492868345,
      "grad_norm": 100.78438568115234,
      "kl": 5.3046875,
      "learning_rate": 3.7567685185203203e-07,
      "loss": 0.3347,
      "reward": 0.6104911118745804,
      "reward_std": 0.17931295558810234,
      "rewards/accuracy_reward": 0.16517858020961285,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125223517418,
      "step": 1336
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1599.5223693847656,
      "epoch": 0.3993727130162049,
      "grad_norm": 153.1273193359375,
      "kl": 7.546875,
      "learning_rate": 3.754513709650008e-07,
      "loss": 0.4674,
      "reward": 0.5647321715950966,
      "reward_std": 0.1779017485678196,
      "rewards/accuracy_reward": 0.1383928656578064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263393059372902,
      "step": 1337
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1419.1317749023438,
      "epoch": 0.3996714211037264,
      "grad_norm": 71.07659149169922,
      "kl": 5.2890625,
      "learning_rate": 3.7522575359917364e-07,
      "loss": 0.4236,
      "reward": 0.5781250298023224,
      "reward_std": 0.1936220731586218,
      "rewards/accuracy_reward": 0.13839286006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4397321715950966,
      "step": 1338
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1441.5000915527344,
      "epoch": 0.39997012919124786,
      "grad_norm": 57.1750602722168,
      "kl": 4.849609375,
      "learning_rate": 3.75e-07,
      "loss": 0.3621,
      "reward": 0.5887276977300644,
      "reward_std": 0.16095156036317348,
      "rewards/accuracy_reward": 0.1383928619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4503348395228386,
      "step": 1339
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1512.4442443847656,
      "epoch": 0.40026883727876933,
      "grad_norm": 109.33737182617188,
      "kl": 5.7265625,
      "learning_rate": 3.747741104130777e-07,
      "loss": 0.4089,
      "reward": 0.581473246216774,
      "reward_std": 0.16761961951851845,
      "rewards/accuracy_reward": 0.13616072316654027,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125149011612,
      "step": 1340
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1577.8415832519531,
      "epoch": 0.4005675453662908,
      "grad_norm": 43.297401428222656,
      "kl": 4.80859375,
      "learning_rate": 3.7454808508415237e-07,
      "loss": 0.3224,
      "reward": 0.517299123108387,
      "reward_std": 0.1896546706557274,
      "rewards/accuracy_reward": 0.0870535746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455559372902,
      "step": 1341
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1471.38623046875,
      "epoch": 0.40086625345381227,
      "grad_norm": 10.072811126708984,
      "kl": 3.6640625,
      "learning_rate": 3.743219242591175e-07,
      "loss": 0.2735,
      "reward": 0.531250037252903,
      "reward_std": 0.12994037196040154,
      "rewards/accuracy_reward": 0.0892857201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4419643059372902,
      "step": 1342
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1524.3907165527344,
      "epoch": 0.40116496154133374,
      "grad_norm": 39.315330505371094,
      "kl": 2.78515625,
      "learning_rate": 3.7409562818401395e-07,
      "loss": 0.2113,
      "reward": 0.5055803880095482,
      "reward_std": 0.19099636003375053,
      "rewards/accuracy_reward": 0.0736607164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.431919664144516,
      "step": 1343
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1559.1563110351562,
      "epoch": 0.4014636696288552,
      "grad_norm": 48.78361129760742,
      "kl": 2.896484375,
      "learning_rate": 3.738691971050296e-07,
      "loss": 0.2417,
      "reward": 0.5044642984867096,
      "reward_std": 0.15767910704016685,
      "rewards/accuracy_reward": 0.06696428963914514,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000223517418,
      "step": 1344
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1556.5067443847656,
      "epoch": 0.4017623777163767,
      "grad_norm": 26.767873764038086,
      "kl": 3.23828125,
      "learning_rate": 3.7364263126849924e-07,
      "loss": 0.2685,
      "reward": 0.5050223469734192,
      "reward_std": 0.1915640141814947,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937649011612,
      "step": 1345
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1523.2723999023438,
      "epoch": 0.40206108580389815,
      "grad_norm": 52.8403205871582,
      "kl": 2.568359375,
      "learning_rate": 3.734159309209044e-07,
      "loss": 0.2298,
      "reward": 0.4977678880095482,
      "reward_std": 0.10809685103595257,
      "rewards/accuracy_reward": 0.04687500209547579,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4508928805589676,
      "step": 1346
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1586.5313415527344,
      "epoch": 0.4023597938914196,
      "grad_norm": 19.748279571533203,
      "kl": 3.69921875,
      "learning_rate": 3.7318909630887276e-07,
      "loss": 0.2391,
      "reward": 0.466517873108387,
      "reward_std": 0.15702650509774685,
      "rewards/accuracy_reward": 0.03348214365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357387661934,
      "step": 1347
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1622.2121276855469,
      "epoch": 0.4026585019789411,
      "grad_norm": 8.773845672607422,
      "kl": 3.95703125,
      "learning_rate": 3.7296212767917813e-07,
      "loss": 0.2685,
      "reward": 0.474330373108387,
      "reward_std": 0.15803285501897335,
      "rewards/accuracy_reward": 0.058035717345774174,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.416294664144516,
      "step": 1348
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1521.8706359863281,
      "epoch": 0.40295721006646257,
      "grad_norm": 19.897489547729492,
      "kl": 3.9921875,
      "learning_rate": 3.7273502527874017e-07,
      "loss": 0.2927,
      "reward": 0.5630580633878708,
      "reward_std": 0.14937039092183113,
      "rewards/accuracy_reward": 0.12946428917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937723517418,
      "step": 1349
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1516.1540832519531,
      "epoch": 0.40325591815398404,
      "grad_norm": 51.03802490234375,
      "kl": 3.0,
      "learning_rate": 3.725077893546241e-07,
      "loss": 0.2527,
      "reward": 0.5982143208384514,
      "reward_std": 0.12906441651284695,
      "rewards/accuracy_reward": 0.1517857201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4464285895228386,
      "step": 1350
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1643.2121276855469,
      "epoch": 0.4035546262415055,
      "grad_norm": 42.61442565917969,
      "kl": 4.84375,
      "learning_rate": 3.722804201540403e-07,
      "loss": 0.3132,
      "reward": 0.4659598395228386,
      "reward_std": 0.20050760731101036,
      "rewards/accuracy_reward": 0.058035718742758036,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4079241305589676,
      "step": 1351
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1532.3081359863281,
      "epoch": 0.403853334329027,
      "grad_norm": 53.0906867980957,
      "kl": 2.953125,
      "learning_rate": 3.7205291792434397e-07,
      "loss": 0.2272,
      "reward": 0.4508928805589676,
      "reward_std": 0.10211499035358429,
      "rewards/accuracy_reward": 0.004464285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4464285969734192,
      "step": 1352
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1510.7322082519531,
      "epoch": 0.40415204241654845,
      "grad_norm": 7.686004638671875,
      "kl": 3.671875,
      "learning_rate": 3.718252829130355e-07,
      "loss": 0.2604,
      "reward": 0.4765625223517418,
      "reward_std": 0.14839400351047516,
      "rewards/accuracy_reward": 0.0558035746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4207589477300644,
      "step": 1353
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1624.33935546875,
      "epoch": 0.4044507505040699,
      "grad_norm": 48.989070892333984,
      "kl": 4.515625,
      "learning_rate": 3.715975153677594e-07,
      "loss": 0.2782,
      "reward": 0.5652902126312256,
      "reward_std": 0.17247237637639046,
      "rewards/accuracy_reward": 0.14062500488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4246651902794838,
      "step": 1354
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1429.0223999023438,
      "epoch": 0.4047494585915914,
      "grad_norm": 13.560033798217773,
      "kl": 4.046875,
      "learning_rate": 3.7136961553630437e-07,
      "loss": 0.3189,
      "reward": 0.5189732387661934,
      "reward_std": 0.12493516132235527,
      "rewards/accuracy_reward": 0.08258928940631449,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4363839477300644,
      "step": 1355
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1511.5335693359375,
      "epoch": 0.40504816667911286,
      "grad_norm": 32.09595489501953,
      "kl": 3.63671875,
      "learning_rate": 3.711415836666032e-07,
      "loss": 0.2944,
      "reward": 0.4760044887661934,
      "reward_std": 0.1568305417895317,
      "rewards/accuracy_reward": 0.040178573690354824,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435825914144516,
      "step": 1356
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1534.790283203125,
      "epoch": 0.4053468747666343,
      "grad_norm": 23.30109977722168,
      "kl": 4.171875,
      "learning_rate": 3.7091342000673216e-07,
      "loss": 0.2897,
      "reward": 0.4972098395228386,
      "reward_std": 0.16137411631643772,
      "rewards/accuracy_reward": 0.07142857555299997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812723517418,
      "step": 1357
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1418.4085388183594,
      "epoch": 0.40564558285415575,
      "grad_norm": 31.130617141723633,
      "kl": 2.78515625,
      "learning_rate": 3.706851248049109e-07,
      "loss": 0.2305,
      "reward": 0.5440848469734192,
      "reward_std": 0.11512686870992184,
      "rewards/accuracy_reward": 0.0915178619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4525669813156128,
      "step": 1358
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1511.7344665527344,
      "epoch": 0.4059442909416772,
      "grad_norm": 41.73582077026367,
      "kl": 3.078125,
      "learning_rate": 3.704566983095022e-07,
      "loss": 0.2581,
      "reward": 0.5803571715950966,
      "reward_std": 0.17149999737739563,
      "rewards/accuracy_reward": 0.12946428824216127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.450892873108387,
      "step": 1359
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1620.9197387695312,
      "epoch": 0.4062429990291987,
      "grad_norm": 17.62242317199707,
      "kl": 3.91015625,
      "learning_rate": 3.7022814076901187e-07,
      "loss": 0.2696,
      "reward": 0.5156250149011612,
      "reward_std": 0.12963330373167992,
      "rewards/accuracy_reward": 0.082589291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357313156128,
      "step": 1360
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1617.7254943847656,
      "epoch": 0.40654170711672016,
      "grad_norm": 21.543987274169922,
      "kl": 3.84765625,
      "learning_rate": 3.6999945243208795e-07,
      "loss": 0.2756,
      "reward": 0.6026785969734192,
      "reward_std": 0.18320314958691597,
      "rewards/accuracy_reward": 0.1830357238650322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.419642873108387,
      "step": 1361
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1549.6005249023438,
      "epoch": 0.40684041520424163,
      "grad_norm": 62.5572509765625,
      "kl": 5.1953125,
      "learning_rate": 3.69770633547521e-07,
      "loss": 0.3651,
      "reward": 0.556919664144516,
      "reward_std": 0.20681850612163544,
      "rewards/accuracy_reward": 0.1093750037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4475446566939354,
      "step": 1362
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1519.8683471679688,
      "epoch": 0.4071391232917631,
      "grad_norm": 47.360172271728516,
      "kl": 4.57421875,
      "learning_rate": 3.695416843642436e-07,
      "loss": 0.3031,
      "reward": 0.517299123108387,
      "reward_std": 0.1952129751443863,
      "rewards/accuracy_reward": 0.08928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428013414144516,
      "step": 1363
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1547.5648193359375,
      "epoch": 0.4074378313792846,
      "grad_norm": 46.98716354370117,
      "kl": 4.81640625,
      "learning_rate": 3.693126051313299e-07,
      "loss": 0.3093,
      "reward": 0.5697544887661934,
      "reward_std": 0.15692155621945858,
      "rewards/accuracy_reward": 0.13839286379516125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.431361623108387,
      "step": 1364
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1513.3125915527344,
      "epoch": 0.40773653946680605,
      "grad_norm": 74.69768524169922,
      "kl": 4.98828125,
      "learning_rate": 3.6908339609799573e-07,
      "loss": 0.3093,
      "reward": 0.4866071566939354,
      "reward_std": 0.17243976891040802,
      "rewards/accuracy_reward": 0.04687500302679837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.439732164144516,
      "step": 1365
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1603.1965026855469,
      "epoch": 0.4080352475543275,
      "grad_norm": 13.175323486328125,
      "kl": 3.90625,
      "learning_rate": 3.68854057513598e-07,
      "loss": 0.2816,
      "reward": 0.5457589626312256,
      "reward_std": 0.17364555783569813,
      "rewards/accuracy_reward": 0.11607143376022577,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875149011612,
      "step": 1366
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1566.3348693847656,
      "epoch": 0.408333955641849,
      "grad_norm": 37.77369689941406,
      "kl": 4.6953125,
      "learning_rate": 3.6862458962763463e-07,
      "loss": 0.3137,
      "reward": 0.6004464626312256,
      "reward_std": 0.17737188376486301,
      "rewards/accuracy_reward": 0.17857143515720963,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750149011612,
      "step": 1367
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1624.1563110351562,
      "epoch": 0.40863266372937046,
      "grad_norm": 33.757781982421875,
      "kl": 3.4296875,
      "learning_rate": 3.6839499268974426e-07,
      "loss": 0.2491,
      "reward": 0.5145089477300644,
      "reward_std": 0.20166927576065063,
      "rewards/accuracy_reward": 0.08035714644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341518059372902,
      "step": 1368
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1564.602783203125,
      "epoch": 0.40893137181689193,
      "grad_norm": 36.540061950683594,
      "kl": 3.271484375,
      "learning_rate": 3.6816526694970576e-07,
      "loss": 0.2632,
      "reward": 0.5731026977300644,
      "reward_std": 0.15532847307622433,
      "rewards/accuracy_reward": 0.1272321492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4458705559372902,
      "step": 1369
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1544.01123046875,
      "epoch": 0.4092300799044134,
      "grad_norm": 44.578365325927734,
      "kl": 4.453125,
      "learning_rate": 3.679354126574383e-07,
      "loss": 0.2939,
      "reward": 0.5680803805589676,
      "reward_std": 0.2071404941380024,
      "rewards/accuracy_reward": 0.12276786495931447,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125223517418,
      "step": 1370
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1618.6585388183594,
      "epoch": 0.4095287879919349,
      "grad_norm": 17.439876556396484,
      "kl": 4.70703125,
      "learning_rate": 3.677054300630007e-07,
      "loss": 0.3349,
      "reward": 0.4508928805589676,
      "reward_std": 0.1469549499452114,
      "rewards/accuracy_reward": 0.017857144121080637,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357387661934,
      "step": 1371
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1464.6340026855469,
      "epoch": 0.40982749607945634,
      "grad_norm": 20.87938690185547,
      "kl": 3.578125,
      "learning_rate": 3.674753194165917e-07,
      "loss": 0.2665,
      "reward": 0.5262277126312256,
      "reward_std": 0.14989223890006542,
      "rewards/accuracy_reward": 0.06919643143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4570312649011612,
      "step": 1372
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1520.2634887695312,
      "epoch": 0.4101262041669778,
      "grad_norm": 17.004886627197266,
      "kl": 3.15234375,
      "learning_rate": 3.672450809685488e-07,
      "loss": 0.2324,
      "reward": 0.5379464477300644,
      "reward_std": 0.19398957304656506,
      "rewards/accuracy_reward": 0.1026785783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435267873108387,
      "step": 1373
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1507.2232971191406,
      "epoch": 0.4104249122544993,
      "grad_norm": 42.45034408569336,
      "kl": 3.13671875,
      "learning_rate": 3.670147149693492e-07,
      "loss": 0.2762,
      "reward": 0.478236623108387,
      "reward_std": 0.16527236253023148,
      "rewards/accuracy_reward": 0.037946431431919336,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.440290205180645,
      "step": 1374
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1580.7902221679688,
      "epoch": 0.41072362034202076,
      "grad_norm": 36.500213623046875,
      "kl": 3.3671875,
      "learning_rate": 3.667842216696084e-07,
      "loss": 0.2735,
      "reward": 0.4447544887661934,
      "reward_std": 0.1446375548839569,
      "rewards/accuracy_reward": 0.01785714365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4268973469734192,
      "step": 1375
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1545.47998046875,
      "epoch": 0.4110223284295422,
      "grad_norm": 18.709720611572266,
      "kl": 3.89453125,
      "learning_rate": 3.665536013200805e-07,
      "loss": 0.3045,
      "reward": 0.5669643133878708,
      "reward_std": 0.15088903717696667,
      "rewards/accuracy_reward": 0.12276786426082253,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.444196455180645,
      "step": 1376
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1461.0268249511719,
      "epoch": 0.4113210365170637,
      "grad_norm": 19.115894317626953,
      "kl": 4.0234375,
      "learning_rate": 3.663228541716578e-07,
      "loss": 0.3283,
      "reward": 0.5965401977300644,
      "reward_std": 0.117519935593009,
      "rewards/accuracy_reward": 0.15178571757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4447544813156128,
      "step": 1377
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1524.3147583007812,
      "epoch": 0.41161974460458517,
      "grad_norm": 33.395790100097656,
      "kl": 4.3203125,
      "learning_rate": 3.660919804753706e-07,
      "loss": 0.2971,
      "reward": 0.6082589626312256,
      "reward_std": 0.13959864526987076,
      "rewards/accuracy_reward": 0.165178582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4430803805589676,
      "step": 1378
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1515.0201721191406,
      "epoch": 0.41191845269210664,
      "grad_norm": 20.93138313293457,
      "kl": 3.77734375,
      "learning_rate": 3.6586098048238693e-07,
      "loss": 0.289,
      "reward": 0.555803619325161,
      "reward_std": 0.1417046468704939,
      "rewards/accuracy_reward": 0.10491072107106447,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4508928805589676,
      "step": 1379
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1470.96435546875,
      "epoch": 0.4122171607796281,
      "grad_norm": 11.871637344360352,
      "kl": 4.00390625,
      "learning_rate": 3.6562985444401205e-07,
      "loss": 0.2963,
      "reward": 0.4927455559372902,
      "reward_std": 0.13928680680692196,
      "rewards/accuracy_reward": 0.05580357322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419887661934,
      "step": 1380
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1398.1965026855469,
      "epoch": 0.4125158688671496,
      "grad_norm": 32.21767807006836,
      "kl": 3.21875,
      "learning_rate": 3.6539860261168854e-07,
      "loss": 0.2274,
      "reward": 0.5792411044239998,
      "reward_std": 0.16181877814233303,
      "rewards/accuracy_reward": 0.12276786426082253,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4564732387661934,
      "step": 1381
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1709.1139221191406,
      "epoch": 0.41281457695467105,
      "grad_norm": 6.037398815155029,
      "kl": 3.8046875,
      "learning_rate": 3.6516722523699566e-07,
      "loss": 0.237,
      "reward": 0.5474330559372902,
      "reward_std": 0.18723542988300323,
      "rewards/accuracy_reward": 0.11383929196745157,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937649011612,
      "step": 1382
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1517.6072082519531,
      "epoch": 0.4131132850421925,
      "grad_norm": 32.25304412841797,
      "kl": 2.845703125,
      "learning_rate": 3.649357225716493e-07,
      "loss": 0.2132,
      "reward": 0.5206473469734192,
      "reward_std": 0.14910650067031384,
      "rewards/accuracy_reward": 0.06919643399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.451450914144516,
      "step": 1383
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1499.4532165527344,
      "epoch": 0.413411993129714,
      "grad_norm": 65.30013275146484,
      "kl": 2.064453125,
      "learning_rate": 3.6470409486750166e-07,
      "loss": 0.1768,
      "reward": 0.529575914144516,
      "reward_std": 0.11234602890908718,
      "rewards/accuracy_reward": 0.08705357508733869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4425223395228386,
      "step": 1384
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1576.0625915527344,
      "epoch": 0.41371070121723547,
      "grad_norm": 56.75543975830078,
      "kl": 3.125,
      "learning_rate": 3.6447234237654104e-07,
      "loss": 0.2835,
      "reward": 0.4921875223517418,
      "reward_std": 0.16849613189697266,
      "rewards/accuracy_reward": 0.06696428824216127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232387661934,
      "step": 1385
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1465.1138916015625,
      "epoch": 0.41400940930475694,
      "grad_norm": 129.37539672851562,
      "kl": 1.8232421875,
      "learning_rate": 3.642404653508915e-07,
      "loss": 0.2061,
      "reward": 0.5412946715950966,
      "reward_std": 0.13545162789523602,
      "rewards/accuracy_reward": 0.09151786006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4497767984867096,
      "step": 1386
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1512.62060546875,
      "epoch": 0.4143081173922784,
      "grad_norm": 66.4278335571289,
      "kl": 2.07421875,
      "learning_rate": 3.640084640428123e-07,
      "loss": 0.186,
      "reward": 0.5178571790456772,
      "reward_std": 0.1457260400056839,
      "rewards/accuracy_reward": 0.06919643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4486607313156128,
      "step": 1387
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1573.3148193359375,
      "epoch": 0.4146068254797999,
      "grad_norm": 22.603727340698242,
      "kl": 2.8828125,
      "learning_rate": 3.6377633870469827e-07,
      "loss": 0.213,
      "reward": 0.5150669887661934,
      "reward_std": 0.198160320520401,
      "rewards/accuracy_reward": 0.0803571455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098395228386,
      "step": 1388
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1474.3326721191406,
      "epoch": 0.41490553356732135,
      "grad_norm": 53.31907653808594,
      "kl": 3.111328125,
      "learning_rate": 3.6354408958907906e-07,
      "loss": 0.2898,
      "reward": 0.545758955180645,
      "reward_std": 0.1775260902941227,
      "rewards/accuracy_reward": 0.1026785746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443080373108387,
      "step": 1389
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1493.77685546875,
      "epoch": 0.4152042416548428,
      "grad_norm": 22.970779418945312,
      "kl": 3.15234375,
      "learning_rate": 3.633117169486189e-07,
      "loss": 0.2451,
      "reward": 0.5245536044239998,
      "reward_std": 0.16777056455612183,
      "rewards/accuracy_reward": 0.08258928777649999,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4419643133878708,
      "step": 1390
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1475.8036499023438,
      "epoch": 0.4155029497423643,
      "grad_norm": 47.83264923095703,
      "kl": 4.796875,
      "learning_rate": 3.630792210361164e-07,
      "loss": 0.3795,
      "reward": 0.4827009290456772,
      "reward_std": 0.17200934328138828,
      "rewards/accuracy_reward": 0.04687500116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4358259066939354,
      "step": 1391
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1460.90185546875,
      "epoch": 0.41580165782988576,
      "grad_norm": 70.47198486328125,
      "kl": 5.1640625,
      "learning_rate": 3.6284660210450434e-07,
      "loss": 0.3802,
      "reward": 0.5078125298023224,
      "reward_std": 0.2051066756248474,
      "rewards/accuracy_reward": 0.07142857555299997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4363839477300644,
      "step": 1392
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1515.7813110351562,
      "epoch": 0.41610036591740723,
      "grad_norm": 19.61629867553711,
      "kl": 3.55859375,
      "learning_rate": 3.6261386040684937e-07,
      "loss": 0.261,
      "reward": 0.5502232387661934,
      "reward_std": 0.12935829907655716,
      "rewards/accuracy_reward": 0.11830357811413705,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4319196566939354,
      "step": 1393
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1641.930908203125,
      "epoch": 0.4163990740049287,
      "grad_norm": 127.1767578125,
      "kl": 6.9609375,
      "learning_rate": 3.623809961963517e-07,
      "loss": 0.4316,
      "reward": 0.5078125149011612,
      "reward_std": 0.1983569785952568,
      "rewards/accuracy_reward": 0.08482143399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 1394
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1583.2768859863281,
      "epoch": 0.4166977820924502,
      "grad_norm": 87.8060531616211,
      "kl": 6.1953125,
      "learning_rate": 3.621480097263446e-07,
      "loss": 0.3973,
      "reward": 0.4430803805589676,
      "reward_std": 0.16278668493032455,
      "rewards/accuracy_reward": 0.020089286845177412,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 1395
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1688.4308471679688,
      "epoch": 0.41699649017997165,
      "grad_norm": 149.83041381835938,
      "kl": 7.5703125,
      "learning_rate": 3.619149012502946e-07,
      "loss": 0.4537,
      "reward": 0.446986623108387,
      "reward_std": 0.15790134854614735,
      "rewards/accuracy_reward": 0.029017857974395156,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4179687723517418,
      "step": 1396
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1560.6808776855469,
      "epoch": 0.4172951982674931,
      "grad_norm": 52.01384735107422,
      "kl": 5.109375,
      "learning_rate": 3.6168167102180095e-07,
      "loss": 0.3328,
      "reward": 0.4464285895228386,
      "reward_std": 0.1442829705774784,
      "rewards/accuracy_reward": 0.01785714295692742,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 1397
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1514.9911193847656,
      "epoch": 0.4175939063550146,
      "grad_norm": 25.073022842407227,
      "kl": 5.4375,
      "learning_rate": 3.6144831929459513e-07,
      "loss": 0.4076,
      "reward": 0.4397321715950966,
      "reward_std": 0.13864734955132008,
      "rewards/accuracy_reward": 0.01562500116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.424107164144516,
      "step": 1398
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1513.9420776367188,
      "epoch": 0.41789261444253606,
      "grad_norm": 11.512999534606934,
      "kl": 3.98828125,
      "learning_rate": 3.612148463225411e-07,
      "loss": 0.3118,
      "reward": 0.5111607313156128,
      "reward_std": 0.16364446468651295,
      "rewards/accuracy_reward": 0.08258928917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 1399
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1446.2500915527344,
      "epoch": 0.4181913225300575,
      "grad_norm": 58.58030319213867,
      "kl": 3.15625,
      "learning_rate": 3.6098125235963445e-07,
      "loss": 0.2997,
      "reward": 0.4927455559372902,
      "reward_std": 0.17848804593086243,
      "rewards/accuracy_reward": 0.04910714481957257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4436384066939354,
      "step": 1400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1577.99560546875,
      "epoch": 0.41849003061757895,
      "grad_norm": 13.345547676086426,
      "kl": 3.90625,
      "learning_rate": 3.607475376600026e-07,
      "loss": 0.2612,
      "reward": 0.447544664144516,
      "reward_std": 0.1498772781342268,
      "rewards/accuracy_reward": 0.024553572526201606,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910969734192,
      "step": 1401
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1500.4866943359375,
      "epoch": 0.4187887387051004,
      "grad_norm": 33.27885055541992,
      "kl": 3.2578125,
      "learning_rate": 3.605137024779041e-07,
      "loss": 0.26,
      "reward": 0.5284598469734192,
      "reward_std": 0.1894755121320486,
      "rewards/accuracy_reward": 0.08482143399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443638414144516,
      "step": 1402
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1587.5022888183594,
      "epoch": 0.4190874467926219,
      "grad_norm": 13.882495880126953,
      "kl": 4.27734375,
      "learning_rate": 3.6027974706772894e-07,
      "loss": 0.3148,
      "reward": 0.4765625298023224,
      "reward_std": 0.145012766122818,
      "rewards/accuracy_reward": 0.05133928847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232313156128,
      "step": 1403
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1583.1384582519531,
      "epoch": 0.41938615488014336,
      "grad_norm": 8.958839416503906,
      "kl": 3.548828125,
      "learning_rate": 3.6004567168399755e-07,
      "loss": 0.2409,
      "reward": 0.4933035969734192,
      "reward_std": 0.18743673339486122,
      "rewards/accuracy_reward": 0.0647321455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 1404
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1565.4420166015625,
      "epoch": 0.41968486296766483,
      "grad_norm": 13.716802597045898,
      "kl": 3.80859375,
      "learning_rate": 3.598114765813609e-07,
      "loss": 0.31,
      "reward": 0.5083705633878708,
      "reward_std": 0.2209208831191063,
      "rewards/accuracy_reward": 0.08928571920841932,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4190848469734192,
      "step": 1405
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1511.0313110351562,
      "epoch": 0.4199835710551863,
      "grad_norm": 19.03940773010254,
      "kl": 3.71484375,
      "learning_rate": 3.595771620146005e-07,
      "loss": 0.2765,
      "reward": 0.5446428880095482,
      "reward_std": 0.1843728944659233,
      "rewards/accuracy_reward": 0.12053572339937091,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.424107164144516,
      "step": 1406
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1569.7969665527344,
      "epoch": 0.42028227914270777,
      "grad_norm": 7.5712409019470215,
      "kl": 4.15625,
      "learning_rate": 3.593427282386275e-07,
      "loss": 0.2899,
      "reward": 0.5200893133878708,
      "reward_std": 0.18085487559437752,
      "rewards/accuracy_reward": 0.0937500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263393133878708,
      "step": 1407
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1628.9063110351562,
      "epoch": 0.42058098723022924,
      "grad_norm": 73.74468994140625,
      "kl": 5.9765625,
      "learning_rate": 3.591081755084829e-07,
      "loss": 0.3905,
      "reward": 0.4497768059372902,
      "reward_std": 0.1802276372909546,
      "rewards/accuracy_reward": 0.06919643399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3805803805589676,
      "step": 1408
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1577.8348999023438,
      "epoch": 0.4208796953177507,
      "grad_norm": 19.72395133972168,
      "kl": 4.12109375,
      "learning_rate": 3.5887350407933707e-07,
      "loss": 0.2968,
      "reward": 0.526227705180645,
      "reward_std": 0.17388656735420227,
      "rewards/accuracy_reward": 0.0892857201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419887661934,
      "step": 1409
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1554.0893249511719,
      "epoch": 0.4211784034052722,
      "grad_norm": 28.011688232421875,
      "kl": 4.30078125,
      "learning_rate": 3.586387142064896e-07,
      "loss": 0.3206,
      "reward": 0.5005580559372902,
      "reward_std": 0.17357107624411583,
      "rewards/accuracy_reward": 0.07589285937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4246651977300644,
      "step": 1410
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1537.9107971191406,
      "epoch": 0.42147711149279365,
      "grad_norm": 111.5328140258789,
      "kl": 2.62890625,
      "learning_rate": 3.584038061453689e-07,
      "loss": 0.2643,
      "reward": 0.5055803880095482,
      "reward_std": 0.12599072977900505,
      "rewards/accuracy_reward": 0.0758928619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875298023224,
      "step": 1411
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1563.2634582519531,
      "epoch": 0.4217758195803151,
      "grad_norm": 94.41519927978516,
      "kl": 2.80859375,
      "learning_rate": 3.5816878015153166e-07,
      "loss": 0.2511,
      "reward": 0.5329241380095482,
      "reward_std": 0.19719773903489113,
      "rewards/accuracy_reward": 0.12053572060540318,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.412388414144516,
      "step": 1412
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1588.7076721191406,
      "epoch": 0.4220745276678366,
      "grad_norm": 55.93451690673828,
      "kl": 2.51171875,
      "learning_rate": 3.579336364806634e-07,
      "loss": 0.1938,
      "reward": 0.5373884066939354,
      "reward_std": 0.22761493548750877,
      "rewards/accuracy_reward": 0.12053571874275804,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4168526902794838,
      "step": 1413
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1509.8728332519531,
      "epoch": 0.42237323575535807,
      "grad_norm": 12.809821128845215,
      "kl": 4.27734375,
      "learning_rate": 3.576983753885773e-07,
      "loss": 0.3024,
      "reward": 0.4380580559372902,
      "reward_std": 0.14709274284541607,
      "rewards/accuracy_reward": 0.01562500116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330559372902,
      "step": 1414
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1384.6340026855469,
      "epoch": 0.42267194384287954,
      "grad_norm": 54.9464225769043,
      "kl": 2.83203125,
      "learning_rate": 3.574629971312143e-07,
      "loss": 0.274,
      "reward": 0.4927455633878708,
      "reward_std": 0.1536429561674595,
      "rewards/accuracy_reward": 0.0357142873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4570312649011612,
      "step": 1415
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1531.2969665527344,
      "epoch": 0.422970651930401,
      "grad_norm": 23.288984298706055,
      "kl": 4.8671875,
      "learning_rate": 3.572275019646429e-07,
      "loss": 0.4149,
      "reward": 0.5329241305589676,
      "reward_std": 0.12793505750596523,
      "rewards/accuracy_reward": 0.10937500488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.423549123108387,
      "step": 1416
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1622.8996276855469,
      "epoch": 0.4232693600179225,
      "grad_norm": 84.50669860839844,
      "kl": 6.5703125,
      "learning_rate": 3.5699189014505884e-07,
      "loss": 0.435,
      "reward": 0.576450914144516,
      "reward_std": 0.1688869632780552,
      "rewards/accuracy_reward": 0.1718750037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.404575914144516,
      "step": 1417
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1502.6563415527344,
      "epoch": 0.42356806810544395,
      "grad_norm": 29.186649322509766,
      "kl": 4.86328125,
      "learning_rate": 3.567561619287846e-07,
      "loss": 0.3552,
      "reward": 0.4380580484867096,
      "reward_std": 0.11341207846999168,
      "rewards/accuracy_reward": 0.004464285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937649011612,
      "step": 1418
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1519.5089721679688,
      "epoch": 0.4238667761929654,
      "grad_norm": 96.34249114990234,
      "kl": 6.21875,
      "learning_rate": 3.5652031757226944e-07,
      "loss": 0.4184,
      "reward": 0.5139509215950966,
      "reward_std": 0.20502140372991562,
      "rewards/accuracy_reward": 0.09598214528523386,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4179687723517418,
      "step": 1419
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1493.4085693359375,
      "epoch": 0.4241654842804869,
      "grad_norm": 29.343441009521484,
      "kl": 4.21875,
      "learning_rate": 3.5628435733208884e-07,
      "loss": 0.3108,
      "reward": 0.4810268133878708,
      "reward_std": 0.14347251504659653,
      "rewards/accuracy_reward": 0.04910714412108064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.431919664144516,
      "step": 1420
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1583.85498046875,
      "epoch": 0.42446419236800836,
      "grad_norm": 50.71596908569336,
      "kl": 5.875,
      "learning_rate": 3.5604828146494446e-07,
      "loss": 0.4257,
      "reward": 0.487723246216774,
      "reward_std": 0.17060846835374832,
      "rewards/accuracy_reward": 0.0781250037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4095982387661934,
      "step": 1421
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1570.9866943359375,
      "epoch": 0.42476290045552983,
      "grad_norm": 14.927299499511719,
      "kl": 3.6953125,
      "learning_rate": 3.5581209022766373e-07,
      "loss": 0.2568,
      "reward": 0.5027901902794838,
      "reward_std": 0.223589938133955,
      "rewards/accuracy_reward": 0.08482143213041127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4179687649011612,
      "step": 1422
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1520.9197082519531,
      "epoch": 0.4250616085430513,
      "grad_norm": 11.268389701843262,
      "kl": 3.80859375,
      "learning_rate": 3.555757838771995e-07,
      "loss": 0.3062,
      "reward": 0.4966517984867096,
      "reward_std": 0.16206291317939758,
      "rewards/accuracy_reward": 0.06696428824216127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875149011612,
      "step": 1423
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1560.7277526855469,
      "epoch": 0.4253603166305728,
      "grad_norm": 32.90434646606445,
      "kl": 4.36328125,
      "learning_rate": 3.5533936267063e-07,
      "loss": 0.3028,
      "reward": 0.5390625223517418,
      "reward_std": 0.18615295737981796,
      "rewards/accuracy_reward": 0.12276786006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4162946715950966,
      "step": 1424
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1510.4866943359375,
      "epoch": 0.42565902471809425,
      "grad_norm": 44.8250617980957,
      "kl": 2.921875,
      "learning_rate": 3.5510282686515827e-07,
      "loss": 0.2537,
      "reward": 0.5552455708384514,
      "reward_std": 0.15971957333385944,
      "rewards/accuracy_reward": 0.1183035783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419813156128,
      "step": 1425
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1488.1250610351562,
      "epoch": 0.4259577328056157,
      "grad_norm": 49.80710983276367,
      "kl": 2.84765625,
      "learning_rate": 3.54866176718112e-07,
      "loss": 0.265,
      "reward": 0.4804687723517418,
      "reward_std": 0.1348081212490797,
      "rewards/accuracy_reward": 0.049107146449387074,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4313616305589676,
      "step": 1426
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1542.5848999023438,
      "epoch": 0.4262564408931372,
      "grad_norm": 31.625080108642578,
      "kl": 3.41796875,
      "learning_rate": 3.546294124869435e-07,
      "loss": 0.2733,
      "reward": 0.5625000298023224,
      "reward_std": 0.2448401041328907,
      "rewards/accuracy_reward": 0.13839286006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.424107164144516,
      "step": 1427
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1634.5960693359375,
      "epoch": 0.42655514898065866,
      "grad_norm": 15.517450332641602,
      "kl": 3.88671875,
      "learning_rate": 3.54392534429229e-07,
      "loss": 0.2862,
      "reward": 0.5044643133878708,
      "reward_std": 0.2133963294327259,
      "rewards/accuracy_reward": 0.09821429220028222,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500223517418,
      "step": 1428
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1473.6875610351562,
      "epoch": 0.42685385706818013,
      "grad_norm": 43.29099655151367,
      "kl": 2.48046875,
      "learning_rate": 3.5415554280266866e-07,
      "loss": 0.2151,
      "reward": 0.5892857387661934,
      "reward_std": 0.19006782211363316,
      "rewards/accuracy_reward": 0.13839286426082253,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4508928805589676,
      "step": 1429
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1550.7098693847656,
      "epoch": 0.4271525651557016,
      "grad_norm": 19.029016494750977,
      "kl": 3.65234375,
      "learning_rate": 3.53918437865086e-07,
      "loss": 0.2727,
      "reward": 0.4603794813156128,
      "reward_std": 0.1681793536990881,
      "rewards/accuracy_reward": 0.035714287078008056,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4246651977300644,
      "step": 1430
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1637.0603332519531,
      "epoch": 0.4274512732432231,
      "grad_norm": 10.773706436157227,
      "kl": 4.984375,
      "learning_rate": 3.536812198744281e-07,
      "loss": 0.3636,
      "reward": 0.4776785895228386,
      "reward_std": 0.16874925792217255,
      "rewards/accuracy_reward": 0.06473214668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4129464402794838,
      "step": 1431
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1546.6607971191406,
      "epoch": 0.42774998133074454,
      "grad_norm": 8.724166870117188,
      "kl": 4.21484375,
      "learning_rate": 3.5344388908876486e-07,
      "loss": 0.3301,
      "reward": 0.4877232387661934,
      "reward_std": 0.166711013764143,
      "rewards/accuracy_reward": 0.0602678619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.427455373108387,
      "step": 1432
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1499.1340026855469,
      "epoch": 0.428048689418266,
      "grad_norm": 76.31558227539062,
      "kl": 5.46484375,
      "learning_rate": 3.5320644576628884e-07,
      "loss": 0.3667,
      "reward": 0.527343787252903,
      "reward_std": 0.14706859178841114,
      "rewards/accuracy_reward": 0.09598214598372579,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4313616305589676,
      "step": 1433
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1613.0447387695312,
      "epoch": 0.4283473975057875,
      "grad_norm": 23.628870010375977,
      "kl": 3.7578125,
      "learning_rate": 3.5296889016531517e-07,
      "loss": 0.2757,
      "reward": 0.4654017984867096,
      "reward_std": 0.13195237703621387,
      "rewards/accuracy_reward": 0.044642860535532236,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4207589402794838,
      "step": 1434
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1527.8482971191406,
      "epoch": 0.42864610559330896,
      "grad_norm": 11.513998031616211,
      "kl": 4.36328125,
      "learning_rate": 3.5273122254428103e-07,
      "loss": 0.3325,
      "reward": 0.514508955180645,
      "reward_std": 0.14873602241277695,
      "rewards/accuracy_reward": 0.1004464291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625149011612,
      "step": 1435
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1513.5692749023438,
      "epoch": 0.42894481368083043,
      "grad_norm": 85.2144775390625,
      "kl": 6.234375,
      "learning_rate": 3.524934431617455e-07,
      "loss": 0.4462,
      "reward": 0.5256696715950966,
      "reward_std": 0.18492629751563072,
      "rewards/accuracy_reward": 0.09151786309666932,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341518059372902,
      "step": 1436
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1447.3929138183594,
      "epoch": 0.4292435217683519,
      "grad_norm": 13.547998428344727,
      "kl": 3.53515625,
      "learning_rate": 3.522555522763892e-07,
      "loss": 0.2963,
      "reward": 0.5474330633878708,
      "reward_std": 0.18129126355051994,
      "rewards/accuracy_reward": 0.1026785783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4447544887661934,
      "step": 1437
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1547.4933471679688,
      "epoch": 0.42954222985587337,
      "grad_norm": 59.071956634521484,
      "kl": 5.3046875,
      "learning_rate": 3.5201755014701417e-07,
      "loss": 0.3525,
      "reward": 0.490513414144516,
      "reward_std": 0.1838302668184042,
      "rewards/accuracy_reward": 0.064732147147879,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812723517418,
      "step": 1438
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1557.091552734375,
      "epoch": 0.42984093794339484,
      "grad_norm": 31.595563888549805,
      "kl": 5.5703125,
      "learning_rate": 3.5177943703254344e-07,
      "loss": 0.3958,
      "reward": 0.5652901902794838,
      "reward_std": 0.13521074689924717,
      "rewards/accuracy_reward": 0.1517857201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4135044813156128,
      "step": 1439
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1499.9420166015625,
      "epoch": 0.4301396460309163,
      "grad_norm": 32.997589111328125,
      "kl": 4.14453125,
      "learning_rate": 3.5154121319202056e-07,
      "loss": 0.3236,
      "reward": 0.519531287252903,
      "reward_std": 0.16492615081369877,
      "rewards/accuracy_reward": 0.08258929033763707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419887661934,
      "step": 1440
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1500.37060546875,
      "epoch": 0.4304383541184378,
      "grad_norm": 94.89285278320312,
      "kl": 5.6328125,
      "learning_rate": 3.5130287888460996e-07,
      "loss": 0.3785,
      "reward": 0.518415205180645,
      "reward_std": 0.17342890799045563,
      "rewards/accuracy_reward": 0.08258928917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435825914144516,
      "step": 1441
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1469.9442749023438,
      "epoch": 0.43073706220595925,
      "grad_norm": 40.93423843383789,
      "kl": 5.12890625,
      "learning_rate": 3.510644343695958e-07,
      "loss": 0.3724,
      "reward": 0.475446455180645,
      "reward_std": 0.12870310619473457,
      "rewards/accuracy_reward": 0.044642859138548374,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035895228386,
      "step": 1442
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1567.0603332519531,
      "epoch": 0.43103577029348067,
      "grad_norm": 81.92736053466797,
      "kl": 5.6484375,
      "learning_rate": 3.508258799063825e-07,
      "loss": 0.3536,
      "reward": 0.4386160969734192,
      "reward_std": 0.12087966687977314,
      "rewards/accuracy_reward": 0.011160714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4274553805589676,
      "step": 1443
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1543.9554138183594,
      "epoch": 0.43133447838100214,
      "grad_norm": 45.483177185058594,
      "kl": 3.30078125,
      "learning_rate": 3.505872157544937e-07,
      "loss": 0.2656,
      "reward": 0.4497768059372902,
      "reward_std": 0.15735751204192638,
      "rewards/accuracy_reward": 0.0334821455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.416294664144516,
      "step": 1444
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1445.2567443847656,
      "epoch": 0.4316331864685236,
      "grad_norm": 21.5874080657959,
      "kl": 3.9453125,
      "learning_rate": 3.5034844217357296e-07,
      "loss": 0.2762,
      "reward": 0.4849330559372902,
      "reward_std": 0.15366273932158947,
      "rewards/accuracy_reward": 0.0468750037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4380580559372902,
      "step": 1445
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1499.8661193847656,
      "epoch": 0.4319318945560451,
      "grad_norm": 9.277941703796387,
      "kl": 3.66015625,
      "learning_rate": 3.5010955942338226e-07,
      "loss": 0.2954,
      "reward": 0.5212053880095482,
      "reward_std": 0.15784416906535625,
      "rewards/accuracy_reward": 0.1071428619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625223517418,
      "step": 1446
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1587.1741943359375,
      "epoch": 0.43223060264356655,
      "grad_norm": 63.69154357910156,
      "kl": 2.8203125,
      "learning_rate": 3.4987056776380263e-07,
      "loss": 0.2343,
      "reward": 0.5078125149011612,
      "reward_std": 0.1770716942846775,
      "rewards/accuracy_reward": 0.08482143236324191,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.422991082072258,
      "step": 1447
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1441.5491943359375,
      "epoch": 0.432529310731088,
      "grad_norm": 54.58761978149414,
      "kl": 2.1875,
      "learning_rate": 3.4963146745483373e-07,
      "loss": 0.1728,
      "reward": 0.5167410969734192,
      "reward_std": 0.1260305643081665,
      "rewards/accuracy_reward": 0.07142857578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125223517418,
      "step": 1448
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1462.5291137695312,
      "epoch": 0.4328280188186095,
      "grad_norm": 57.36056900024414,
      "kl": 2.681640625,
      "learning_rate": 3.493922587565932e-07,
      "loss": 0.242,
      "reward": 0.5636160969734192,
      "reward_std": 0.1607311051338911,
      "rewards/accuracy_reward": 0.1339285783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875223517418,
      "step": 1449
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1555.2098693847656,
      "epoch": 0.43312672690613097,
      "grad_norm": 75.47383880615234,
      "kl": 3.25390625,
      "learning_rate": 3.4915294192931653e-07,
      "loss": 0.2861,
      "reward": 0.5552455484867096,
      "reward_std": 0.15638377144932747,
      "rewards/accuracy_reward": 0.12946429080329835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812649011612,
      "step": 1450
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1499.2232666015625,
      "epoch": 0.43342543499365244,
      "grad_norm": 55.58460235595703,
      "kl": 2.71875,
      "learning_rate": 3.489135172333571e-07,
      "loss": 0.229,
      "reward": 0.5318080633878708,
      "reward_std": 0.15975451469421387,
      "rewards/accuracy_reward": 0.09821429220028222,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937649011612,
      "step": 1451
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1398.4442443847656,
      "epoch": 0.4337241430811739,
      "grad_norm": 77.27457427978516,
      "kl": 2.486328125,
      "learning_rate": 3.486739849291855e-07,
      "loss": 0.2463,
      "reward": 0.6523437798023224,
      "reward_std": 0.1756640113890171,
      "rewards/accuracy_reward": 0.2098214365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4425223395228386,
      "step": 1452
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1590.3884582519531,
      "epoch": 0.4340228511686954,
      "grad_norm": 43.664154052734375,
      "kl": 3.078125,
      "learning_rate": 3.4843434527738946e-07,
      "loss": 0.2385,
      "reward": 0.4497768059372902,
      "reward_std": 0.1464109607040882,
      "rewards/accuracy_reward": 0.0200892873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875149011612,
      "step": 1453
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1522.0670166015625,
      "epoch": 0.43432155925621685,
      "grad_norm": 35.21329116821289,
      "kl": 3.52734375,
      "learning_rate": 3.4819459853867325e-07,
      "loss": 0.2855,
      "reward": 0.490513414144516,
      "reward_std": 0.15257912501692772,
      "rewards/accuracy_reward": 0.058035716181620955,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776977300644,
      "step": 1454
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1490.2902221679688,
      "epoch": 0.4346202673437383,
      "grad_norm": 53.89543914794922,
      "kl": 3.3984375,
      "learning_rate": 3.47954744973858e-07,
      "loss": 0.2765,
      "reward": 0.4642857313156128,
      "reward_std": 0.1521309707313776,
      "rewards/accuracy_reward": 0.033482144586741924,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035895228386,
      "step": 1455
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1535.40185546875,
      "epoch": 0.4349189754312598,
      "grad_norm": 10.348383903503418,
      "kl": 3.4375,
      "learning_rate": 3.477147848438808e-07,
      "loss": 0.2616,
      "reward": 0.5117187798023224,
      "reward_std": 0.21689007431268692,
      "rewards/accuracy_reward": 0.0691964328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4425223395228386,
      "step": 1456
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1605.7724304199219,
      "epoch": 0.43521768351878126,
      "grad_norm": 12.209020614624023,
      "kl": 4.419921875,
      "learning_rate": 3.4747471840979474e-07,
      "loss": 0.3037,
      "reward": 0.521205373108387,
      "reward_std": 0.14623479172587395,
      "rewards/accuracy_reward": 0.0848214365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4363839402794838,
      "step": 1457
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1509.3013916015625,
      "epoch": 0.43551639160630273,
      "grad_norm": 57.3026123046875,
      "kl": 5.6484375,
      "learning_rate": 3.472345459327686e-07,
      "loss": 0.4028,
      "reward": 0.5909598469734192,
      "reward_std": 0.20330269634723663,
      "rewards/accuracy_reward": 0.1607142947614193,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455559372902,
      "step": 1458
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1494.9219360351562,
      "epoch": 0.4358150996938242,
      "grad_norm": 26.43115234375,
      "kl": 4.28125,
      "learning_rate": 3.469942676740866e-07,
      "loss": 0.3275,
      "reward": 0.537946455180645,
      "reward_std": 0.14555110223591328,
      "rewards/accuracy_reward": 0.09821429150179029,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.439732164144516,
      "step": 1459
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1504.8594360351562,
      "epoch": 0.4361138077813457,
      "grad_norm": 42.313411712646484,
      "kl": 4.2890625,
      "learning_rate": 3.467538838951477e-07,
      "loss": 0.2896,
      "reward": 0.4732143133878708,
      "reward_std": 0.15532775782048702,
      "rewards/accuracy_reward": 0.029017858672887087,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.444196455180645,
      "step": 1460
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1538.9598999023438,
      "epoch": 0.43641251586886715,
      "grad_norm": 26.194128036499023,
      "kl": 4.3671875,
      "learning_rate": 3.4651339485746604e-07,
      "loss": 0.3041,
      "reward": 0.509486623108387,
      "reward_std": 0.14994021505117416,
      "rewards/accuracy_reward": 0.06473214598372579,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.444754496216774,
      "step": 1461
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1608.1161193847656,
      "epoch": 0.4367112239563886,
      "grad_norm": 7.320902347564697,
      "kl": 4.19140625,
      "learning_rate": 3.462728008226702e-07,
      "loss": 0.303,
      "reward": 0.4732143059372902,
      "reward_std": 0.17594783008098602,
      "rewards/accuracy_reward": 0.042410716880112886,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035969734192,
      "step": 1462
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1653.4063110351562,
      "epoch": 0.4370099320439101,
      "grad_norm": 20.456615447998047,
      "kl": 3.9375,
      "learning_rate": 3.4603210205250274e-07,
      "loss": 0.2577,
      "reward": 0.497209832072258,
      "reward_std": 0.20526098273694515,
      "rewards/accuracy_reward": 0.06473214365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.432477705180645,
      "step": 1463
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1597.2768859863281,
      "epoch": 0.43730864013143156,
      "grad_norm": 53.321109771728516,
      "kl": 4.92578125,
      "learning_rate": 3.457912988088204e-07,
      "loss": 0.3355,
      "reward": 0.4654017984867096,
      "reward_std": 0.14293274469673634,
      "rewards/accuracy_reward": 0.033482145285233855,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4319196566939354,
      "step": 1464
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1526.5715026855469,
      "epoch": 0.43760734821895303,
      "grad_norm": 40.586669921875,
      "kl": 4.89453125,
      "learning_rate": 3.455503913535935e-07,
      "loss": 0.3583,
      "reward": 0.4787946790456772,
      "reward_std": 0.1226488221436739,
      "rewards/accuracy_reward": 0.04464285937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341518059372902,
      "step": 1465
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1609.5893859863281,
      "epoch": 0.4379060563064745,
      "grad_norm": 68.1153793334961,
      "kl": 5.1015625,
      "learning_rate": 3.453093799489058e-07,
      "loss": 0.3132,
      "reward": 0.4274553805589676,
      "reward_std": 0.12730297818779945,
      "rewards/accuracy_reward": 0.01116071455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.416294664144516,
      "step": 1466
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1517.0916137695312,
      "epoch": 0.43820476439399597,
      "grad_norm": 34.04472351074219,
      "kl": 3.26953125,
      "learning_rate": 3.45068264856954e-07,
      "loss": 0.2822,
      "reward": 0.486607164144516,
      "reward_std": 0.12591844610869884,
      "rewards/accuracy_reward": 0.05133928940631449,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4352678805589676,
      "step": 1467
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1498.4509582519531,
      "epoch": 0.43850347248151744,
      "grad_norm": 60.066123962402344,
      "kl": 2.5625,
      "learning_rate": 3.4482704634004776e-07,
      "loss": 0.2427,
      "reward": 0.5502232313156128,
      "reward_std": 0.15972594544291496,
      "rewards/accuracy_reward": 0.10491072107106447,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125223517418,
      "step": 1468
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1657.2277526855469,
      "epoch": 0.4388021805690389,
      "grad_norm": 8.72428035736084,
      "kl": 3.78515625,
      "learning_rate": 3.445857246606091e-07,
      "loss": 0.2448,
      "reward": 0.4414062649011612,
      "reward_std": 0.13980548828840256,
      "rewards/accuracy_reward": 0.013392857741564512,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4280134066939354,
      "step": 1469
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1545.3438110351562,
      "epoch": 0.4391008886565604,
      "grad_norm": 11.99938678741455,
      "kl": 3.26953125,
      "learning_rate": 3.4434430008117244e-07,
      "loss": 0.2451,
      "reward": 0.5083705559372902,
      "reward_std": 0.1546725481748581,
      "rewards/accuracy_reward": 0.06473214505240321,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443638414144516,
      "step": 1470
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1526.4420471191406,
      "epoch": 0.43939959674408186,
      "grad_norm": 55.323673248291016,
      "kl": 2.5546875,
      "learning_rate": 3.4410277286438406e-07,
      "loss": 0.2262,
      "reward": 0.569196455180645,
      "reward_std": 0.14800458028912544,
      "rewards/accuracy_reward": 0.12946429196745157,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.439732164144516,
      "step": 1471
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1448.294677734375,
      "epoch": 0.4396983048316033,
      "grad_norm": 24.32415008544922,
      "kl": 3.083984375,
      "learning_rate": 3.438611432730018e-07,
      "loss": 0.257,
      "reward": 0.4893973469734192,
      "reward_std": 0.14233149029314518,
      "rewards/accuracy_reward": 0.058035716880112886,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4313616380095482,
      "step": 1472
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1589.7590026855469,
      "epoch": 0.4399970129191248,
      "grad_norm": 8.666537284851074,
      "kl": 3.47265625,
      "learning_rate": 3.436194115698951e-07,
      "loss": 0.2496,
      "reward": 0.498325914144516,
      "reward_std": 0.11551776342093945,
      "rewards/accuracy_reward": 0.0781250037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4202009066939354,
      "step": 1473
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1503.8348999023438,
      "epoch": 0.44029572100664627,
      "grad_norm": 43.407936096191406,
      "kl": 2.63671875,
      "learning_rate": 3.433775780180443e-07,
      "loss": 0.2349,
      "reward": 0.5078125298023224,
      "reward_std": 0.12676135264337063,
      "rewards/accuracy_reward": 0.05357143213041127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4542410969734192,
      "step": 1474
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1487.1496276855469,
      "epoch": 0.44059442909416774,
      "grad_norm": 6.639986038208008,
      "kl": 3.345703125,
      "learning_rate": 3.4313564288054057e-07,
      "loss": 0.2639,
      "reward": 0.577008955180645,
      "reward_std": 0.1256884355098009,
      "rewards/accuracy_reward": 0.1250000058207661,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.452008955180645,
      "step": 1475
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1568.7165832519531,
      "epoch": 0.4408931371816892,
      "grad_norm": 18.400861740112305,
      "kl": 3.572265625,
      "learning_rate": 3.428936064205856e-07,
      "loss": 0.2642,
      "reward": 0.5658482313156128,
      "reward_std": 0.2613684870302677,
      "rewards/accuracy_reward": 0.129464291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4363839477300644,
      "step": 1476
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1639.6898193359375,
      "epoch": 0.4411918452692107,
      "grad_norm": 58.76015090942383,
      "kl": 5.453125,
      "learning_rate": 3.426514689014915e-07,
      "loss": 0.3518,
      "reward": 0.5251116380095482,
      "reward_std": 0.1464933268725872,
      "rewards/accuracy_reward": 0.08705357322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4380580559372902,
      "step": 1477
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1561.919677734375,
      "epoch": 0.44149055335673215,
      "grad_norm": 72.22064971923828,
      "kl": 5.54296875,
      "learning_rate": 3.4240923058668e-07,
      "loss": 0.332,
      "reward": 0.4910714477300644,
      "reward_std": 0.11900698393583298,
      "rewards/accuracy_reward": 0.06250000302679837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 1478
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1569.6116638183594,
      "epoch": 0.4417892614442536,
      "grad_norm": 56.408321380615234,
      "kl": 5.47265625,
      "learning_rate": 3.4216689173968274e-07,
      "loss": 0.3496,
      "reward": 0.483258955180645,
      "reward_std": 0.14187972992658615,
      "rewards/accuracy_reward": 0.05580357555299997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4274553805589676,
      "step": 1479
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1566.555908203125,
      "epoch": 0.4420879695317751,
      "grad_norm": 12.119752883911133,
      "kl": 3.984375,
      "learning_rate": 3.4192445262414055e-07,
      "loss": 0.2823,
      "reward": 0.5747768059372902,
      "reward_std": 0.14305485226213932,
      "rewards/accuracy_reward": 0.13616072130389512,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4386160969734192,
      "step": 1480
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1520.4330749511719,
      "epoch": 0.44238667761929656,
      "grad_norm": 22.189239501953125,
      "kl": 3.9765625,
      "learning_rate": 3.4168191350380353e-07,
      "loss": 0.2782,
      "reward": 0.497767873108387,
      "reward_std": 0.1384769007563591,
      "rewards/accuracy_reward": 0.0647321455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357313156128,
      "step": 1481
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1538.6161499023438,
      "epoch": 0.44268538570681804,
      "grad_norm": 34.014244079589844,
      "kl": 4.52734375,
      "learning_rate": 3.414392746425304e-07,
      "loss": 0.3381,
      "reward": 0.5457589477300644,
      "reward_std": 0.13536109775304794,
      "rewards/accuracy_reward": 0.1205357201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232313156128,
      "step": 1482
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1558.8103637695312,
      "epoch": 0.4429840937943395,
      "grad_norm": 82.58380889892578,
      "kl": 2.3828125,
      "learning_rate": 3.4119653630428837e-07,
      "loss": 0.2231,
      "reward": 0.4938616305589676,
      "reward_std": 0.20607616938650608,
      "rewards/accuracy_reward": 0.06473214365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294887661934,
      "step": 1483
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1595.9063110351562,
      "epoch": 0.443282801881861,
      "grad_norm": 71.35625457763672,
      "kl": 2.099609375,
      "learning_rate": 3.409536987531532e-07,
      "loss": 0.1807,
      "reward": 0.5117187723517418,
      "reward_std": 0.14415345899760723,
      "rewards/accuracy_reward": 0.07589286006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435825914144516,
      "step": 1484
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1463.3259887695312,
      "epoch": 0.44358150996938245,
      "grad_norm": 99.4144515991211,
      "kl": 2.330078125,
      "learning_rate": 3.407107622533081e-07,
      "loss": 0.2604,
      "reward": 0.4972098469734192,
      "reward_std": 0.16033474542200565,
      "rewards/accuracy_reward": 0.06473214575089514,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776977300644,
      "step": 1485
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1539.9621276855469,
      "epoch": 0.44388021805690386,
      "grad_norm": 74.43984985351562,
      "kl": 2.060546875,
      "learning_rate": 3.404677270690443e-07,
      "loss": 0.1973,
      "reward": 0.5848214477300644,
      "reward_std": 0.1568812057375908,
      "rewards/accuracy_reward": 0.1517857201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357387661934,
      "step": 1486
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1640.8572082519531,
      "epoch": 0.44417892614442533,
      "grad_norm": 37.137569427490234,
      "kl": 2.84765625,
      "learning_rate": 3.402245934647603e-07,
      "loss": 0.2019,
      "reward": 0.518415205180645,
      "reward_std": 0.18236207589507103,
      "rewards/accuracy_reward": 0.1026785783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4157366305589676,
      "step": 1487
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1533.2389221191406,
      "epoch": 0.4444776342319468,
      "grad_norm": 11.866114616394043,
      "kl": 4.046875,
      "learning_rate": 3.399813617049615e-07,
      "loss": 0.2837,
      "reward": 0.547991082072258,
      "reward_std": 0.16181799583137035,
      "rewards/accuracy_reward": 0.1071428619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4408482313156128,
      "step": 1488
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1586.3147888183594,
      "epoch": 0.4447763423194683,
      "grad_norm": 8.2020902633667,
      "kl": 3.83984375,
      "learning_rate": 3.397380320542603e-07,
      "loss": 0.2906,
      "reward": 0.4659598395228386,
      "reward_std": 0.13205212727189064,
      "rewards/accuracy_reward": 0.029017859371379018,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419813156128,
      "step": 1489
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1571.9844360351562,
      "epoch": 0.44507505040698975,
      "grad_norm": 128.61541748046875,
      "kl": 7.0625,
      "learning_rate": 3.394946047773755e-07,
      "loss": 0.4516,
      "reward": 0.5005580633878708,
      "reward_std": 0.14292092248797417,
      "rewards/accuracy_reward": 0.08258929080329835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4179687798023224,
      "step": 1490
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1519.2924499511719,
      "epoch": 0.4453737584945112,
      "grad_norm": 45.22102737426758,
      "kl": 4.16015625,
      "learning_rate": 3.3925108013913217e-07,
      "loss": 0.2896,
      "reward": 0.5111607313156128,
      "reward_std": 0.16807224228978157,
      "rewards/accuracy_reward": 0.08258929057046771,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 1491
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1505.8482360839844,
      "epoch": 0.4456724665820327,
      "grad_norm": 38.634185791015625,
      "kl": 3.82421875,
      "learning_rate": 3.390074584044612e-07,
      "loss": 0.2456,
      "reward": 0.467633955180645,
      "reward_std": 0.13928714580833912,
      "rewards/accuracy_reward": 0.03348214481957257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341518059372902,
      "step": 1492
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1439.7679443359375,
      "epoch": 0.44597117466955416,
      "grad_norm": 15.070361137390137,
      "kl": 3.1875,
      "learning_rate": 3.3876373983839904e-07,
      "loss": 0.2673,
      "reward": 0.5390625149011612,
      "reward_std": 0.1723210271447897,
      "rewards/accuracy_reward": 0.09151786053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.447544664144516,
      "step": 1493
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1623.99560546875,
      "epoch": 0.44626988275707563,
      "grad_norm": 38.99897384643555,
      "kl": 4.4140625,
      "learning_rate": 3.385199247060878e-07,
      "loss": 0.2821,
      "reward": 0.447544664144516,
      "reward_std": 0.14903432875871658,
      "rewards/accuracy_reward": 0.022321430034935474,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232387661934,
      "step": 1494
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1688.185302734375,
      "epoch": 0.4465685908445971,
      "grad_norm": 22.86096954345703,
      "kl": 4.47265625,
      "learning_rate": 3.382760132727744e-07,
      "loss": 0.2946,
      "reward": 0.5033482387661934,
      "reward_std": 0.20057121478021145,
      "rewards/accuracy_reward": 0.08705357392318547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.416294664144516,
      "step": 1495
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1551.3348999023438,
      "epoch": 0.4468672989321186,
      "grad_norm": 38.5744743347168,
      "kl": 4.9453125,
      "learning_rate": 3.380320058038105e-07,
      "loss": 0.3182,
      "reward": 0.509486623108387,
      "reward_std": 0.14495181292295456,
      "rewards/accuracy_reward": 0.0915178619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4179687649011612,
      "step": 1496
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1547.85498046875,
      "epoch": 0.44716600701964004,
      "grad_norm": 32.32548522949219,
      "kl": 3.8125,
      "learning_rate": 3.377879025646523e-07,
      "loss": 0.305,
      "reward": 0.4464285895228386,
      "reward_std": 0.13425000198185444,
      "rewards/accuracy_reward": 0.02008928661234677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263393059372902,
      "step": 1497
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1646.5670166015625,
      "epoch": 0.4474647151071615,
      "grad_norm": 15.072153091430664,
      "kl": 3.32421875,
      "learning_rate": 3.375437038208603e-07,
      "loss": 0.2123,
      "reward": 0.491629496216774,
      "reward_std": 0.160063149407506,
      "rewards/accuracy_reward": 0.06919643096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330559372902,
      "step": 1498
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1623.4241943359375,
      "epoch": 0.447763423194683,
      "grad_norm": 5.24835205078125,
      "kl": 3.67578125,
      "learning_rate": 3.372994098380987e-07,
      "loss": 0.2491,
      "reward": 0.5027901977300644,
      "reward_std": 0.16960198618471622,
      "rewards/accuracy_reward": 0.07589286100119352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4268973395228386,
      "step": 1499
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1423.0603332519531,
      "epoch": 0.44806213128220446,
      "grad_norm": 19.80891227722168,
      "kl": 3.4453125,
      "learning_rate": 3.370550208821355e-07,
      "loss": 0.2943,
      "reward": 0.542410746216774,
      "reward_std": 0.18154528737068176,
      "rewards/accuracy_reward": 0.1004464328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4419643059372902,
      "step": 1500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1470.6898193359375,
      "epoch": 0.4483608393697259,
      "grad_norm": 29.724716186523438,
      "kl": 3.375,
      "learning_rate": 3.368105372188419e-07,
      "loss": 0.2923,
      "reward": 0.6149553805589676,
      "reward_std": 0.1389238890260458,
      "rewards/accuracy_reward": 0.16741072246804833,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4475446715950966,
      "step": 1501
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1558.1875915527344,
      "epoch": 0.4486595474572474,
      "grad_norm": 6.352499961853027,
      "kl": 3.0234375,
      "learning_rate": 3.3656595911419217e-07,
      "loss": 0.2082,
      "reward": 0.4860491380095482,
      "reward_std": 0.18849169462919235,
      "rewards/accuracy_reward": 0.058035718044266105,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4280134215950966,
      "step": 1502
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1565.8036499023438,
      "epoch": 0.44895825554476887,
      "grad_norm": 18.030637741088867,
      "kl": 4.5,
      "learning_rate": 3.3632128683426323e-07,
      "loss": 0.3112,
      "reward": 0.4592634066939354,
      "reward_std": 0.15346484631299973,
      "rewards/accuracy_reward": 0.026785715948790312,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776902794838,
      "step": 1503
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1601.0759582519531,
      "epoch": 0.44925696363229034,
      "grad_norm": 43.94953155517578,
      "kl": 5.046875,
      "learning_rate": 3.360765206452346e-07,
      "loss": 0.3307,
      "reward": 0.4525669887661934,
      "reward_std": 0.14980255626142025,
      "rewards/accuracy_reward": 0.022321430034935474,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455559372902,
      "step": 1504
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1543.4889221191406,
      "epoch": 0.4495556717198118,
      "grad_norm": 30.042516708374023,
      "kl": 4.8359375,
      "learning_rate": 3.35831660813388e-07,
      "loss": 0.3344,
      "reward": 0.5161830633878708,
      "reward_std": 0.1402545627206564,
      "rewards/accuracy_reward": 0.0848214291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4313616305589676,
      "step": 1505
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1676.6049499511719,
      "epoch": 0.4498543798073333,
      "grad_norm": 42.993568420410156,
      "kl": 3.1953125,
      "learning_rate": 3.355867076051068e-07,
      "loss": 0.2286,
      "reward": 0.4927455633878708,
      "reward_std": 0.1646582894027233,
      "rewards/accuracy_reward": 0.07812500419095159,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205559372902,
      "step": 1506
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1582.7590026855469,
      "epoch": 0.45015308789485475,
      "grad_norm": 13.065696716308594,
      "kl": 4.1015625,
      "learning_rate": 3.353416612868761e-07,
      "loss": 0.2979,
      "reward": 0.4949776902794838,
      "reward_std": 0.16566613502800465,
      "rewards/accuracy_reward": 0.06473214505240321,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455484867096,
      "step": 1507
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1593.8572387695312,
      "epoch": 0.4504517959823762,
      "grad_norm": 36.64884948730469,
      "kl": 3.48828125,
      "learning_rate": 3.350965221252824e-07,
      "loss": 0.2778,
      "reward": 0.548549123108387,
      "reward_std": 0.14168071933090687,
      "rewards/accuracy_reward": 0.10491071944124997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443638414144516,
      "step": 1508
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1591.6027526855469,
      "epoch": 0.4507505040698977,
      "grad_norm": 36.84761428833008,
      "kl": 4.8046875,
      "learning_rate": 3.348512903870129e-07,
      "loss": 0.3053,
      "reward": 0.4497767984867096,
      "reward_std": 0.1509563997387886,
      "rewards/accuracy_reward": 0.022321430034935474,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4274553805589676,
      "step": 1509
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1587.0179138183594,
      "epoch": 0.45104921215741917,
      "grad_norm": 30.43524169921875,
      "kl": 3.6328125,
      "learning_rate": 3.346059663388558e-07,
      "loss": 0.2904,
      "reward": 0.544642873108387,
      "reward_std": 0.13620512187480927,
      "rewards/accuracy_reward": 0.11607143143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 1510
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1455.8616943359375,
      "epoch": 0.45134792024494064,
      "grad_norm": 33.104095458984375,
      "kl": 2.546875,
      "learning_rate": 3.3436055024769956e-07,
      "loss": 0.215,
      "reward": 0.4609375223517418,
      "reward_std": 0.10308028198778629,
      "rewards/accuracy_reward": 0.008928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4520089477300644,
      "step": 1511
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1600.2656860351562,
      "epoch": 0.4516466283324621,
      "grad_norm": 23.384510040283203,
      "kl": 4.92578125,
      "learning_rate": 3.341150423805329e-07,
      "loss": 0.3487,
      "reward": 0.4988839626312256,
      "reward_std": 0.1812890563160181,
      "rewards/accuracy_reward": 0.0803571455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268059372902,
      "step": 1512
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1417.7567749023438,
      "epoch": 0.4519453364199836,
      "grad_norm": 27.088300704956055,
      "kl": 2.89453125,
      "learning_rate": 3.3386944300444433e-07,
      "loss": 0.2482,
      "reward": 0.6875000447034836,
      "reward_std": 0.18422217667102814,
      "rewards/accuracy_reward": 0.2366071492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4508928805589676,
      "step": 1513
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1489.8304443359375,
      "epoch": 0.45224404450750505,
      "grad_norm": 9.002484321594238,
      "kl": 3.859375,
      "learning_rate": 3.336237523866218e-07,
      "loss": 0.2967,
      "reward": 0.6060268133878708,
      "reward_std": 0.1347727421671152,
      "rewards/accuracy_reward": 0.16964286286383867,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4363839477300644,
      "step": 1514
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1484.7634582519531,
      "epoch": 0.4525427525950265,
      "grad_norm": 27.8223876953125,
      "kl": 3.33203125,
      "learning_rate": 3.333779707943527e-07,
      "loss": 0.2867,
      "reward": 0.5011160895228386,
      "reward_std": 0.1420827228575945,
      "rewards/accuracy_reward": 0.058035716880112886,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443080373108387,
      "step": 1515
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1449.5201416015625,
      "epoch": 0.452841460682548,
      "grad_norm": 52.28901672363281,
      "kl": 2.767578125,
      "learning_rate": 3.331320984950233e-07,
      "loss": 0.2475,
      "reward": 0.5719866305589676,
      "reward_std": 0.1677960641682148,
      "rewards/accuracy_reward": 0.12946429662406445,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4425223395228386,
      "step": 1516
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1625.49560546875,
      "epoch": 0.45314016877006946,
      "grad_norm": 60.30420684814453,
      "kl": 6.2890625,
      "learning_rate": 3.328861357561186e-07,
      "loss": 0.4202,
      "reward": 0.4910714477300644,
      "reward_std": 0.13211988098919392,
      "rewards/accuracy_reward": 0.07366071757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4174107313156128,
      "step": 1517
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1580.3616638183594,
      "epoch": 0.45343887685759093,
      "grad_norm": 9.959226608276367,
      "kl": 3.390625,
      "learning_rate": 3.326400828452218e-07,
      "loss": 0.2469,
      "reward": 0.5703125223517418,
      "reward_std": 0.14995422959327698,
      "rewards/accuracy_reward": 0.13839286309666932,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.431919664144516,
      "step": 1518
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1618.0402526855469,
      "epoch": 0.4537375849451124,
      "grad_norm": 15.71210765838623,
      "kl": 4.37109375,
      "learning_rate": 3.3239394003001453e-07,
      "loss": 0.3111,
      "reward": 0.4386160895228386,
      "reward_std": 0.15605469979345798,
      "rewards/accuracy_reward": 0.02455357206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625223517418,
      "step": 1519
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1550.0982971191406,
      "epoch": 0.4540362930326339,
      "grad_norm": 53.627159118652344,
      "kl": 4.71484375,
      "learning_rate": 3.32147707578276e-07,
      "loss": 0.3368,
      "reward": 0.4765625223517418,
      "reward_std": 0.15795458853244781,
      "rewards/accuracy_reward": 0.042410714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341518059372902,
      "step": 1520
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1599.47998046875,
      "epoch": 0.45433500112015535,
      "grad_norm": 92.64781951904297,
      "kl": 5.8515625,
      "learning_rate": 3.3190138575788294e-07,
      "loss": 0.3588,
      "reward": 0.4285714477300644,
      "reward_std": 0.1163560189306736,
      "rewards/accuracy_reward": 0.0066964291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 1521
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1469.4152221679688,
      "epoch": 0.4546337092076768,
      "grad_norm": 38.57954406738281,
      "kl": 3.0703125,
      "learning_rate": 3.316549748368094e-07,
      "loss": 0.2453,
      "reward": 0.4938616305589676,
      "reward_std": 0.13310185261070728,
      "rewards/accuracy_reward": 0.053571431431919336,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4402901977300644,
      "step": 1522
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1576.7790832519531,
      "epoch": 0.4549324172951983,
      "grad_norm": 37.82828903198242,
      "kl": 5.26171875,
      "learning_rate": 3.314084750831263e-07,
      "loss": 0.3584,
      "reward": 0.4720982387661934,
      "reward_std": 0.16154007986187935,
      "rewards/accuracy_reward": 0.04241071501746774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875298023224,
      "step": 1523
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1543.19873046875,
      "epoch": 0.45523112538271976,
      "grad_norm": 72.25894165039062,
      "kl": 5.796875,
      "learning_rate": 3.3116188676500116e-07,
      "loss": 0.4166,
      "reward": 0.528459832072258,
      "reward_std": 0.16235975921154022,
      "rewards/accuracy_reward": 0.1049107201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4235491156578064,
      "step": 1524
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1505.185302734375,
      "epoch": 0.45552983347024123,
      "grad_norm": 18.44721031188965,
      "kl": 4.01953125,
      "learning_rate": 3.309152101506979e-07,
      "loss": 0.2914,
      "reward": 0.4994419887661934,
      "reward_std": 0.12928788736462593,
      "rewards/accuracy_reward": 0.0647321455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098469734192,
      "step": 1525
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1553.6607971191406,
      "epoch": 0.4558285415577627,
      "grad_norm": 35.04893493652344,
      "kl": 2.89453125,
      "learning_rate": 3.306684455085766e-07,
      "loss": 0.2387,
      "reward": 0.5279018208384514,
      "reward_std": 0.1604275368154049,
      "rewards/accuracy_reward": 0.07589286286383867,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4520089477300644,
      "step": 1526
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1548.3014221191406,
      "epoch": 0.45612724964528417,
      "grad_norm": 11.50898551940918,
      "kl": 4.09765625,
      "learning_rate": 3.3042159310709294e-07,
      "loss": 0.3116,
      "reward": 0.6177455559372902,
      "reward_std": 0.159631185233593,
      "rewards/accuracy_reward": 0.18526786309666932,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776902794838,
      "step": 1527
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1595.4130249023438,
      "epoch": 0.45642595773280564,
      "grad_norm": 18.10394287109375,
      "kl": 3.9296875,
      "learning_rate": 3.3017465321479807e-07,
      "loss": 0.3052,
      "reward": 0.5133928805589676,
      "reward_std": 0.19171272590756416,
      "rewards/accuracy_reward": 0.08705357811413705,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263392984867096,
      "step": 1528
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1528.7857666015625,
      "epoch": 0.45672466582032706,
      "grad_norm": 29.602020263671875,
      "kl": 3.06640625,
      "learning_rate": 3.299276261003386e-07,
      "loss": 0.2645,
      "reward": 0.494977705180645,
      "reward_std": 0.19291643798351288,
      "rewards/accuracy_reward": 0.051339288242161274,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443638414144516,
      "step": 1529
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1509.0514221191406,
      "epoch": 0.45702337390784853,
      "grad_norm": 10.063702583312988,
      "kl": 2.4765625,
      "learning_rate": 3.2968051203245563e-07,
      "loss": 0.1769,
      "reward": 0.6032366305589676,
      "reward_std": 0.21050818637013435,
      "rewards/accuracy_reward": 0.1517857201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4514509066939354,
      "step": 1530
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1581.0714721679688,
      "epoch": 0.45732208199537,
      "grad_norm": 30.84069061279297,
      "kl": 3.8046875,
      "learning_rate": 3.294333112799851e-07,
      "loss": 0.2768,
      "reward": 0.4726562574505806,
      "reward_std": 0.12849678099155426,
      "rewards/accuracy_reward": 0.04687500232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812649011612,
      "step": 1531
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1515.9844360351562,
      "epoch": 0.45762079008289147,
      "grad_norm": 11.016000747680664,
      "kl": 3.521484375,
      "learning_rate": 3.291860241118571e-07,
      "loss": 0.2573,
      "reward": 0.6132812798023224,
      "reward_std": 0.19825753197073936,
      "rewards/accuracy_reward": 0.1808035783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.432477705180645,
      "step": 1532
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1682.74560546875,
      "epoch": 0.45791949817041294,
      "grad_norm": 22.07780647277832,
      "kl": 3.6015625,
      "learning_rate": 3.28938650797096e-07,
      "loss": 0.2411,
      "reward": 0.4570312798023224,
      "reward_std": 0.13189779594540596,
      "rewards/accuracy_reward": 0.04464285937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.412388414144516,
      "step": 1533
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1622.18310546875,
      "epoch": 0.4582182062579344,
      "grad_norm": 5.965703964233398,
      "kl": 3.65625,
      "learning_rate": 3.286911916048197e-07,
      "loss": 0.2527,
      "reward": 0.4793526902794838,
      "reward_std": 0.1735621690750122,
      "rewards/accuracy_reward": 0.04687500302679837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.432477705180645,
      "step": 1534
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1618.3572082519531,
      "epoch": 0.4585169143454559,
      "grad_norm": 72.41387176513672,
      "kl": 4.8203125,
      "learning_rate": 3.284436468042394e-07,
      "loss": 0.3124,
      "reward": 0.5680803805589676,
      "reward_std": 0.15009262412786484,
      "rewards/accuracy_reward": 0.1361607201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.431919664144516,
      "step": 1535
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1566.0513916015625,
      "epoch": 0.45881562243297735,
      "grad_norm": 6.137658596038818,
      "kl": 4.25390625,
      "learning_rate": 3.2819601666465975e-07,
      "loss": 0.307,
      "reward": 0.5340401902794838,
      "reward_std": 0.1548993308097124,
      "rewards/accuracy_reward": 0.10044643189758062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937723517418,
      "step": 1536
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1534.4554443359375,
      "epoch": 0.4591143305204988,
      "grad_norm": 54.062076568603516,
      "kl": 4.828125,
      "learning_rate": 3.2794830145547804e-07,
      "loss": 0.3313,
      "reward": 0.5401785895228386,
      "reward_std": 0.16778460703790188,
      "rewards/accuracy_reward": 0.10714286379516125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357313156128,
      "step": 1537
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1638.7254943847656,
      "epoch": 0.4594130386080203,
      "grad_norm": 89.19023132324219,
      "kl": 5.52734375,
      "learning_rate": 3.2770050144618405e-07,
      "loss": 0.334,
      "reward": 0.553571455180645,
      "reward_std": 0.1346146371215582,
      "rewards/accuracy_reward": 0.1339285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.419642873108387,
      "step": 1538
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1447.7433776855469,
      "epoch": 0.45971174669554177,
      "grad_norm": 12.270461082458496,
      "kl": 3.30859375,
      "learning_rate": 3.2745261690635987e-07,
      "loss": 0.2763,
      "reward": 0.5664062798023224,
      "reward_std": 0.16433368995785713,
      "rewards/accuracy_reward": 0.11607143632136285,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4503348469734192,
      "step": 1539
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1611.8148193359375,
      "epoch": 0.46001045478306324,
      "grad_norm": 11.946927070617676,
      "kl": 3.517578125,
      "learning_rate": 3.2720464810567985e-07,
      "loss": 0.2456,
      "reward": 0.4743303880095482,
      "reward_std": 0.16173738799989223,
      "rewards/accuracy_reward": 0.033482145285233855,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4408482238650322,
      "step": 1540
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1504.0357666015625,
      "epoch": 0.4603091628705847,
      "grad_norm": 19.580068588256836,
      "kl": 3.3203125,
      "learning_rate": 3.269565953139094e-07,
      "loss": 0.2606,
      "reward": 0.522321455180645,
      "reward_std": 0.17548412643373013,
      "rewards/accuracy_reward": 0.08482143585570157,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000223517418,
      "step": 1541
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1650.96435546875,
      "epoch": 0.4606078709581062,
      "grad_norm": 28.742706298828125,
      "kl": 4.46484375,
      "learning_rate": 3.2670845880090587e-07,
      "loss": 0.2845,
      "reward": 0.5133928880095482,
      "reward_std": 0.14286575093865395,
      "rewards/accuracy_reward": 0.09598214668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4174107387661934,
      "step": 1542
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1719.8438415527344,
      "epoch": 0.46090657904562765,
      "grad_norm": 40.35120391845703,
      "kl": 5.37109375,
      "learning_rate": 3.264602388366174e-07,
      "loss": 0.3464,
      "reward": 0.4051339402794838,
      "reward_std": 0.1628190465271473,
      "rewards/accuracy_reward": 0.0133928582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3917410895228386,
      "step": 1543
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1552.8527221679688,
      "epoch": 0.4612052871331491,
      "grad_norm": 89.93563842773438,
      "kl": 1.576171875,
      "learning_rate": 3.26211935691083e-07,
      "loss": 0.1509,
      "reward": 0.5440848544239998,
      "reward_std": 0.1459367722272873,
      "rewards/accuracy_reward": 0.09375000116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4503348544239998,
      "step": 1544
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1517.7723999023438,
      "epoch": 0.4615039952206706,
      "grad_norm": 85.09063720703125,
      "kl": 2.30859375,
      "learning_rate": 3.259635496344321e-07,
      "loss": 0.2216,
      "reward": 0.5558036044239998,
      "reward_std": 0.1861688680946827,
      "rewards/accuracy_reward": 0.12053572200238705,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435267873108387,
      "step": 1545
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1596.4263916015625,
      "epoch": 0.46180270330819206,
      "grad_norm": 34.276458740234375,
      "kl": 3.40625,
      "learning_rate": 3.2571508093688443e-07,
      "loss": 0.2693,
      "reward": 0.458705373108387,
      "reward_std": 0.14789941906929016,
      "rewards/accuracy_reward": 0.024553571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341518059372902,
      "step": 1546
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1615.2925109863281,
      "epoch": 0.46210141139571353,
      "grad_norm": 33.636871337890625,
      "kl": 3.41796875,
      "learning_rate": 3.254665298687496e-07,
      "loss": 0.2709,
      "reward": 0.4375000149011612,
      "reward_std": 0.13667511753737926,
      "rewards/accuracy_reward": 0.008928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714402794838,
      "step": 1547
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1665.727783203125,
      "epoch": 0.462400119483235,
      "grad_norm": 17.28813362121582,
      "kl": 4.625,
      "learning_rate": 3.2521789670042683e-07,
      "loss": 0.2981,
      "reward": 0.5340401902794838,
      "reward_std": 0.18080026656389236,
      "rewards/accuracy_reward": 0.1227678619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4112723469734192,
      "step": 1548
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1516.3594360351562,
      "epoch": 0.4626988275707565,
      "grad_norm": 22.431278228759766,
      "kl": 4.3359375,
      "learning_rate": 3.249691817024046e-07,
      "loss": 0.2993,
      "reward": 0.4960937723517418,
      "reward_std": 0.1388443373143673,
      "rewards/accuracy_reward": 0.06919643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4268973395228386,
      "step": 1549
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1526.3348999023438,
      "epoch": 0.46299753565827795,
      "grad_norm": 30.365924835205078,
      "kl": 4.6171875,
      "learning_rate": 3.2472038514526036e-07,
      "loss": 0.3297,
      "reward": 0.494419664144516,
      "reward_std": 0.15363386273384094,
      "rewards/accuracy_reward": 0.06473214668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875223517418,
      "step": 1550
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1559.0045471191406,
      "epoch": 0.4632962437457994,
      "grad_norm": 38.4730224609375,
      "kl": 2.8125,
      "learning_rate": 3.244715072996605e-07,
      "loss": 0.2349,
      "reward": 0.5323661044239998,
      "reward_std": 0.12168741971254349,
      "rewards/accuracy_reward": 0.0848214328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.447544664144516,
      "step": 1551
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1571.024658203125,
      "epoch": 0.4635949518333209,
      "grad_norm": 10.339950561523438,
      "kl": 3.9140625,
      "learning_rate": 3.242225484363597e-07,
      "loss": 0.3001,
      "reward": 0.5385044887661934,
      "reward_std": 0.199076347053051,
      "rewards/accuracy_reward": 0.10491071990691125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937723517418,
      "step": 1552
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1556.0134582519531,
      "epoch": 0.46389365992084236,
      "grad_norm": 9.494263648986816,
      "kl": 3.9296875,
      "learning_rate": 3.2397350882620073e-07,
      "loss": 0.3112,
      "reward": 0.4838169887661934,
      "reward_std": 0.13662613928318024,
      "rewards/accuracy_reward": 0.0491071455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098395228386,
      "step": 1553
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1632.8192749023438,
      "epoch": 0.46419236800836383,
      "grad_norm": 10.194806098937988,
      "kl": 4.4453125,
      "learning_rate": 3.2372438874011444e-07,
      "loss": 0.3105,
      "reward": 0.4598214477300644,
      "reward_std": 0.12361835315823555,
      "rewards/accuracy_reward": 0.0424107164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4174107313156128,
      "step": 1554
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1631.2791137695312,
      "epoch": 0.4644910760958853,
      "grad_norm": 5.9457268714904785,
      "kl": 4.29296875,
      "learning_rate": 3.2347518844911894e-07,
      "loss": 0.3065,
      "reward": 0.494977705180645,
      "reward_std": 0.1569997202605009,
      "rewards/accuracy_reward": 0.06473214668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455559372902,
      "step": 1555
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1629.4732971191406,
      "epoch": 0.4647897841834068,
      "grad_norm": 56.91993713378906,
      "kl": 4.5078125,
      "learning_rate": 3.2322590822431973e-07,
      "loss": 0.2911,
      "reward": 0.5837053954601288,
      "reward_std": 0.23714406602084637,
      "rewards/accuracy_reward": 0.15625000558793545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.427455373108387,
      "step": 1556
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1543.5692749023438,
      "epoch": 0.46508849227092824,
      "grad_norm": 29.340484619140625,
      "kl": 4.2109375,
      "learning_rate": 3.2297654833690923e-07,
      "loss": 0.2816,
      "reward": 0.5083705559372902,
      "reward_std": 0.14019900187849998,
      "rewards/accuracy_reward": 0.07589286239817739,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776977300644,
      "step": 1557
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1581.0335693359375,
      "epoch": 0.4653872003584497,
      "grad_norm": 90.05815887451172,
      "kl": 2.0546875,
      "learning_rate": 3.227271090581667e-07,
      "loss": 0.1945,
      "reward": 0.5758928880095482,
      "reward_std": 0.17292364686727524,
      "rewards/accuracy_reward": 0.13169643515720963,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.444196455180645,
      "step": 1558
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1599.9576721191406,
      "epoch": 0.4656859084459712,
      "grad_norm": 65.6618881225586,
      "kl": 2.81640625,
      "learning_rate": 3.2247759065945745e-07,
      "loss": 0.2279,
      "reward": 0.4419643059372902,
      "reward_std": 0.16049774549901485,
      "rewards/accuracy_reward": 0.020089287078008056,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 1559
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1693.5536804199219,
      "epoch": 0.46598461653349266,
      "grad_norm": 37.3591423034668,
      "kl": 4.9921875,
      "learning_rate": 3.222279934122332e-07,
      "loss": 0.3103,
      "reward": 0.4642857313156128,
      "reward_std": 0.1592205073684454,
      "rewards/accuracy_reward": 0.058035716880112886,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500149011612,
      "step": 1560
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1608.9241638183594,
      "epoch": 0.46628332462101413,
      "grad_norm": 61.330169677734375,
      "kl": 2.08203125,
      "learning_rate": 3.219783175880312e-07,
      "loss": 0.1713,
      "reward": 0.5217634290456772,
      "reward_std": 0.1701500378549099,
      "rewards/accuracy_reward": 0.08482143515720963,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419813156128,
      "step": 1561
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1619.2232666015625,
      "epoch": 0.4665820327085356,
      "grad_norm": 48.30647277832031,
      "kl": 2.8671875,
      "learning_rate": 3.217285634584743e-07,
      "loss": 0.2203,
      "reward": 0.4732143059372902,
      "reward_std": 0.17660940252244473,
      "rewards/accuracy_reward": 0.04017857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357387661934,
      "step": 1562
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1475.6384582519531,
      "epoch": 0.46688074079605707,
      "grad_norm": 42.27071762084961,
      "kl": 2.3984375,
      "learning_rate": 3.2147873129527054e-07,
      "loss": 0.1991,
      "reward": 0.5552455633878708,
      "reward_std": 0.18660282716155052,
      "rewards/accuracy_reward": 0.10937500558793545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4458705559372902,
      "step": 1563
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1567.2545471191406,
      "epoch": 0.46717944888357854,
      "grad_norm": 62.49799346923828,
      "kl": 4.78125,
      "learning_rate": 3.2122882137021273e-07,
      "loss": 0.3108,
      "reward": 0.470982164144516,
      "reward_std": 0.18207499384880066,
      "rewards/accuracy_reward": 0.04241071501746774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 1564
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1621.69873046875,
      "epoch": 0.4674781569711,
      "grad_norm": 61.61653137207031,
      "kl": 5.3984375,
      "learning_rate": 3.209788339551785e-07,
      "loss": 0.3547,
      "reward": 0.5156250223517418,
      "reward_std": 0.15292499586939812,
      "rewards/accuracy_reward": 0.10044643492437899,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4151785969734192,
      "step": 1565
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1650.9040832519531,
      "epoch": 0.4677768650586215,
      "grad_norm": 103.44009399414062,
      "kl": 5.984375,
      "learning_rate": 3.207287693221296e-07,
      "loss": 0.3605,
      "reward": 0.479352705180645,
      "reward_std": 0.1632378101348877,
      "rewards/accuracy_reward": 0.060267859837040305,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4190848469734192,
      "step": 1566
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1393.1496276855469,
      "epoch": 0.46807557314614295,
      "grad_norm": 38.79689407348633,
      "kl": 4.21875,
      "learning_rate": 3.2047862774311187e-07,
      "loss": 0.3088,
      "reward": 0.5390625298023224,
      "reward_std": 0.18461522832512856,
      "rewards/accuracy_reward": 0.0937500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125298023224,
      "step": 1567
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1518.2344360351562,
      "epoch": 0.4683742812336644,
      "grad_norm": 33.23543930053711,
      "kl": 3.984375,
      "learning_rate": 3.202284094902547e-07,
      "loss": 0.2903,
      "reward": 0.6383928805589676,
      "reward_std": 0.16594479978084564,
      "rewards/accuracy_reward": 0.19196429592557251,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4464285895228386,
      "step": 1568
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1589.930908203125,
      "epoch": 0.4686729893211859,
      "grad_norm": 25.382930755615234,
      "kl": 4.2109375,
      "learning_rate": 3.199781148357712e-07,
      "loss": 0.3079,
      "reward": 0.5027901977300644,
      "reward_std": 0.22245344147086143,
      "rewards/accuracy_reward": 0.08258928847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.420200914144516,
      "step": 1569
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1608.415283203125,
      "epoch": 0.46897169740870737,
      "grad_norm": 19.218530654907227,
      "kl": 3.4140625,
      "learning_rate": 3.1972774405195736e-07,
      "loss": 0.2359,
      "reward": 0.5251116454601288,
      "reward_std": 0.17153823748230934,
      "rewards/accuracy_reward": 0.09151786309666932,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937723517418,
      "step": 1570
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1629.10498046875,
      "epoch": 0.46927040549622884,
      "grad_norm": 34.38093185424805,
      "kl": 3.37890625,
      "learning_rate": 3.19477297411192e-07,
      "loss": 0.2601,
      "reward": 0.5418527126312256,
      "reward_std": 0.1840256005525589,
      "rewards/accuracy_reward": 0.11830357578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4235491305589676,
      "step": 1571
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1644.6473999023438,
      "epoch": 0.46956911358375025,
      "grad_norm": 31.99071502685547,
      "kl": 3.2109375,
      "learning_rate": 3.192267751859366e-07,
      "loss": 0.2251,
      "reward": 0.4681919887661934,
      "reward_std": 0.11340091750025749,
      "rewards/accuracy_reward": 0.04017857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4280134066939354,
      "step": 1572
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1619.4263916015625,
      "epoch": 0.4698678216712717,
      "grad_norm": 44.48149871826172,
      "kl": 3.2265625,
      "learning_rate": 3.1897617764873477e-07,
      "loss": 0.2344,
      "reward": 0.5279018059372902,
      "reward_std": 0.1346139870584011,
      "rewards/accuracy_reward": 0.08705357392318547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4408482313156128,
      "step": 1573
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1505.6206359863281,
      "epoch": 0.4701665297587932,
      "grad_norm": 31.10768699645996,
      "kl": 2.376953125,
      "learning_rate": 3.187255050722121e-07,
      "loss": 0.2023,
      "reward": 0.6116071790456772,
      "reward_std": 0.14574355073273182,
      "rewards/accuracy_reward": 0.16517857648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4464285895228386,
      "step": 1574
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1603.7388916015625,
      "epoch": 0.47046523784631467,
      "grad_norm": 23.50874137878418,
      "kl": 2.97265625,
      "learning_rate": 3.184747577290756e-07,
      "loss": 0.2236,
      "reward": 0.5027901977300644,
      "reward_std": 0.11774946935474873,
      "rewards/accuracy_reward": 0.05357143096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4492187723517418,
      "step": 1575
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1535.8661499023438,
      "epoch": 0.47076394593383614,
      "grad_norm": 10.750513076782227,
      "kl": 2.921875,
      "learning_rate": 3.18223935892114e-07,
      "loss": 0.2114,
      "reward": 0.5312500223517418,
      "reward_std": 0.1793605498969555,
      "rewards/accuracy_reward": 0.08705357322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4441964477300644,
      "step": 1576
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1622.2723999023438,
      "epoch": 0.4710626540213576,
      "grad_norm": 9.507172584533691,
      "kl": 4.21875,
      "learning_rate": 3.1797303983419684e-07,
      "loss": 0.2863,
      "reward": 0.4888393059372902,
      "reward_std": 0.12348124012351036,
      "rewards/accuracy_reward": 0.06250000116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263393133878708,
      "step": 1577
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1501.0447082519531,
      "epoch": 0.4713613621088791,
      "grad_norm": 17.04880714416504,
      "kl": 3.32421875,
      "learning_rate": 3.177220698282743e-07,
      "loss": 0.2475,
      "reward": 0.5206473544239998,
      "reward_std": 0.1659452598541975,
      "rewards/accuracy_reward": 0.08482143213041127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4358259215950966,
      "step": 1578
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1623.4844665527344,
      "epoch": 0.47166007019640055,
      "grad_norm": 19.65870475769043,
      "kl": 4.984375,
      "learning_rate": 3.1747102614737734e-07,
      "loss": 0.3338,
      "reward": 0.4648437798023224,
      "reward_std": 0.10936222970485687,
      "rewards/accuracy_reward": 0.0357142873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294887661934,
      "step": 1579
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1555.9085388183594,
      "epoch": 0.471958778283922,
      "grad_norm": 36.362430572509766,
      "kl": 2.65625,
      "learning_rate": 3.172199090646169e-07,
      "loss": 0.2174,
      "reward": 0.5513393059372902,
      "reward_std": 0.1377736497670412,
      "rewards/accuracy_reward": 0.09821429150179029,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4531250223517418,
      "step": 1580
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1509.3147888183594,
      "epoch": 0.4722574863714435,
      "grad_norm": 14.71524429321289,
      "kl": 3.98046875,
      "learning_rate": 3.1696871885318354e-07,
      "loss": 0.2729,
      "reward": 0.5301339626312256,
      "reward_std": 0.14458046853542328,
      "rewards/accuracy_reward": 0.0892857201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4408482313156128,
      "step": 1581
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1516.2054443359375,
      "epoch": 0.47255619445896496,
      "grad_norm": 98.69776916503906,
      "kl": 4.51171875,
      "learning_rate": 3.1671745578634767e-07,
      "loss": 0.3097,
      "reward": 0.5904017984867096,
      "reward_std": 0.14577055722475052,
      "rewards/accuracy_reward": 0.15401786286383867,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4363839477300644,
      "step": 1582
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1521.0313110351562,
      "epoch": 0.47285490254648643,
      "grad_norm": 12.08185863494873,
      "kl": 3.98828125,
      "learning_rate": 3.16466120137459e-07,
      "loss": 0.3266,
      "reward": 0.559709832072258,
      "reward_std": 0.1821522656828165,
      "rewards/accuracy_reward": 0.12276786286383867,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419813156128,
      "step": 1583
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1546.9040832519531,
      "epoch": 0.4731536106340079,
      "grad_norm": 10.836796760559082,
      "kl": 4.203125,
      "learning_rate": 3.16214712179946e-07,
      "loss": 0.3069,
      "reward": 0.4977678880095482,
      "reward_std": 0.13999264873564243,
      "rewards/accuracy_reward": 0.058035716880112886,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4397321715950966,
      "step": 1584
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1560.3795471191406,
      "epoch": 0.4734523187215294,
      "grad_norm": 32.49421691894531,
      "kl": 3.6953125,
      "learning_rate": 3.1596323218731583e-07,
      "loss": 0.3059,
      "reward": 0.5546875298023224,
      "reward_std": 0.15854356437921524,
      "rewards/accuracy_reward": 0.12723214854486287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.427455373108387,
      "step": 1585
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1468.0915832519531,
      "epoch": 0.47375102680905085,
      "grad_norm": 16.733299255371094,
      "kl": 3.361328125,
      "learning_rate": 3.157116804331542e-07,
      "loss": 0.2518,
      "reward": 0.585937537252903,
      "reward_std": 0.1736029703170061,
      "rewards/accuracy_reward": 0.14508929592557251,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4408482387661934,
      "step": 1586
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1500.790283203125,
      "epoch": 0.4740497348965723,
      "grad_norm": 25.521772384643555,
      "kl": 3.15625,
      "learning_rate": 3.1546005719112466e-07,
      "loss": 0.264,
      "reward": 0.5379464626312256,
      "reward_std": 0.11564843729138374,
      "rewards/accuracy_reward": 0.08928571757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4486607313156128,
      "step": 1587
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1544.05810546875,
      "epoch": 0.4743484429840938,
      "grad_norm": 14.779484748840332,
      "kl": 3.859375,
      "learning_rate": 3.1520836273496876e-07,
      "loss": 0.2732,
      "reward": 0.5546875149011612,
      "reward_std": 0.15612070448696613,
      "rewards/accuracy_reward": 0.1160714328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4386160895228386,
      "step": 1588
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1464.6496276855469,
      "epoch": 0.47464715107161526,
      "grad_norm": 63.947322845458984,
      "kl": 3.31640625,
      "learning_rate": 3.149565973385052e-07,
      "loss": 0.3219,
      "reward": 0.5312500298023224,
      "reward_std": 0.12138961628079414,
      "rewards/accuracy_reward": 0.08928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4419643059372902,
      "step": 1589
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1566.8594665527344,
      "epoch": 0.47494585915913673,
      "grad_norm": 23.637889862060547,
      "kl": 3.86328125,
      "learning_rate": 3.147047612756302e-07,
      "loss": 0.3248,
      "reward": 0.4888393059372902,
      "reward_std": 0.15691402554512024,
      "rewards/accuracy_reward": 0.060267860535532236,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428571455180645,
      "step": 1590
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1572.9978332519531,
      "epoch": 0.4752445672466582,
      "grad_norm": 50.00650405883789,
      "kl": 4.8515625,
      "learning_rate": 3.1445285482031667e-07,
      "loss": 0.3137,
      "reward": 0.5468750074505806,
      "reward_std": 0.171761654317379,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 1591
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1514.8504943847656,
      "epoch": 0.47554327533417967,
      "grad_norm": 12.264228820800781,
      "kl": 3.80078125,
      "learning_rate": 3.142008782466142e-07,
      "loss": 0.2759,
      "reward": 0.569754496216774,
      "reward_std": 0.16302416287362576,
      "rewards/accuracy_reward": 0.1428571492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4268973395228386,
      "step": 1592
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1558.05810546875,
      "epoch": 0.47584198342170114,
      "grad_norm": 21.55459213256836,
      "kl": 3.45703125,
      "learning_rate": 3.1394883182864844e-07,
      "loss": 0.222,
      "reward": 0.546316996216774,
      "reward_std": 0.17352686449885368,
      "rewards/accuracy_reward": 0.1026785746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443638414144516,
      "step": 1593
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1635.2232971191406,
      "epoch": 0.4761406915092226,
      "grad_norm": 15.093851089477539,
      "kl": 3.3828125,
      "learning_rate": 3.136967158406214e-07,
      "loss": 0.2464,
      "reward": 0.5133928805589676,
      "reward_std": 0.10575600154697895,
      "rewards/accuracy_reward": 0.07589286053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000223517418,
      "step": 1594
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1640.8527526855469,
      "epoch": 0.4764393995967441,
      "grad_norm": 40.03180694580078,
      "kl": 5.03125,
      "learning_rate": 3.134445305568105e-07,
      "loss": 0.3387,
      "reward": 0.4810268059372902,
      "reward_std": 0.191373523324728,
      "rewards/accuracy_reward": 0.07142857648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4095982387661934,
      "step": 1595
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1643.66748046875,
      "epoch": 0.47673810768426556,
      "grad_norm": 23.007722854614258,
      "kl": 3.248046875,
      "learning_rate": 3.1319227625156853e-07,
      "loss": 0.2371,
      "reward": 0.5518973469734192,
      "reward_std": 0.1575364824384451,
      "rewards/accuracy_reward": 0.11607143096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4358259215950966,
      "step": 1596
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1572.9152221679688,
      "epoch": 0.477036815771787,
      "grad_norm": 74.79681396484375,
      "kl": 2.078125,
      "learning_rate": 3.129399531993235e-07,
      "loss": 0.187,
      "reward": 0.5290178805589676,
      "reward_std": 0.1349135059863329,
      "rewards/accuracy_reward": 0.08928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.439732164144516,
      "step": 1597
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1485.5804443359375,
      "epoch": 0.4773355238593085,
      "grad_norm": 27.57383918762207,
      "kl": 4.3203125,
      "learning_rate": 3.1268756167457823e-07,
      "loss": 0.3266,
      "reward": 0.5993303880095482,
      "reward_std": 0.16706236079335213,
      "rewards/accuracy_reward": 0.1651785783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341518059372902,
      "step": 1598
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1646.9085693359375,
      "epoch": 0.47763423194682997,
      "grad_norm": 18.822795867919922,
      "kl": 3.375,
      "learning_rate": 3.1243510195190984e-07,
      "loss": 0.2399,
      "reward": 0.4804687723517418,
      "reward_std": 0.14158467017114162,
      "rewards/accuracy_reward": 0.04017857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4402901977300644,
      "step": 1599
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1611.4822082519531,
      "epoch": 0.47793294003435144,
      "grad_norm": 15.994298934936523,
      "kl": 3.2890625,
      "learning_rate": 3.1218257430596985e-07,
      "loss": 0.2325,
      "reward": 0.5239955484867096,
      "reward_std": 0.16436783596873283,
      "rewards/accuracy_reward": 0.07812500605359674,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4458705559372902,
      "step": 1600
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1586.5134582519531,
      "epoch": 0.4782316481218729,
      "grad_norm": 31.604032516479492,
      "kl": 3.59765625,
      "learning_rate": 3.119299790114836e-07,
      "loss": 0.2808,
      "reward": 0.5145089402794838,
      "reward_std": 0.12953691743314266,
      "rewards/accuracy_reward": 0.08258928847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.431919664144516,
      "step": 1601
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1630.5000610351562,
      "epoch": 0.4785303562093944,
      "grad_norm": 11.223973274230957,
      "kl": 3.634765625,
      "learning_rate": 3.1167731634324994e-07,
      "loss": 0.2538,
      "reward": 0.5306919887661934,
      "reward_std": 0.12438629567623138,
      "rewards/accuracy_reward": 0.09151786006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.439174123108387,
      "step": 1602
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1746.6563110351562,
      "epoch": 0.47882906429691585,
      "grad_norm": 82.99742889404297,
      "kl": 5.9453125,
      "learning_rate": 3.1142458657614104e-07,
      "loss": 0.3618,
      "reward": 0.4603794887661934,
      "reward_std": 0.1524161472916603,
      "rewards/accuracy_reward": 0.05580357392318547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4045759066939354,
      "step": 1603
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1662.8013916015625,
      "epoch": 0.4791277723844373,
      "grad_norm": 88.9205322265625,
      "kl": 6.0234375,
      "learning_rate": 3.1117178998510237e-07,
      "loss": 0.348,
      "reward": 0.4603794813156128,
      "reward_std": 0.1535599809139967,
      "rewards/accuracy_reward": 0.0401785746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4202009215950966,
      "step": 1604
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1580.5402221679688,
      "epoch": 0.4794264804719588,
      "grad_norm": 27.044967651367188,
      "kl": 3.75390625,
      "learning_rate": 3.109189268451516e-07,
      "loss": 0.2562,
      "reward": 0.545200914144516,
      "reward_std": 0.12642978690564632,
      "rewards/accuracy_reward": 0.09151786123402417,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4536830559372902,
      "step": 1605
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1582.51123046875,
      "epoch": 0.47972518855948026,
      "grad_norm": 21.528409957885742,
      "kl": 4.34375,
      "learning_rate": 3.106659974313791e-07,
      "loss": 0.2878,
      "reward": 0.581473246216774,
      "reward_std": 0.13880587741732597,
      "rewards/accuracy_reward": 0.1383928656578064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4430803805589676,
      "step": 1606
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1566.4241638183594,
      "epoch": 0.48002389664700174,
      "grad_norm": 26.889516830444336,
      "kl": 3.6953125,
      "learning_rate": 3.1041300201894725e-07,
      "loss": 0.2483,
      "reward": 0.5373884215950966,
      "reward_std": 0.18998852744698524,
      "rewards/accuracy_reward": 0.10267857555299997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098469734192,
      "step": 1607
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1631.82373046875,
      "epoch": 0.4803226047345232,
      "grad_norm": 14.014554023742676,
      "kl": 3.83203125,
      "learning_rate": 3.101599408830904e-07,
      "loss": 0.2543,
      "reward": 0.5329241454601288,
      "reward_std": 0.18147086165845394,
      "rewards/accuracy_reward": 0.098214291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098395228386,
      "step": 1608
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1539.5223999023438,
      "epoch": 0.4806213128220447,
      "grad_norm": 32.147117614746094,
      "kl": 2.77734375,
      "learning_rate": 3.099068142991142e-07,
      "loss": 0.2087,
      "reward": 0.5044643059372902,
      "reward_std": 0.18120743706822395,
      "rewards/accuracy_reward": 0.0647321455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4397321566939354,
      "step": 1609
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1521.1585388183594,
      "epoch": 0.48092002090956615,
      "grad_norm": 14.000472068786621,
      "kl": 2.986328125,
      "learning_rate": 3.096536225423954e-07,
      "loss": 0.2222,
      "reward": 0.4737723469734192,
      "reward_std": 0.13127341866493225,
      "rewards/accuracy_reward": 0.029017857974395156,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4447544813156128,
      "step": 1610
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1623.7188415527344,
      "epoch": 0.4812187289970876,
      "grad_norm": 37.74376678466797,
      "kl": 3.640625,
      "learning_rate": 3.094003658883821e-07,
      "loss": 0.2782,
      "reward": 0.5569196790456772,
      "reward_std": 0.17462435364723206,
      "rewards/accuracy_reward": 0.12276786682195961,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341518059372902,
      "step": 1611
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1657.6228332519531,
      "epoch": 0.4815174370846091,
      "grad_norm": 29.947187423706055,
      "kl": 3.64453125,
      "learning_rate": 3.0914704461259255e-07,
      "loss": 0.2558,
      "reward": 0.5212053805589676,
      "reward_std": 0.1573030073195696,
      "rewards/accuracy_reward": 0.09598214738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232387661934,
      "step": 1612
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1693.9107666015625,
      "epoch": 0.48181614517213056,
      "grad_norm": 15.240757942199707,
      "kl": 4.29296875,
      "learning_rate": 3.088936589906155e-07,
      "loss": 0.2749,
      "reward": 0.4352678805589676,
      "reward_std": 0.1524873785674572,
      "rewards/accuracy_reward": 0.026785715715959668,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.408482164144516,
      "step": 1613
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1712.1139221191406,
      "epoch": 0.48211485325965203,
      "grad_norm": 10.832959175109863,
      "kl": 3.5625,
      "learning_rate": 3.086402092981096e-07,
      "loss": 0.2321,
      "reward": 0.5385044887661934,
      "reward_std": 0.15707102604210377,
      "rewards/accuracy_reward": 0.11830357951112092,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.420200914144516,
      "step": 1614
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1564.9263916015625,
      "epoch": 0.48241356134717345,
      "grad_norm": 7.38329553604126,
      "kl": 3.388671875,
      "learning_rate": 3.0838669581080334e-07,
      "loss": 0.2406,
      "reward": 0.5318080633878708,
      "reward_std": 0.15097877755761147,
      "rewards/accuracy_reward": 0.08482143515720963,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4469866305589676,
      "step": 1615
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1668.0603332519531,
      "epoch": 0.4827122694346949,
      "grad_norm": 8.034990310668945,
      "kl": 3.376953125,
      "learning_rate": 3.0813311880449466e-07,
      "loss": 0.2071,
      "reward": 0.474888414144516,
      "reward_std": 0.15962528437376022,
      "rewards/accuracy_reward": 0.03348214412108064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4414062723517418,
      "step": 1616
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1559.60498046875,
      "epoch": 0.4830109775222164,
      "grad_norm": 37.72300720214844,
      "kl": 4.8515625,
      "learning_rate": 3.0787947855505025e-07,
      "loss": 0.3389,
      "reward": 0.5864955484867096,
      "reward_std": 0.17711183056235313,
      "rewards/accuracy_reward": 0.16294643771834671,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.423549123108387,
      "step": 1617
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1595.3348999023438,
      "epoch": 0.48330968560973786,
      "grad_norm": 31.655437469482422,
      "kl": 4.26953125,
      "learning_rate": 3.0762577533840606e-07,
      "loss": 0.2795,
      "reward": 0.5396205484867096,
      "reward_std": 0.1576180700212717,
      "rewards/accuracy_reward": 0.11160714854486287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4280134066939354,
      "step": 1618
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1615.43310546875,
      "epoch": 0.48360839369725933,
      "grad_norm": 10.194987297058105,
      "kl": 4.078125,
      "learning_rate": 3.073720094305662e-07,
      "loss": 0.2891,
      "reward": 0.5686384215950966,
      "reward_std": 0.15565957874059677,
      "rewards/accuracy_reward": 0.1428571492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812649011612,
      "step": 1619
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1561.27685546875,
      "epoch": 0.4839071017847808,
      "grad_norm": 10.495670318603516,
      "kl": 3.4921875,
      "learning_rate": 3.0711818110760313e-07,
      "loss": 0.2713,
      "reward": 0.5217634066939354,
      "reward_std": 0.18035482801496983,
      "rewards/accuracy_reward": 0.0937500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4280134066939354,
      "step": 1620
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1681.0313110351562,
      "epoch": 0.4842058098723023,
      "grad_norm": 43.72733688354492,
      "kl": 2.89453125,
      "learning_rate": 3.068642906456572e-07,
      "loss": 0.2237,
      "reward": 0.4676339477300644,
      "reward_std": 0.20020072534680367,
      "rewards/accuracy_reward": 0.05803571571595967,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4095982313156128,
      "step": 1621
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1608.6429443359375,
      "epoch": 0.48450451795982374,
      "grad_norm": 80.25322723388672,
      "kl": 1.953125,
      "learning_rate": 3.0661033832093636e-07,
      "loss": 0.1704,
      "reward": 0.5468750149011612,
      "reward_std": 0.16547834873199463,
      "rewards/accuracy_reward": 0.1004464328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4464285895228386,
      "step": 1622
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1448.5915832519531,
      "epoch": 0.4848032260473452,
      "grad_norm": 15.838859558105469,
      "kl": 3.26171875,
      "learning_rate": 3.063563244097159e-07,
      "loss": 0.2468,
      "reward": 0.6668527200818062,
      "reward_std": 0.16469843313097954,
      "rewards/accuracy_reward": 0.21651787403970957,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4503348395228386,
      "step": 1623
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1540.9219055175781,
      "epoch": 0.4851019341348667,
      "grad_norm": 10.395541191101074,
      "kl": 3.39453125,
      "learning_rate": 3.061022491883378e-07,
      "loss": 0.2279,
      "reward": 0.5279018133878708,
      "reward_std": 0.15787720493972301,
      "rewards/accuracy_reward": 0.08035714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.447544664144516,
      "step": 1624
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1614.696533203125,
      "epoch": 0.48540064222238816,
      "grad_norm": 31.912437438964844,
      "kl": 3.84765625,
      "learning_rate": 3.058481129332114e-07,
      "loss": 0.2524,
      "reward": 0.4525669813156128,
      "reward_std": 0.14270638301968575,
      "rewards/accuracy_reward": 0.020089287078008056,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776977300644,
      "step": 1625
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1548.9241943359375,
      "epoch": 0.4856993503099096,
      "grad_norm": 41.06375503540039,
      "kl": 4.67578125,
      "learning_rate": 3.0559391592081173e-07,
      "loss": 0.3377,
      "reward": 0.5279018133878708,
      "reward_std": 0.17372934147715569,
      "rewards/accuracy_reward": 0.10267857555299997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232238650322,
      "step": 1626
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1635.6384582519531,
      "epoch": 0.4859980583974311,
      "grad_norm": 21.54033660888672,
      "kl": 3.703125,
      "learning_rate": 3.0533965842768037e-07,
      "loss": 0.2198,
      "reward": 0.5150669887661934,
      "reward_std": 0.19958655163645744,
      "rewards/accuracy_reward": 0.08482143469154835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455559372902,
      "step": 1627
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1596.5313110351562,
      "epoch": 0.48629676648495257,
      "grad_norm": 100.40029907226562,
      "kl": 1.890625,
      "learning_rate": 3.050853407304245e-07,
      "loss": 0.1781,
      "reward": 0.506138414144516,
      "reward_std": 0.17503629811108112,
      "rewards/accuracy_reward": 0.06696428963914514,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4391741305589676,
      "step": 1628
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1529.1295166015625,
      "epoch": 0.48659547457247404,
      "grad_norm": 111.42853546142578,
      "kl": 1.544921875,
      "learning_rate": 3.0483096310571687e-07,
      "loss": 0.1891,
      "reward": 0.5825893133878708,
      "reward_std": 0.19627559185028076,
      "rewards/accuracy_reward": 0.13616072246804833,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4464285969734192,
      "step": 1629
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1611.6652526855469,
      "epoch": 0.4868941826599955,
      "grad_norm": 117.06646728515625,
      "kl": 1.939453125,
      "learning_rate": 3.0457652583029535e-07,
      "loss": 0.1851,
      "reward": 0.4799107387661934,
      "reward_std": 0.1327403448522091,
      "rewards/accuracy_reward": 0.04910714505240321,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035895228386,
      "step": 1630
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1490.8237609863281,
      "epoch": 0.487192890747517,
      "grad_norm": 110.13700103759766,
      "kl": 1.658203125,
      "learning_rate": 3.0432202918096283e-07,
      "loss": 0.2052,
      "reward": 0.4983259066939354,
      "reward_std": 0.13787688687443733,
      "rewards/accuracy_reward": 0.06026785937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4380580559372902,
      "step": 1631
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1647.9263916015625,
      "epoch": 0.48749159883503845,
      "grad_norm": 40.727535247802734,
      "kl": 2.99609375,
      "learning_rate": 3.040674734345867e-07,
      "loss": 0.2216,
      "reward": 0.5385044813156128,
      "reward_std": 0.1657584086060524,
      "rewards/accuracy_reward": 0.1049107201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937798023224,
      "step": 1632
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1564.6630249023438,
      "epoch": 0.4877903069225599,
      "grad_norm": 42.8799934387207,
      "kl": 3.125,
      "learning_rate": 3.0381285886809867e-07,
      "loss": 0.2713,
      "reward": 0.4687500149011612,
      "reward_std": 0.16412829980254173,
      "rewards/accuracy_reward": 0.037946430733427405,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035895228386,
      "step": 1633
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1620.4152221679688,
      "epoch": 0.4880890150100814,
      "grad_norm": 28.52981948852539,
      "kl": 3.2109375,
      "learning_rate": 3.0355818575849443e-07,
      "loss": 0.2458,
      "reward": 0.5418526977300644,
      "reward_std": 0.1686634048819542,
      "rewards/accuracy_reward": 0.10714286053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098395228386,
      "step": 1634
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1640.69873046875,
      "epoch": 0.48838772309760287,
      "grad_norm": 21.642587661743164,
      "kl": 3.24609375,
      "learning_rate": 3.033034543828332e-07,
      "loss": 0.2368,
      "reward": 0.503906287252903,
      "reward_std": 0.21493875980377197,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776977300644,
      "step": 1635
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1591.6228332519531,
      "epoch": 0.48868643118512434,
      "grad_norm": 49.88224411010742,
      "kl": 4.1171875,
      "learning_rate": 3.030486650182381e-07,
      "loss": 0.2587,
      "reward": 0.557477705180645,
      "reward_std": 0.18476995266973972,
      "rewards/accuracy_reward": 0.11160714644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4458705559372902,
      "step": 1636
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1551.6205749511719,
      "epoch": 0.4889851392726458,
      "grad_norm": 60.93635940551758,
      "kl": 4.5625,
      "learning_rate": 3.0279381794189466e-07,
      "loss": 0.3116,
      "reward": 0.474888414144516,
      "reward_std": 0.1465768702328205,
      "rewards/accuracy_reward": 0.029017858672887087,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4458705559372902,
      "step": 1637
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1541.1429138183594,
      "epoch": 0.4892838473601673,
      "grad_norm": 41.259971618652344,
      "kl": 4.62109375,
      "learning_rate": 3.025389134310517e-07,
      "loss": 0.3341,
      "reward": 0.5083705559372902,
      "reward_std": 0.17395376414060593,
      "rewards/accuracy_reward": 0.07366071874275804,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.434709832072258,
      "step": 1638
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1559.9978332519531,
      "epoch": 0.48958255544768875,
      "grad_norm": 16.896432876586914,
      "kl": 3.431640625,
      "learning_rate": 3.022839517630203e-07,
      "loss": 0.2416,
      "reward": 0.5279017984867096,
      "reward_std": 0.11502685211598873,
      "rewards/accuracy_reward": 0.0848214328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443080373108387,
      "step": 1639
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1565.7545471191406,
      "epoch": 0.4898812635352102,
      "grad_norm": 111.08955383300781,
      "kl": 6.1953125,
      "learning_rate": 3.0202893321517374e-07,
      "loss": 0.416,
      "reward": 0.506138414144516,
      "reward_std": 0.21261441707611084,
      "rewards/accuracy_reward": 0.06919643376022577,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419813156128,
      "step": 1640
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1600.1384582519531,
      "epoch": 0.4901799716227317,
      "grad_norm": 41.925968170166016,
      "kl": 4.67578125,
      "learning_rate": 3.0177385806494726e-07,
      "loss": 0.2947,
      "reward": 0.5797991305589676,
      "reward_std": 0.16164417937397957,
      "rewards/accuracy_reward": 0.14955357578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455484867096,
      "step": 1641
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1529.8639221191406,
      "epoch": 0.49047867971025316,
      "grad_norm": 56.106483459472656,
      "kl": 4.5703125,
      "learning_rate": 3.0151872658983753e-07,
      "loss": 0.2975,
      "reward": 0.5591517984867096,
      "reward_std": 0.204350546002388,
      "rewards/accuracy_reward": 0.113839291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125223517418,
      "step": 1642
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1635.0983276367188,
      "epoch": 0.49077738779777463,
      "grad_norm": 59.32293701171875,
      "kl": 5.03125,
      "learning_rate": 3.012635390674027e-07,
      "loss": 0.3155,
      "reward": 0.595982164144516,
      "reward_std": 0.1909337416291237,
      "rewards/accuracy_reward": 0.1607142947614193,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4352678805589676,
      "step": 1643
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1551.1451416015625,
      "epoch": 0.4910760958852961,
      "grad_norm": 10.997405052185059,
      "kl": 3.255859375,
      "learning_rate": 3.010082957752617e-07,
      "loss": 0.2317,
      "reward": 0.5217634215950966,
      "reward_std": 0.1865183226764202,
      "rewards/accuracy_reward": 0.07142857508733869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4503348469734192,
      "step": 1644
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1654.7791137695312,
      "epoch": 0.4913748039728176,
      "grad_norm": 23.717939376831055,
      "kl": 3.98828125,
      "learning_rate": 3.0075299699109425e-07,
      "loss": 0.262,
      "reward": 0.4614955633878708,
      "reward_std": 0.14975935220718384,
      "rewards/accuracy_reward": 0.022321430267766118,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4391741305589676,
      "step": 1645
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1510.0848693847656,
      "epoch": 0.49167351206033905,
      "grad_norm": 21.089120864868164,
      "kl": 2.7109375,
      "learning_rate": 3.004976429926404e-07,
      "loss": 0.2141,
      "reward": 0.5909598469734192,
      "reward_std": 0.17239592224359512,
      "rewards/accuracy_reward": 0.1450892947614193,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4458705559372902,
      "step": 1646
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1519.7411193847656,
      "epoch": 0.4919722201478605,
      "grad_norm": 12.201148986816406,
      "kl": 2.859375,
      "learning_rate": 3.002422340577003e-07,
      "loss": 0.2377,
      "reward": 0.6266741380095482,
      "reward_std": 0.1403534021228552,
      "rewards/accuracy_reward": 0.1830357275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443638414144516,
      "step": 1647
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1542.9419860839844,
      "epoch": 0.492270928235382,
      "grad_norm": 10.617005348205566,
      "kl": 3.63671875,
      "learning_rate": 2.9998677046413383e-07,
      "loss": 0.2845,
      "reward": 0.5675223469734192,
      "reward_std": 0.153344189748168,
      "rewards/accuracy_reward": 0.1250000074505806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.442522332072258,
      "step": 1648
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1626.1295471191406,
      "epoch": 0.49256963632290346,
      "grad_norm": 17.984851837158203,
      "kl": 3.4609375,
      "learning_rate": 2.9973125248986026e-07,
      "loss": 0.2449,
      "reward": 0.5172991380095482,
      "reward_std": 0.18670892529189587,
      "rewards/accuracy_reward": 0.08258928917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098395228386,
      "step": 1649
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1656.1920166015625,
      "epoch": 0.49286834441042493,
      "grad_norm": 17.57750129699707,
      "kl": 3.39453125,
      "learning_rate": 2.994756804128582e-07,
      "loss": 0.2218,
      "reward": 0.4988839626312256,
      "reward_std": 0.1599156092852354,
      "rewards/accuracy_reward": 0.058035718044266105,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4408482387661934,
      "step": 1650
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1470.9509887695312,
      "epoch": 0.4931670524979464,
      "grad_norm": 61.94028091430664,
      "kl": 2.546875,
      "learning_rate": 2.99220054511165e-07,
      "loss": 0.25,
      "reward": 0.5295759215950966,
      "reward_std": 0.1381382755935192,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4581473395228386,
      "step": 1651
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1577.7902526855469,
      "epoch": 0.49346576058546787,
      "grad_norm": 32.88896942138672,
      "kl": 4.3984375,
      "learning_rate": 2.9896437506287654e-07,
      "loss": 0.2957,
      "reward": 0.5825893208384514,
      "reward_std": 0.14154371060431004,
      "rewards/accuracy_reward": 0.15625000861473382,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263393059372902,
      "step": 1652
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1583.4911499023438,
      "epoch": 0.49376446867298934,
      "grad_norm": 15.538206100463867,
      "kl": 3.1875,
      "learning_rate": 2.9870864234614706e-07,
      "loss": 0.2173,
      "reward": 0.4681919813156128,
      "reward_std": 0.14478963240981102,
      "rewards/accuracy_reward": 0.03125000116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419813156128,
      "step": 1653
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1710.3884582519531,
      "epoch": 0.4940631767605108,
      "grad_norm": 11.06654167175293,
      "kl": 4.0625,
      "learning_rate": 2.9845285663918876e-07,
      "loss": 0.2636,
      "reward": 0.4877232313156128,
      "reward_std": 0.15105099231004715,
      "rewards/accuracy_reward": 0.05357143119908869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341518059372902,
      "step": 1654
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1520.5692749023438,
      "epoch": 0.4943618848480323,
      "grad_norm": 9.066946029663086,
      "kl": 3.09765625,
      "learning_rate": 2.9819701822027123e-07,
      "loss": 0.2356,
      "reward": 0.5318080633878708,
      "reward_std": 0.16076518595218658,
      "rewards/accuracy_reward": 0.0892857164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4425223395228386,
      "step": 1655
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1633.97998046875,
      "epoch": 0.49466059293555376,
      "grad_norm": 44.1402587890625,
      "kl": 4.42578125,
      "learning_rate": 2.9794112736772166e-07,
      "loss": 0.2724,
      "reward": 0.4626116305589676,
      "reward_std": 0.15510892868041992,
      "rewards/accuracy_reward": 0.026785715017467737,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435825914144516,
      "step": 1656
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1607.6585388183594,
      "epoch": 0.4949593010230752,
      "grad_norm": 21.735055923461914,
      "kl": 3.828125,
      "learning_rate": 2.9768518435992427e-07,
      "loss": 0.296,
      "reward": 0.5357143059372902,
      "reward_std": 0.16888817213475704,
      "rewards/accuracy_reward": 0.1049107201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.430803582072258,
      "step": 1657
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1437.6451416015625,
      "epoch": 0.49525800911059664,
      "grad_norm": 23.107942581176758,
      "kl": 3.28515625,
      "learning_rate": 2.9742918947531996e-07,
      "loss": 0.2672,
      "reward": 0.5680803880095482,
      "reward_std": 0.13175320997834206,
      "rewards/accuracy_reward": 0.1116071492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4564732387661934,
      "step": 1658
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1615.8616638183594,
      "epoch": 0.4955567171981181,
      "grad_norm": 54.83425521850586,
      "kl": 3.30859375,
      "learning_rate": 2.9717314299240606e-07,
      "loss": 0.2688,
      "reward": 0.4492187649011612,
      "reward_std": 0.14813455007970333,
      "rewards/accuracy_reward": 0.020089286845177412,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294887661934,
      "step": 1659
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1523.9710388183594,
      "epoch": 0.4958554252856396,
      "grad_norm": 10.82107925415039,
      "kl": 2.865234375,
      "learning_rate": 2.9691704518973603e-07,
      "loss": 0.1888,
      "reward": 0.5228794887661934,
      "reward_std": 0.15354564413428307,
      "rewards/accuracy_reward": 0.07589286286383867,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.446986623108387,
      "step": 1660
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1639.6563110351562,
      "epoch": 0.49615413337316105,
      "grad_norm": 68.94493865966797,
      "kl": 4.54296875,
      "learning_rate": 2.966608963459193e-07,
      "loss": 0.2543,
      "reward": 0.501674123108387,
      "reward_std": 0.1595360618084669,
      "rewards/accuracy_reward": 0.08928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.412388414144516,
      "step": 1661
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1690.2478332519531,
      "epoch": 0.4964528414606825,
      "grad_norm": 51.9144401550293,
      "kl": 4.859375,
      "learning_rate": 2.964046967396205e-07,
      "loss": 0.2756,
      "reward": 0.5172991380095482,
      "reward_std": 0.14233889803290367,
      "rewards/accuracy_reward": 0.08928572130389512,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4280134066939354,
      "step": 1662
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1574.1340026855469,
      "epoch": 0.496751549548204,
      "grad_norm": 36.03261184692383,
      "kl": 3.80078125,
      "learning_rate": 2.961484466495598e-07,
      "loss": 0.2245,
      "reward": 0.4648437649011612,
      "reward_std": 0.12114820070564747,
      "rewards/accuracy_reward": 0.020089285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4447544813156128,
      "step": 1663
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1650.4040832519531,
      "epoch": 0.49705025763572547,
      "grad_norm": 11.337340354919434,
      "kl": 4.05859375,
      "learning_rate": 2.958921463545122e-07,
      "loss": 0.2739,
      "reward": 0.486049123108387,
      "reward_std": 0.2034171111881733,
      "rewards/accuracy_reward": 0.051339288242161274,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098395228386,
      "step": 1664
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1675.9442443847656,
      "epoch": 0.49734896572324694,
      "grad_norm": 21.9161319732666,
      "kl": 3.52734375,
      "learning_rate": 2.956357961333073e-07,
      "loss": 0.2511,
      "reward": 0.5117187723517418,
      "reward_std": 0.16327261365950108,
      "rewards/accuracy_reward": 0.0647321455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4469866305589676,
      "step": 1665
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1733.4398498535156,
      "epoch": 0.4976476738107684,
      "grad_norm": 32.65961456298828,
      "kl": 4.6953125,
      "learning_rate": 2.9537939626482895e-07,
      "loss": 0.2837,
      "reward": 0.5279018133878708,
      "reward_std": 0.18383228033781052,
      "rewards/accuracy_reward": 0.11160714598372579,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4162946566939354,
      "step": 1666
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1657.2099304199219,
      "epoch": 0.4979463818982899,
      "grad_norm": 21.11324691772461,
      "kl": 4.046875,
      "learning_rate": 2.9512294702801515e-07,
      "loss": 0.2521,
      "reward": 0.5474330633878708,
      "reward_std": 0.18474116921424866,
      "rewards/accuracy_reward": 0.1183035746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294887661934,
      "step": 1667
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1638.196533203125,
      "epoch": 0.49824508998581135,
      "grad_norm": 33.767112731933594,
      "kl": 4.009765625,
      "learning_rate": 2.948664487018575e-07,
      "loss": 0.2482,
      "reward": 0.497209832072258,
      "reward_std": 0.16728181391954422,
      "rewards/accuracy_reward": 0.06250000232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098469734192,
      "step": 1668
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1588.3572082519531,
      "epoch": 0.4985437980733328,
      "grad_norm": 35.746307373046875,
      "kl": 2.53125,
      "learning_rate": 2.946099015654011e-07,
      "loss": 0.1953,
      "reward": 0.5228794887661934,
      "reward_std": 0.1865379586815834,
      "rewards/accuracy_reward": 0.08035714412108064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4425223395228386,
      "step": 1669
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1573.3371276855469,
      "epoch": 0.4988425061608543,
      "grad_norm": 60.449493408203125,
      "kl": 2.484375,
      "learning_rate": 2.9435330589774397e-07,
      "loss": 0.2002,
      "reward": 0.5295759215950966,
      "reward_std": 0.1064897608011961,
      "rewards/accuracy_reward": 0.082589291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.446986623108387,
      "step": 1670
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1640.2723999023438,
      "epoch": 0.49914121424837576,
      "grad_norm": 52.75844955444336,
      "kl": 2.3583984375,
      "learning_rate": 2.940966619780371e-07,
      "loss": 0.1735,
      "reward": 0.4776785969734192,
      "reward_std": 0.15617157891392708,
      "rewards/accuracy_reward": 0.03794643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.439732164144516,
      "step": 1671
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1547.63623046875,
      "epoch": 0.49943992233589724,
      "grad_norm": 19.882774353027344,
      "kl": 2.62109375,
      "learning_rate": 2.93839970085484e-07,
      "loss": 0.1842,
      "reward": 0.5094866380095482,
      "reward_std": 0.17309125140309334,
      "rewards/accuracy_reward": 0.0580357164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.451450914144516,
      "step": 1672
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1643.51123046875,
      "epoch": 0.4997386304234187,
      "grad_norm": 25.425533294677734,
      "kl": 4.45703125,
      "learning_rate": 2.935832304993402e-07,
      "loss": 0.287,
      "reward": 0.4771205484867096,
      "reward_std": 0.12992664612829685,
      "rewards/accuracy_reward": 0.0558035746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4213169813156128,
      "step": 1673
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1586.6942443847656,
      "epoch": 0.5000373385109402,
      "grad_norm": 12.34133529663086,
      "kl": 3.01953125,
      "learning_rate": 2.933264434989132e-07,
      "loss": 0.1938,
      "reward": 0.5452009290456772,
      "reward_std": 0.16517481207847595,
      "rewards/accuracy_reward": 0.09821428917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.446986623108387,
      "step": 1674
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1694.665283203125,
      "epoch": 0.5003360465984616,
      "grad_norm": 13.038610458374023,
      "kl": 3.54296875,
      "learning_rate": 2.930696093635622e-07,
      "loss": 0.2234,
      "reward": 0.504464328289032,
      "reward_std": 0.21597828716039658,
      "rewards/accuracy_reward": 0.06473214598372579,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4397321566939354,
      "step": 1675
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1621.8482666015625,
      "epoch": 0.5006347546859832,
      "grad_norm": 25.91422462463379,
      "kl": 2.87109375,
      "learning_rate": 2.9281272837269744e-07,
      "loss": 0.1909,
      "reward": 0.546316996216774,
      "reward_std": 0.11880997382104397,
      "rewards/accuracy_reward": 0.09821428917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.448102705180645,
      "step": 1676
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1527.2098693847656,
      "epoch": 0.5009334627735046,
      "grad_norm": 10.468071937561035,
      "kl": 3.12109375,
      "learning_rate": 2.925558008057803e-07,
      "loss": 0.2032,
      "reward": 0.5195312723517418,
      "reward_std": 0.1293280217796564,
      "rewards/accuracy_reward": 0.06696428847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4525669813156128,
      "step": 1677
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1625.3884887695312,
      "epoch": 0.5012321708610261,
      "grad_norm": 37.449989318847656,
      "kl": 3.080078125,
      "learning_rate": 2.9229882694232285e-07,
      "loss": 0.2147,
      "reward": 0.495535746216774,
      "reward_std": 0.1395468283444643,
      "rewards/accuracy_reward": 0.06026786030270159,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435267873108387,
      "step": 1678
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1586.0536499023438,
      "epoch": 0.5015308789485475,
      "grad_norm": 54.981346130371094,
      "kl": 2.37890625,
      "learning_rate": 2.9204180706188735e-07,
      "loss": 0.1903,
      "reward": 0.5089285895228386,
      "reward_std": 0.13314694166183472,
      "rewards/accuracy_reward": 0.05803571757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.450892873108387,
      "step": 1679
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1674.2054443359375,
      "epoch": 0.501829587036069,
      "grad_norm": 6.192504405975342,
      "kl": 3.515625,
      "learning_rate": 2.917847414440864e-07,
      "loss": 0.2105,
      "reward": 0.459263414144516,
      "reward_std": 0.1515572890639305,
      "rewards/accuracy_reward": 0.026785715017467737,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776977300644,
      "step": 1680
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1641.3616943359375,
      "epoch": 0.5021282951235905,
      "grad_norm": 6.696020603179932,
      "kl": 3.6875,
      "learning_rate": 2.91527630368582e-07,
      "loss": 0.2436,
      "reward": 0.521763414144516,
      "reward_std": 0.15224834345281124,
      "rewards/accuracy_reward": 0.09375000465661287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4280134066939354,
      "step": 1681
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1638.30810546875,
      "epoch": 0.5024270032111119,
      "grad_norm": 17.795812606811523,
      "kl": 3.2421875,
      "learning_rate": 2.9127047411508596e-07,
      "loss": 0.2034,
      "reward": 0.4849330559372902,
      "reward_std": 0.11843020841479301,
      "rewards/accuracy_reward": 0.04910714412108064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4358259066939354,
      "step": 1682
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1576.88623046875,
      "epoch": 0.5027257112986334,
      "grad_norm": 48.165035247802734,
      "kl": 2.90625,
      "learning_rate": 2.9101327296335897e-07,
      "loss": 0.231,
      "reward": 0.5156250223517418,
      "reward_std": 0.15507948398590088,
      "rewards/accuracy_reward": 0.07142857392318547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.444196455180645,
      "step": 1683
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1650.544677734375,
      "epoch": 0.5030244193861548,
      "grad_norm": 34.356109619140625,
      "kl": 4.14453125,
      "learning_rate": 2.9075602719321073e-07,
      "loss": 0.2419,
      "reward": 0.5864955633878708,
      "reward_std": 0.17027808167040348,
      "rewards/accuracy_reward": 0.15848215040750802,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4280134066939354,
      "step": 1684
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1585.9755554199219,
      "epoch": 0.5033231274736764,
      "grad_norm": 35.28438949584961,
      "kl": 3.92578125,
      "learning_rate": 2.9049873708449946e-07,
      "loss": 0.2574,
      "reward": 0.471540205180645,
      "reward_std": 0.151686517521739,
      "rewards/accuracy_reward": 0.03571428661234677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4358259215950966,
      "step": 1685
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1594.3482971191406,
      "epoch": 0.5036218355611978,
      "grad_norm": 38.939884185791016,
      "kl": 4.8359375,
      "learning_rate": 2.902414029171316e-07,
      "loss": 0.3417,
      "reward": 0.5106027126312256,
      "reward_std": 0.18022429384291172,
      "rewards/accuracy_reward": 0.08035714412108064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455559372902,
      "step": 1686
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1686.5938110351562,
      "epoch": 0.5039205436487193,
      "grad_norm": 29.193008422851562,
      "kl": 4.82421875,
      "learning_rate": 2.8998402497106133e-07,
      "loss": 0.3009,
      "reward": 0.4776785969734192,
      "reward_std": 0.17025581002235413,
      "rewards/accuracy_reward": 0.05357143050059676,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.424107164144516,
      "step": 1687
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1507.62060546875,
      "epoch": 0.5042192517362407,
      "grad_norm": 30.875288009643555,
      "kl": 3.630859375,
      "learning_rate": 2.897266035262908e-07,
      "loss": 0.2258,
      "reward": 0.564732164144516,
      "reward_std": 0.13637754693627357,
      "rewards/accuracy_reward": 0.1160714365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4486607387661934,
      "step": 1688
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1615.6452026367188,
      "epoch": 0.5045179598237622,
      "grad_norm": 16.45070457458496,
      "kl": 3.7109375,
      "learning_rate": 2.894691388628693e-07,
      "loss": 0.2348,
      "reward": 0.510044664144516,
      "reward_std": 0.14795215614140034,
      "rewards/accuracy_reward": 0.07366071711294353,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.436383955180645,
      "step": 1689
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1621.13623046875,
      "epoch": 0.5048166679112837,
      "grad_norm": 32.02790069580078,
      "kl": 3.86328125,
      "learning_rate": 2.892116312608931e-07,
      "loss": 0.2311,
      "reward": 0.5167410969734192,
      "reward_std": 0.13097978569567204,
      "rewards/accuracy_reward": 0.08705357392318547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875223517418,
      "step": 1690
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1663.7880249023438,
      "epoch": 0.5051153759988052,
      "grad_norm": 30.566143035888672,
      "kl": 3.53125,
      "learning_rate": 2.889540810005052e-07,
      "loss": 0.2019,
      "reward": 0.5435268133878708,
      "reward_std": 0.13941910490393639,
      "rewards/accuracy_reward": 0.1049107201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4386160895228386,
      "step": 1691
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1683.3817749023438,
      "epoch": 0.5054140840863266,
      "grad_norm": 8.324190139770508,
      "kl": 3.95703125,
      "learning_rate": 2.886964883618951e-07,
      "loss": 0.2567,
      "reward": 0.4966518059372902,
      "reward_std": 0.12623316049575806,
      "rewards/accuracy_reward": 0.0558035746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4408482313156128,
      "step": 1692
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1636.5134887695312,
      "epoch": 0.5057127921738481,
      "grad_norm": 14.826581954956055,
      "kl": 3.2734375,
      "learning_rate": 2.884388536252983e-07,
      "loss": 0.2329,
      "reward": 0.471540205180645,
      "reward_std": 0.14235100150108337,
      "rewards/accuracy_reward": 0.03125000116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4402901977300644,
      "step": 1693
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1582.1005249023438,
      "epoch": 0.5060115002613695,
      "grad_norm": 28.0665283203125,
      "kl": 2.76953125,
      "learning_rate": 2.881811770709962e-07,
      "loss": 0.1933,
      "reward": 0.6473214328289032,
      "reward_std": 0.15412876196205616,
      "rewards/accuracy_reward": 0.1919642947614193,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.455357164144516,
      "step": 1694
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1661.47998046875,
      "epoch": 0.5063102083488911,
      "grad_norm": 9.365323066711426,
      "kl": 2.41796875,
      "learning_rate": 2.8792345897931556e-07,
      "loss": 0.157,
      "reward": 0.5279018208384514,
      "reward_std": 0.18057582527399063,
      "rewards/accuracy_reward": 0.08482143189758062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443080373108387,
      "step": 1695
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1659.08935546875,
      "epoch": 0.5066089164364125,
      "grad_norm": 20.349254608154297,
      "kl": 3.435546875,
      "learning_rate": 2.8766569963062847e-07,
      "loss": 0.2295,
      "reward": 0.5340401977300644,
      "reward_std": 0.16161047108471394,
      "rewards/accuracy_reward": 0.09598214831203222,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4380580559372902,
      "step": 1696
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1603.1138916015625,
      "epoch": 0.506907624523934,
      "grad_norm": 17.645732879638672,
      "kl": 3.4453125,
      "learning_rate": 2.874078993053519e-07,
      "loss": 0.2271,
      "reward": 0.5597098469734192,
      "reward_std": 0.18300314620137215,
      "rewards/accuracy_reward": 0.11160715157166123,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4481026977300644,
      "step": 1697
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1599.93310546875,
      "epoch": 0.5072063326114554,
      "grad_norm": 7.805138111114502,
      "kl": 3.078125,
      "learning_rate": 2.871500582839471e-07,
      "loss": 0.2113,
      "reward": 0.5463169887661934,
      "reward_std": 0.15872856229543686,
      "rewards/accuracy_reward": 0.1004464365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4458705633878708,
      "step": 1698
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1602.9286499023438,
      "epoch": 0.507505040698977,
      "grad_norm": 51.21775817871094,
      "kl": 4.6875,
      "learning_rate": 2.8689217684692014e-07,
      "loss": 0.2819,
      "reward": 0.544642873108387,
      "reward_std": 0.20269916579127312,
      "rewards/accuracy_reward": 0.11607143515720963,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 1699
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1544.700927734375,
      "epoch": 0.5078037487864984,
      "grad_norm": 12.312024116516113,
      "kl": 3.150390625,
      "learning_rate": 2.8663425527482047e-07,
      "loss": 0.2183,
      "reward": 0.490513414144516,
      "reward_std": 0.1532817929983139,
      "rewards/accuracy_reward": 0.044642859138548374,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4458705559372902,
      "step": 1700
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1665.1094665527344,
      "epoch": 0.5081024568740199,
      "grad_norm": 7.044699668884277,
      "kl": 2.86328125,
      "learning_rate": 2.863762938482417e-07,
      "loss": 0.1765,
      "reward": 0.588727705180645,
      "reward_std": 0.19333726540207863,
      "rewards/accuracy_reward": 0.13839286379516125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4503348395228386,
      "step": 1701
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1552.8951721191406,
      "epoch": 0.5084011649615413,
      "grad_norm": 15.801091194152832,
      "kl": 3.51953125,
      "learning_rate": 2.861182928478204e-07,
      "loss": 0.246,
      "reward": 0.5033482387661934,
      "reward_std": 0.16814018599689007,
      "rewards/accuracy_reward": 0.06250000488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4408482387661934,
      "step": 1702
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1656.74560546875,
      "epoch": 0.5086998730490628,
      "grad_norm": 23.930551528930664,
      "kl": 3.98828125,
      "learning_rate": 2.8586025255423653e-07,
      "loss": 0.2494,
      "reward": 0.5926339626312256,
      "reward_std": 0.1494963150471449,
      "rewards/accuracy_reward": 0.15625000558793545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.436383955180645,
      "step": 1703
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1576.4911193847656,
      "epoch": 0.5089985811365842,
      "grad_norm": 32.162025451660156,
      "kl": 3.05078125,
      "learning_rate": 2.8560217324821253e-07,
      "loss": 0.2356,
      "reward": 0.4949776977300644,
      "reward_std": 0.10406588017940521,
      "rewards/accuracy_reward": 0.04241071501746774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4525669887661934,
      "step": 1704
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1756.6429443359375,
      "epoch": 0.5092972892241058,
      "grad_norm": 58.297550201416016,
      "kl": 5.296875,
      "learning_rate": 2.8534405521051334e-07,
      "loss": 0.3031,
      "reward": 0.4977678880095482,
      "reward_std": 0.13423941284418106,
      "rewards/accuracy_reward": 0.07812500349245965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4196428805589676,
      "step": 1705
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1571.2634582519531,
      "epoch": 0.5095959973116272,
      "grad_norm": 7.30522346496582,
      "kl": 2.966796875,
      "learning_rate": 2.850858987219463e-07,
      "loss": 0.1995,
      "reward": 0.4899553805589676,
      "reward_std": 0.13860627822577953,
      "rewards/accuracy_reward": 0.035714287078008056,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4542410969734192,
      "step": 1706
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1671.4398193359375,
      "epoch": 0.5098947053991487,
      "grad_norm": 34.25748062133789,
      "kl": 2.87890625,
      "learning_rate": 2.848277040633602e-07,
      "loss": 0.2095,
      "reward": 0.4793526977300644,
      "reward_std": 0.12168661318719387,
      "rewards/accuracy_reward": 0.04464285937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098395228386,
      "step": 1707
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1629.7277526855469,
      "epoch": 0.5101934134866701,
      "grad_norm": 9.296675682067871,
      "kl": 2.958984375,
      "learning_rate": 2.845694715156456e-07,
      "loss": 0.19,
      "reward": 0.5334821715950966,
      "reward_std": 0.13663618452847004,
      "rewards/accuracy_reward": 0.080357147147879,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4531250149011612,
      "step": 1708
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1647.4286193847656,
      "epoch": 0.5104921215741917,
      "grad_norm": 15.512045860290527,
      "kl": 2.630859375,
      "learning_rate": 2.843112013597341e-07,
      "loss": 0.1849,
      "reward": 0.572544664144516,
      "reward_std": 0.1516516599804163,
      "rewards/accuracy_reward": 0.11383929220028222,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4587053805589676,
      "step": 1709
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1586.5782165527344,
      "epoch": 0.5107908296617131,
      "grad_norm": 31.277374267578125,
      "kl": 2.5703125,
      "learning_rate": 2.840528938765984e-07,
      "loss": 0.1766,
      "reward": 0.5970982387661934,
      "reward_std": 0.17505653202533722,
      "rewards/accuracy_reward": 0.1473214328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4497768059372902,
      "step": 1710
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1551.6094360351562,
      "epoch": 0.5110895377492346,
      "grad_norm": 13.63786792755127,
      "kl": 3.328125,
      "learning_rate": 2.8379454934725165e-07,
      "loss": 0.2456,
      "reward": 0.6858259290456772,
      "reward_std": 0.25161556527018547,
      "rewards/accuracy_reward": 0.238839291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.446986623108387,
      "step": 1711
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1576.1764221191406,
      "epoch": 0.511388245836756,
      "grad_norm": 12.431355476379395,
      "kl": 2.9453125,
      "learning_rate": 2.835361680527475e-07,
      "loss": 0.1949,
      "reward": 0.5463169813156128,
      "reward_std": 0.11601202562451363,
      "rewards/accuracy_reward": 0.0915178619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4547991305589676,
      "step": 1712
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1560.6005249023438,
      "epoch": 0.5116869539242775,
      "grad_norm": 33.89189147949219,
      "kl": 3.048828125,
      "learning_rate": 2.832777502741794e-07,
      "loss": 0.2441,
      "reward": 0.5301339402794838,
      "reward_std": 0.18215370550751686,
      "rewards/accuracy_reward": 0.0892857201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4408482313156128,
      "step": 1713
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1664.5067749023438,
      "epoch": 0.511985662011799,
      "grad_norm": 19.321317672729492,
      "kl": 3.712890625,
      "learning_rate": 2.830192962926806e-07,
      "loss": 0.2385,
      "reward": 0.4955357238650322,
      "reward_std": 0.14425735734403133,
      "rewards/accuracy_reward": 0.05803571571595967,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000149011612,
      "step": 1714
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1537.7478332519531,
      "epoch": 0.5122843700993205,
      "grad_norm": 7.224479675292969,
      "kl": 2.80078125,
      "learning_rate": 2.827608063894236e-07,
      "loss": 0.1833,
      "reward": 0.537388414144516,
      "reward_std": 0.14651411026716232,
      "rewards/accuracy_reward": 0.08258929010480642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4547991305589676,
      "step": 1715
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1571.1785888671875,
      "epoch": 0.5125830781868419,
      "grad_norm": 24.615938186645508,
      "kl": 3.48828125,
      "learning_rate": 2.825022808456201e-07,
      "loss": 0.2104,
      "reward": 0.5318080633878708,
      "reward_std": 0.17444274947047234,
      "rewards/accuracy_reward": 0.09375000605359674,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4380580559372902,
      "step": 1716
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1592.51123046875,
      "epoch": 0.5128817862743634,
      "grad_norm": 39.29931640625,
      "kl": 3.09765625,
      "learning_rate": 2.8224371994252073e-07,
      "loss": 0.2395,
      "reward": 0.5608259066939354,
      "reward_std": 0.16877485439181328,
      "rewards/accuracy_reward": 0.11383928917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4469866305589676,
      "step": 1717
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1660.5179443359375,
      "epoch": 0.5131804943618848,
      "grad_norm": 11.275134086608887,
      "kl": 3.24609375,
      "learning_rate": 2.8198512396141425e-07,
      "loss": 0.1945,
      "reward": 0.5245535969734192,
      "reward_std": 0.14664108119904995,
      "rewards/accuracy_reward": 0.07366071967408061,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4508928805589676,
      "step": 1718
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1545.8527221679688,
      "epoch": 0.5134792024494064,
      "grad_norm": 29.886648178100586,
      "kl": 3.39453125,
      "learning_rate": 2.8172649318362775e-07,
      "loss": 0.2544,
      "reward": 0.577008955180645,
      "reward_std": 0.15247288346290588,
      "rewards/accuracy_reward": 0.13839286309666932,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4386160895228386,
      "step": 1719
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1672.2188415527344,
      "epoch": 0.5137779105369278,
      "grad_norm": 18.022705078125,
      "kl": 4.46484375,
      "learning_rate": 2.8146782789052623e-07,
      "loss": 0.2832,
      "reward": 0.4531250149011612,
      "reward_std": 0.16237447038292885,
      "rewards/accuracy_reward": 0.02455357275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 1720
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1701.6384582519531,
      "epoch": 0.5140766186244493,
      "grad_norm": 53.16096496582031,
      "kl": 4.60546875,
      "learning_rate": 2.8120912836351216e-07,
      "loss": 0.2732,
      "reward": 0.474888414144516,
      "reward_std": 0.1595217101275921,
      "rewards/accuracy_reward": 0.049107145285233855,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812723517418,
      "step": 1721
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1698.8616943359375,
      "epoch": 0.5143753267119707,
      "grad_norm": 50.88163757324219,
      "kl": 4.02734375,
      "learning_rate": 2.8095039488402524e-07,
      "loss": 0.2784,
      "reward": 0.5011160969734192,
      "reward_std": 0.18065334483981133,
      "rewards/accuracy_reward": 0.07142857369035482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875149011612,
      "step": 1722
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1618.7545166015625,
      "epoch": 0.5146740347994921,
      "grad_norm": 18.025619506835938,
      "kl": 3.033203125,
      "learning_rate": 2.806916277335421e-07,
      "loss": 0.208,
      "reward": 0.5491071566939354,
      "reward_std": 0.12111184559762478,
      "rewards/accuracy_reward": 0.1049107164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4441964402794838,
      "step": 1723
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1696.9822387695312,
      "epoch": 0.5149727428870137,
      "grad_norm": 38.09000778198242,
      "kl": 4.83984375,
      "learning_rate": 2.8043282719357616e-07,
      "loss": 0.2816,
      "reward": 0.5156250223517418,
      "reward_std": 0.1665725726634264,
      "rewards/accuracy_reward": 0.10044643213041127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4151785895228386,
      "step": 1724
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1734.6473693847656,
      "epoch": 0.5152714509745351,
      "grad_norm": 49.2379035949707,
      "kl": 5.7109375,
      "learning_rate": 2.801739935456769e-07,
      "loss": 0.3535,
      "reward": 0.4860491305589676,
      "reward_std": 0.1667410135269165,
      "rewards/accuracy_reward": 0.06696428963914514,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4190848395228386,
      "step": 1725
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1572.7567749023438,
      "epoch": 0.5155701590620566,
      "grad_norm": 7.968415260314941,
      "kl": 3.87890625,
      "learning_rate": 2.7991512707143006e-07,
      "loss": 0.2729,
      "reward": 0.476004496216774,
      "reward_std": 0.15905269049108028,
      "rewards/accuracy_reward": 0.03794643096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4380580559372902,
      "step": 1726
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1592.9085388183594,
      "epoch": 0.515868867149578,
      "grad_norm": 12.466891288757324,
      "kl": 3.546875,
      "learning_rate": 2.7965622805245705e-07,
      "loss": 0.2417,
      "reward": 0.4832589402794838,
      "reward_std": 0.12968280166387558,
      "rewards/accuracy_reward": 0.04687500232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4363839477300644,
      "step": 1727
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1616.85498046875,
      "epoch": 0.5161675752370996,
      "grad_norm": 11.699602127075195,
      "kl": 4.33984375,
      "learning_rate": 2.7939729677041444e-07,
      "loss": 0.3027,
      "reward": 0.579799123108387,
      "reward_std": 0.1764991171658039,
      "rewards/accuracy_reward": 0.14955357648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455559372902,
      "step": 1728
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1674.2411499023438,
      "epoch": 0.516466283324621,
      "grad_norm": 39.4326171875,
      "kl": 4.29296875,
      "learning_rate": 2.791383335069942e-07,
      "loss": 0.2455,
      "reward": 0.4966518059372902,
      "reward_std": 0.1555672474205494,
      "rewards/accuracy_reward": 0.0803571455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.416294664144516,
      "step": 1729
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1530.18310546875,
      "epoch": 0.5167649914121425,
      "grad_norm": 32.17528533935547,
      "kl": 3.171875,
      "learning_rate": 2.78879338543923e-07,
      "loss": 0.2435,
      "reward": 0.6607143059372902,
      "reward_std": 0.13966858945786953,
      "rewards/accuracy_reward": 0.2075892984867096,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4531250223517418,
      "step": 1730
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1666.1786499023438,
      "epoch": 0.5170636994996639,
      "grad_norm": 34.014766693115234,
      "kl": 4.53515625,
      "learning_rate": 2.786203121629619e-07,
      "loss": 0.2797,
      "reward": 0.5323660895228386,
      "reward_std": 0.18930118530988693,
      "rewards/accuracy_reward": 0.0915178582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4408482387661934,
      "step": 1731
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1638.5670471191406,
      "epoch": 0.5173624075871854,
      "grad_norm": 45.9080810546875,
      "kl": 4.29296875,
      "learning_rate": 2.783612546459063e-07,
      "loss": 0.2373,
      "reward": 0.5044643133878708,
      "reward_std": 0.16251758486032486,
      "rewards/accuracy_reward": 0.0736607164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035895228386,
      "step": 1732
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1503.9754943847656,
      "epoch": 0.5176611156747069,
      "grad_norm": 36.12447738647461,
      "kl": 2.84765625,
      "learning_rate": 2.781021662745853e-07,
      "loss": 0.2299,
      "reward": 0.557477705180645,
      "reward_std": 0.17240197770297527,
      "rewards/accuracy_reward": 0.1116071455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4458705559372902,
      "step": 1733
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1577.555908203125,
      "epoch": 0.5179598237622284,
      "grad_norm": 14.236021995544434,
      "kl": 3.671875,
      "learning_rate": 2.7784304733086176e-07,
      "loss": 0.2521,
      "reward": 0.5809152126312256,
      "reward_std": 0.15254950150847435,
      "rewards/accuracy_reward": 0.145089291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4358259215950966,
      "step": 1734
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1567.4041137695312,
      "epoch": 0.5182585318497498,
      "grad_norm": 28.387414932250977,
      "kl": 3.453125,
      "learning_rate": 2.775838980966316e-07,
      "loss": 0.2288,
      "reward": 0.5290178805589676,
      "reward_std": 0.11452163383364677,
      "rewards/accuracy_reward": 0.08482143236324191,
      "rewards/format_reward": 0.0022321429569274187,
      "rewards/tag_count_reward": 0.4419643059372902,
      "step": 1735
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1717.0915832519531,
      "epoch": 0.5185572399372713,
      "grad_norm": 37.269073486328125,
      "kl": 3.8359375,
      "learning_rate": 2.7732471885382383e-07,
      "loss": 0.2295,
      "reward": 0.5106027126312256,
      "reward_std": 0.19032376818358898,
      "rewards/accuracy_reward": 0.07142857578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4391741305589676,
      "step": 1736
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1659.6652526855469,
      "epoch": 0.5188559480247927,
      "grad_norm": 61.883785247802734,
      "kl": 5.5,
      "learning_rate": 2.7706550988440006e-07,
      "loss": 0.3331,
      "reward": 0.435825914144516,
      "reward_std": 0.15172969177365303,
      "rewards/accuracy_reward": 0.020089285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.415736623108387,
      "step": 1737
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1615.8996276855469,
      "epoch": 0.5191546561123143,
      "grad_norm": 88.1395492553711,
      "kl": 4.96875,
      "learning_rate": 2.768062714703543e-07,
      "loss": 0.275,
      "reward": 0.4564732313156128,
      "reward_std": 0.14758897572755814,
      "rewards/accuracy_reward": 0.033482145285233855,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 1738
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1563.7031860351562,
      "epoch": 0.5194533641998357,
      "grad_norm": 67.6760482788086,
      "kl": 2.087890625,
      "learning_rate": 2.765470038937124e-07,
      "loss": 0.1871,
      "reward": 0.5357143059372902,
      "reward_std": 0.11059441789984703,
      "rewards/accuracy_reward": 0.08258928847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4531250223517418,
      "step": 1739
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1644.3951416015625,
      "epoch": 0.5197520722873572,
      "grad_norm": 13.766127586364746,
      "kl": 4.10546875,
      "learning_rate": 2.7628770743653213e-07,
      "loss": 0.3013,
      "reward": 0.5172991305589676,
      "reward_std": 0.15774049796164036,
      "rewards/accuracy_reward": 0.0915178598370403,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812649011612,
      "step": 1740
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1541.0558471679688,
      "epoch": 0.5200507803748786,
      "grad_norm": 54.18669891357422,
      "kl": 3.126953125,
      "learning_rate": 2.760283823809027e-07,
      "loss": 0.2774,
      "reward": 0.5452009215950966,
      "reward_std": 0.15356412529945374,
      "rewards/accuracy_reward": 0.1093750037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435825914144516,
      "step": 1741
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1541.3482971191406,
      "epoch": 0.5203494884624001,
      "grad_norm": 33.631744384765625,
      "kl": 3.3046875,
      "learning_rate": 2.7576902900894426e-07,
      "loss": 0.2612,
      "reward": 0.553013414144516,
      "reward_std": 0.20481225103139877,
      "rewards/accuracy_reward": 0.11383928917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4391741305589676,
      "step": 1742
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1546.7389221191406,
      "epoch": 0.5206481965499216,
      "grad_norm": 62.504573822021484,
      "kl": 2.25,
      "learning_rate": 2.7550964760280794e-07,
      "loss": 0.1813,
      "reward": 0.4626116305589676,
      "reward_std": 0.13229131791740656,
      "rewards/accuracy_reward": 0.0200892873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4425223469734192,
      "step": 1743
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1645.9710693359375,
      "epoch": 0.5209469046374431,
      "grad_norm": 34.03953170776367,
      "kl": 2.423828125,
      "learning_rate": 2.752502384446752e-07,
      "loss": 0.1714,
      "reward": 0.5117187649011612,
      "reward_std": 0.15512298047542572,
      "rewards/accuracy_reward": 0.06919643119908869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4425223469734192,
      "step": 1744
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1658.3170166015625,
      "epoch": 0.5212456127249645,
      "grad_norm": 50.95347595214844,
      "kl": 2.6328125,
      "learning_rate": 2.749908018167578e-07,
      "loss": 0.1869,
      "reward": 0.5111607313156128,
      "reward_std": 0.17601805552840233,
      "rewards/accuracy_reward": 0.07366071850992739,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000149011612,
      "step": 1745
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1671.5826416015625,
      "epoch": 0.521544320812486,
      "grad_norm": 11.679443359375,
      "kl": 3.56640625,
      "learning_rate": 2.7473133800129745e-07,
      "loss": 0.239,
      "reward": 0.6015625149011612,
      "reward_std": 0.1607975121587515,
      "rewards/accuracy_reward": 0.1785714365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 1746
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1572.1741943359375,
      "epoch": 0.5218430289000074,
      "grad_norm": 33.101806640625,
      "kl": 2.62109375,
      "learning_rate": 2.7447184728056524e-07,
      "loss": 0.2042,
      "reward": 0.529575914144516,
      "reward_std": 0.1589144691824913,
      "rewards/accuracy_reward": 0.08482143213041127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4447544887661934,
      "step": 1747
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1660.8795166015625,
      "epoch": 0.522141736987529,
      "grad_norm": 15.556615829467773,
      "kl": 4.015625,
      "learning_rate": 2.742123299368617e-07,
      "loss": 0.2553,
      "reward": 0.4263393059372902,
      "reward_std": 0.12485009990632534,
      "rewards/accuracy_reward": 0.006696428870782256,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4196428805589676,
      "step": 1748
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1575.8304443359375,
      "epoch": 0.5224404450750504,
      "grad_norm": 29.764310836791992,
      "kl": 4.43359375,
      "learning_rate": 2.7395278625251623e-07,
      "loss": 0.3109,
      "reward": 0.4804687723517418,
      "reward_std": 0.11687616631388664,
      "rewards/accuracy_reward": 0.044642859138548374,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4358259066939354,
      "step": 1749
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1554.02685546875,
      "epoch": 0.5227391531625719,
      "grad_norm": 35.57770919799805,
      "kl": 4.76953125,
      "learning_rate": 2.7369321650988697e-07,
      "loss": 0.3418,
      "reward": 0.4542410969734192,
      "reward_std": 0.15828418172895908,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 1750
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1601.8996276855469,
      "epoch": 0.5230378612500933,
      "grad_norm": 25.79292869567871,
      "kl": 3.5703125,
      "learning_rate": 2.7343362099136036e-07,
      "loss": 0.2452,
      "reward": 0.458705373108387,
      "reward_std": 0.09508886840194464,
      "rewards/accuracy_reward": 0.004464285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4542410895228386,
      "step": 1751
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1655.2590026855469,
      "epoch": 0.5233365693376149,
      "grad_norm": 52.47477722167969,
      "kl": 5.234375,
      "learning_rate": 2.731739999793508e-07,
      "loss": 0.3182,
      "reward": 0.5680803805589676,
      "reward_std": 0.2094019427895546,
      "rewards/accuracy_reward": 0.14508929289877415,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 1752
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1567.3661499023438,
      "epoch": 0.5236352774251363,
      "grad_norm": 130.18553161621094,
      "kl": 6.23828125,
      "learning_rate": 2.7291435375630057e-07,
      "loss": 0.3576,
      "reward": 0.5351562649011612,
      "reward_std": 0.16126592084765434,
      "rewards/accuracy_reward": 0.11160714668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4235491305589676,
      "step": 1753
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1635.18310546875,
      "epoch": 0.5239339855126578,
      "grad_norm": 78.28179168701172,
      "kl": 5.19921875,
      "learning_rate": 2.7265468260467934e-07,
      "loss": 0.3046,
      "reward": 0.4547991380095482,
      "reward_std": 0.16429843939840794,
      "rewards/accuracy_reward": 0.0334821455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4213169813156128,
      "step": 1754
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1624.4264221191406,
      "epoch": 0.5242326936001792,
      "grad_norm": 47.4450798034668,
      "kl": 4.3125,
      "learning_rate": 2.7239498680698375e-07,
      "loss": 0.2723,
      "reward": 0.4843750149011612,
      "reward_std": 0.13483248464763165,
      "rewards/accuracy_reward": 0.051339288242161274,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357387661934,
      "step": 1755
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1559.5692443847656,
      "epoch": 0.5245314016877007,
      "grad_norm": 8.70786190032959,
      "kl": 2.80859375,
      "learning_rate": 2.7213526664573756e-07,
      "loss": 0.1914,
      "reward": 0.5172991380095482,
      "reward_std": 0.15818733349442482,
      "rewards/accuracy_reward": 0.0758928619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4414062723517418,
      "step": 1756
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1633.9554138183594,
      "epoch": 0.5248301097752222,
      "grad_norm": 45.53723907470703,
      "kl": 4.5390625,
      "learning_rate": 2.718755224034907e-07,
      "loss": 0.2902,
      "reward": 0.5000000149011612,
      "reward_std": 0.15746591798961163,
      "rewards/accuracy_reward": 0.07589286286383867,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.424107164144516,
      "step": 1757
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1587.08935546875,
      "epoch": 0.5251288178627437,
      "grad_norm": 19.14122200012207,
      "kl": 3.46484375,
      "learning_rate": 2.716157543628195e-07,
      "loss": 0.2691,
      "reward": 0.5513392984867096,
      "reward_std": 0.15758421644568443,
      "rewards/accuracy_reward": 0.10491071757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4464285969734192,
      "step": 1758
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1595.6942749023438,
      "epoch": 0.5254275259502651,
      "grad_norm": 36.462257385253906,
      "kl": 4.75,
      "learning_rate": 2.713559628063261e-07,
      "loss": 0.3221,
      "reward": 0.521205373108387,
      "reward_std": 0.18046706169843674,
      "rewards/accuracy_reward": 0.0959821492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232238650322,
      "step": 1759
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1566.5647888183594,
      "epoch": 0.5257262340377866,
      "grad_norm": 29.310579299926758,
      "kl": 3.314453125,
      "learning_rate": 2.710961480166383e-07,
      "loss": 0.2121,
      "reward": 0.5770089626312256,
      "reward_std": 0.19902991876006126,
      "rewards/accuracy_reward": 0.12723214738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4497768133878708,
      "step": 1760
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1639.44873046875,
      "epoch": 0.526024942125308,
      "grad_norm": 10.898287773132324,
      "kl": 4.01171875,
      "learning_rate": 2.7083631027640916e-07,
      "loss": 0.2856,
      "reward": 0.5005580708384514,
      "reward_std": 0.1660266313701868,
      "rewards/accuracy_reward": 0.07142857578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294887661934,
      "step": 1761
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1616.0335693359375,
      "epoch": 0.5263236502128296,
      "grad_norm": 18.60226058959961,
      "kl": 3.251953125,
      "learning_rate": 2.705764498683165e-07,
      "loss": 0.2423,
      "reward": 0.5535714626312256,
      "reward_std": 0.1172682624310255,
      "rewards/accuracy_reward": 0.11830357555299997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4352678805589676,
      "step": 1762
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1685.0536499023438,
      "epoch": 0.526622358300351,
      "grad_norm": 28.885034561157227,
      "kl": 4.19140625,
      "learning_rate": 2.7031656707506315e-07,
      "loss": 0.2606,
      "reward": 0.518415205180645,
      "reward_std": 0.1853452157229185,
      "rewards/accuracy_reward": 0.10937500419095159,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4090401977300644,
      "step": 1763
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1596.6161193847656,
      "epoch": 0.5269210663878725,
      "grad_norm": 78.760986328125,
      "kl": 2.578125,
      "learning_rate": 2.7005666217937605e-07,
      "loss": 0.2221,
      "reward": 0.4737723395228386,
      "reward_std": 0.12881378643214703,
      "rewards/accuracy_reward": 0.04241071501746774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.431361623108387,
      "step": 1764
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1719.2054443359375,
      "epoch": 0.5272197744753939,
      "grad_norm": 8.007554054260254,
      "kl": 2.7734375,
      "learning_rate": 2.6979673546400635e-07,
      "loss": 0.1474,
      "reward": 0.479352705180645,
      "reward_std": 0.12781946547329426,
      "rewards/accuracy_reward": 0.0513392873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4280134066939354,
      "step": 1765
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1625.5893249511719,
      "epoch": 0.5275184825629153,
      "grad_norm": 31.861757278442383,
      "kl": 2.54296875,
      "learning_rate": 2.695367872117286e-07,
      "loss": 0.1666,
      "reward": 0.5117187649011612,
      "reward_std": 0.129623893648386,
      "rewards/accuracy_reward": 0.08035714668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4313616305589676,
      "step": 1766
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1652.7947387695312,
      "epoch": 0.5278171906504369,
      "grad_norm": 78.7196044921875,
      "kl": 2.525390625,
      "learning_rate": 2.6927681770534126e-07,
      "loss": 0.1863,
      "reward": 0.5189732387661934,
      "reward_std": 0.12173357605934143,
      "rewards/accuracy_reward": 0.0781250037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4408482387661934,
      "step": 1767
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1697.4420166015625,
      "epoch": 0.5281158987379583,
      "grad_norm": 58.66118621826172,
      "kl": 1.916015625,
      "learning_rate": 2.690168272276656e-07,
      "loss": 0.141,
      "reward": 0.4687500223517418,
      "reward_std": 0.1304073203355074,
      "rewards/accuracy_reward": 0.017857143422588706,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.450892873108387,
      "step": 1768
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1627.5223999023438,
      "epoch": 0.5284146068254798,
      "grad_norm": 62.1063346862793,
      "kl": 2.232421875,
      "learning_rate": 2.687568160615457e-07,
      "loss": 0.1542,
      "reward": 0.4988839477300644,
      "reward_std": 0.1047830106690526,
      "rewards/accuracy_reward": 0.05357143096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125223517418,
      "step": 1769
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1529.4397888183594,
      "epoch": 0.5287133149130012,
      "grad_norm": 57.12336349487305,
      "kl": 1.8828125,
      "learning_rate": 2.6849678448984835e-07,
      "loss": 0.1534,
      "reward": 0.515066996216774,
      "reward_std": 0.10974467173218727,
      "rewards/accuracy_reward": 0.0602678582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4547991380095482,
      "step": 1770
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1619.0447082519531,
      "epoch": 0.5290120230005227,
      "grad_norm": 16.558101654052734,
      "kl": 3.68359375,
      "learning_rate": 2.682367327954624e-07,
      "loss": 0.2529,
      "reward": 0.550781287252903,
      "reward_std": 0.14278215169906616,
      "rewards/accuracy_reward": 0.1227678656578064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428013414144516,
      "step": 1771
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1582.8036499023438,
      "epoch": 0.5293107310880442,
      "grad_norm": 41.929752349853516,
      "kl": 2.765625,
      "learning_rate": 2.6797666126129855e-07,
      "loss": 0.2302,
      "reward": 0.6238839626312256,
      "reward_std": 0.19059934839606285,
      "rewards/accuracy_reward": 0.1785714365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125298023224,
      "step": 1772
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1566.8058776855469,
      "epoch": 0.5296094391755657,
      "grad_norm": 10.776983261108398,
      "kl": 3.22265625,
      "learning_rate": 2.6771657017028906e-07,
      "loss": 0.2202,
      "reward": 0.568080373108387,
      "reward_std": 0.1851207111030817,
      "rewards/accuracy_reward": 0.12500000465661287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4430803805589676,
      "step": 1773
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1580.0781555175781,
      "epoch": 0.5299081472630871,
      "grad_norm": 49.52845764160156,
      "kl": 4.091796875,
      "learning_rate": 2.674564598053877e-07,
      "loss": 0.2446,
      "reward": 0.517857164144516,
      "reward_std": 0.16952192783355713,
      "rewards/accuracy_reward": 0.082589291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4352678805589676,
      "step": 1774
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1635.1808776855469,
      "epoch": 0.5302068553506086,
      "grad_norm": 25.246688842773438,
      "kl": 3.8203125,
      "learning_rate": 2.6719633044956897e-07,
      "loss": 0.2524,
      "reward": 0.5150669738650322,
      "reward_std": 0.17731483653187752,
      "rewards/accuracy_reward": 0.07366071757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4414062723517418,
      "step": 1775
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1679.2098999023438,
      "epoch": 0.53050556343813,
      "grad_norm": 11.587535858154297,
      "kl": 4.140625,
      "learning_rate": 2.6693618238582806e-07,
      "loss": 0.2558,
      "reward": 0.5401786044239998,
      "reward_std": 0.19198604673147202,
      "rewards/accuracy_reward": 0.10937500279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035969734192,
      "step": 1776
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1644.3304443359375,
      "epoch": 0.5308042715256516,
      "grad_norm": 21.94377326965332,
      "kl": 3.765625,
      "learning_rate": 2.6667601589718074e-07,
      "loss": 0.2302,
      "reward": 0.6054687798023224,
      "reward_std": 0.1847195401787758,
      "rewards/accuracy_reward": 0.1562500111758709,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4492187723517418,
      "step": 1777
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1608.6965026855469,
      "epoch": 0.531102979613173,
      "grad_norm": 16.150959014892578,
      "kl": 3.73828125,
      "learning_rate": 2.664158312666625e-07,
      "loss": 0.2477,
      "reward": 0.553571455180645,
      "reward_std": 0.19976021721959114,
      "rewards/accuracy_reward": 0.11383928963914514,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.439732164144516,
      "step": 1778
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1655.8750915527344,
      "epoch": 0.5314016877006945,
      "grad_norm": 69.86961364746094,
      "kl": 4.93359375,
      "learning_rate": 2.661556287773288e-07,
      "loss": 0.3018,
      "reward": 0.5613839402794838,
      "reward_std": 0.13027407601475716,
      "rewards/accuracy_reward": 0.1361607201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232387661934,
      "step": 1779
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1650.6965026855469,
      "epoch": 0.5317003957882159,
      "grad_norm": 22.816179275512695,
      "kl": 4.3046875,
      "learning_rate": 2.6589540871225437e-07,
      "loss": 0.2419,
      "reward": 0.4720982387661934,
      "reward_std": 0.1627504676580429,
      "rewards/accuracy_reward": 0.0357142873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.436383955180645,
      "step": 1780
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1605.0759887695312,
      "epoch": 0.5319991038757375,
      "grad_norm": 11.295478820800781,
      "kl": 3.40625,
      "learning_rate": 2.656351713545332e-07,
      "loss": 0.2377,
      "reward": 0.525669664144516,
      "reward_std": 0.1638424601405859,
      "rewards/accuracy_reward": 0.08705357578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4386160969734192,
      "step": 1781
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1585.0469665527344,
      "epoch": 0.5322978119632589,
      "grad_norm": 45.36103439331055,
      "kl": 3.875,
      "learning_rate": 2.6537491698727804e-07,
      "loss": 0.2221,
      "reward": 0.5753348395228386,
      "reward_std": 0.14683369547128677,
      "rewards/accuracy_reward": 0.13616071757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.439174123108387,
      "step": 1782
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1674.0938110351562,
      "epoch": 0.5325965200507804,
      "grad_norm": 27.492841720581055,
      "kl": 3.544921875,
      "learning_rate": 2.6511464589362006e-07,
      "loss": 0.2075,
      "reward": 0.5033482313156128,
      "reward_std": 0.1878223568201065,
      "rewards/accuracy_reward": 0.0625000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4408482387661934,
      "step": 1783
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1716.3148193359375,
      "epoch": 0.5328952281383018,
      "grad_norm": 14.676650047302246,
      "kl": 3.58984375,
      "learning_rate": 2.648543583567088e-07,
      "loss": 0.2355,
      "reward": 0.518973246216774,
      "reward_std": 0.16148994863033295,
      "rewards/accuracy_reward": 0.0758928619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4430803805589676,
      "step": 1784
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1553.9130249023438,
      "epoch": 0.5331939362258233,
      "grad_norm": 7.853994369506836,
      "kl": 2.873046875,
      "learning_rate": 2.6459405465971146e-07,
      "loss": 0.1916,
      "reward": 0.6406250447034836,
      "reward_std": 0.1768389791250229,
      "rewards/accuracy_reward": 0.1919642984867096,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4486607313156128,
      "step": 1785
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1546.9889221191406,
      "epoch": 0.5334926443133448,
      "grad_norm": 78.41841888427734,
      "kl": 1.84375,
      "learning_rate": 2.643337350858131e-07,
      "loss": 0.14,
      "reward": 0.552455373108387,
      "reward_std": 0.12744706496596336,
      "rewards/accuracy_reward": 0.1004464328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4520089477300644,
      "step": 1786
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1597.0246276855469,
      "epoch": 0.5337913524008663,
      "grad_norm": 74.7909927368164,
      "kl": 1.75390625,
      "learning_rate": 2.640733999182157e-07,
      "loss": 0.1445,
      "reward": 0.6093750447034836,
      "reward_std": 0.15403219126164913,
      "rewards/accuracy_reward": 0.15625000558793545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4531250223517418,
      "step": 1787
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1664.8750610351562,
      "epoch": 0.5340900604883877,
      "grad_norm": 96.9845962524414,
      "kl": 1.9765625,
      "learning_rate": 2.638130494401386e-07,
      "loss": 0.1649,
      "reward": 0.5742187649011612,
      "reward_std": 0.15029161609709263,
      "rewards/accuracy_reward": 0.13392857648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.440290205180645,
      "step": 1788
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1559.96435546875,
      "epoch": 0.5343887685759092,
      "grad_norm": 61.88276290893555,
      "kl": 1.556640625,
      "learning_rate": 2.6355268393481753e-07,
      "loss": 0.1277,
      "reward": 0.5279018208384514,
      "reward_std": 0.0803921278566122,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4564732387661934,
      "step": 1789
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1545.7857666015625,
      "epoch": 0.5346874766634306,
      "grad_norm": 37.51866912841797,
      "kl": 1.7568359375,
      "learning_rate": 2.632923036855046e-07,
      "loss": 0.1401,
      "reward": 0.5569196715950966,
      "reward_std": 0.12648136168718338,
      "rewards/accuracy_reward": 0.09821428824216127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.458705373108387,
      "step": 1790
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1606.5246276855469,
      "epoch": 0.5349861847509522,
      "grad_norm": 78.58927154541016,
      "kl": 1.5546875,
      "learning_rate": 2.6303190897546816e-07,
      "loss": 0.1472,
      "reward": 0.5362723544239998,
      "reward_std": 0.15003644116222858,
      "rewards/accuracy_reward": 0.0848214328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4514509066939354,
      "step": 1791
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1623.5402221679688,
      "epoch": 0.5352848928384736,
      "grad_norm": 31.3472957611084,
      "kl": 2.47265625,
      "learning_rate": 2.6277150008799196e-07,
      "loss": 0.1761,
      "reward": 0.4994419887661934,
      "reward_std": 0.16809764504432678,
      "rewards/accuracy_reward": 0.060267860535532236,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.439174123108387,
      "step": 1792
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1642.4286499023438,
      "epoch": 0.5355836009259951,
      "grad_norm": 55.71942138671875,
      "kl": 2.13671875,
      "learning_rate": 2.625110773063754e-07,
      "loss": 0.1594,
      "reward": 0.6093750223517418,
      "reward_std": 0.17745032906532288,
      "rewards/accuracy_reward": 0.16741072246804833,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4419643059372902,
      "step": 1793
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1618.5313110351562,
      "epoch": 0.5358823090135165,
      "grad_norm": 24.259843826293945,
      "kl": 2.091796875,
      "learning_rate": 2.6225064091393293e-07,
      "loss": 0.1335,
      "reward": 0.5239955559372902,
      "reward_std": 0.14079313911497593,
      "rewards/accuracy_reward": 0.0758928619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4481026977300644,
      "step": 1794
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1639.94873046875,
      "epoch": 0.536181017101038,
      "grad_norm": 64.32665252685547,
      "kl": 1.8515625,
      "learning_rate": 2.619901911939938e-07,
      "loss": 0.1563,
      "reward": 0.6729911044239998,
      "reward_std": 0.20768187940120697,
      "rewards/accuracy_reward": 0.2299107275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4430803805589676,
      "step": 1795
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1593.1406860351562,
      "epoch": 0.5364797251885595,
      "grad_norm": 48.83095169067383,
      "kl": 2.93359375,
      "learning_rate": 2.61729728429902e-07,
      "loss": 0.2512,
      "reward": 0.541852705180645,
      "reward_std": 0.2201291024684906,
      "rewards/accuracy_reward": 0.09598214831203222,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4458705633878708,
      "step": 1796
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1636.1295471191406,
      "epoch": 0.536778433276081,
      "grad_norm": 24.956937789916992,
      "kl": 4.33203125,
      "learning_rate": 2.614692529050151e-07,
      "loss": 0.2768,
      "reward": 0.5306919887661934,
      "reward_std": 0.16055604070425034,
      "rewards/accuracy_reward": 0.09821429080329835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324777126312256,
      "step": 1797
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1672.77685546875,
      "epoch": 0.5370771413636024,
      "grad_norm": 31.89096450805664,
      "kl": 4.02734375,
      "learning_rate": 2.612087649027052e-07,
      "loss": 0.231,
      "reward": 0.541294664144516,
      "reward_std": 0.13896547071635723,
      "rewards/accuracy_reward": 0.09821428917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443080373108387,
      "step": 1798
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1641.6764221191406,
      "epoch": 0.5373758494511239,
      "grad_norm": 67.78101348876953,
      "kl": 4.837890625,
      "learning_rate": 2.6094826470635774e-07,
      "loss": 0.2667,
      "reward": 0.5161830708384514,
      "reward_std": 0.15377869084477425,
      "rewards/accuracy_reward": 0.07589285937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4402901902794838,
      "step": 1799
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1723.4443054199219,
      "epoch": 0.5376745575386453,
      "grad_norm": 16.118839263916016,
      "kl": 4.24609375,
      "learning_rate": 2.606877525993713e-07,
      "loss": 0.2614,
      "reward": 0.5061384066939354,
      "reward_std": 0.17579124122858047,
      "rewards/accuracy_reward": 0.07589286053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455484867096,
      "step": 1800
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1683.1630249023438,
      "epoch": 0.5379732656261669,
      "grad_norm": 108.90744018554688,
      "kl": 5.86328125,
      "learning_rate": 2.6042722886515745e-07,
      "loss": 0.3297,
      "reward": 0.5133928880095482,
      "reward_std": 0.15270156040787697,
      "rewards/accuracy_reward": 0.0892857164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.424107164144516,
      "step": 1801
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1720.5648193359375,
      "epoch": 0.5382719737136883,
      "grad_norm": 65.82952880859375,
      "kl": 5.421875,
      "learning_rate": 2.601666937871407e-07,
      "loss": 0.3045,
      "reward": 0.5284598469734192,
      "reward_std": 0.16147832199931145,
      "rewards/accuracy_reward": 0.1004464328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4280134066939354,
      "step": 1802
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1661.1898193359375,
      "epoch": 0.5385706818012098,
      "grad_norm": 87.2514419555664,
      "kl": 5.205078125,
      "learning_rate": 2.5990614764875747e-07,
      "loss": 0.3195,
      "reward": 0.4949776902794838,
      "reward_std": 0.1483394019305706,
      "rewards/accuracy_reward": 0.06250000232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776977300644,
      "step": 1803
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1643.3840026855469,
      "epoch": 0.5388693898887312,
      "grad_norm": 51.15807342529297,
      "kl": 4.94921875,
      "learning_rate": 2.5964559073345654e-07,
      "loss": 0.2927,
      "reward": 0.450892873108387,
      "reward_std": 0.14215877279639244,
      "rewards/accuracy_reward": 0.01785714295692742,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357313156128,
      "step": 1804
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1587.4643249511719,
      "epoch": 0.5391680979762528,
      "grad_norm": 38.97306823730469,
      "kl": 2.81640625,
      "learning_rate": 2.5938502332469827e-07,
      "loss": 0.2283,
      "reward": 0.4525669887661934,
      "reward_std": 0.10540596209466457,
      "rewards/accuracy_reward": 0.0022321429569274187,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4503348395228386,
      "step": 1805
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1710.3460388183594,
      "epoch": 0.5394668060637742,
      "grad_norm": 64.27703857421875,
      "kl": 2.419921875,
      "learning_rate": 2.5912444570595435e-07,
      "loss": 0.1645,
      "reward": 0.5111607313156128,
      "reward_std": 0.14220664277672768,
      "rewards/accuracy_reward": 0.07366071944124997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000149011612,
      "step": 1806
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1712.930908203125,
      "epoch": 0.5397655141512957,
      "grad_norm": 49.46287155151367,
      "kl": 3.228515625,
      "learning_rate": 2.5886385816070793e-07,
      "loss": 0.2212,
      "reward": 0.5033482313156128,
      "reward_std": 0.2209962196648121,
      "rewards/accuracy_reward": 0.0915178619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4118303805589676,
      "step": 1807
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1676.7009582519531,
      "epoch": 0.5400642222388171,
      "grad_norm": 42.49440002441406,
      "kl": 4.38671875,
      "learning_rate": 2.586032609724525e-07,
      "loss": 0.2492,
      "reward": 0.478794664144516,
      "reward_std": 0.16028334572911263,
      "rewards/accuracy_reward": 0.07812500465661287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4006696566939354,
      "step": 1808
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1540.04248046875,
      "epoch": 0.5403629303263385,
      "grad_norm": 54.58527374267578,
      "kl": 2.5537109375,
      "learning_rate": 2.583426544246924e-07,
      "loss": 0.2039,
      "reward": 0.494419664144516,
      "reward_std": 0.11774345673620701,
      "rewards/accuracy_reward": 0.044642857974395156,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4497768059372902,
      "step": 1809
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1564.2701416015625,
      "epoch": 0.5406616384138601,
      "grad_norm": 10.956871032714844,
      "kl": 2.77734375,
      "learning_rate": 2.5808203880094194e-07,
      "loss": 0.1744,
      "reward": 0.4877232387661934,
      "reward_std": 0.13426114991307259,
      "rewards/accuracy_reward": 0.035714286379516125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4520089477300644,
      "step": 1810
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1650.5112609863281,
      "epoch": 0.5409603465013815,
      "grad_norm": 45.98472595214844,
      "kl": 2.306640625,
      "learning_rate": 2.578214143847254e-07,
      "loss": 0.1646,
      "reward": 0.5591518133878708,
      "reward_std": 0.11571215279400349,
      "rewards/accuracy_reward": 0.11160714644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.447544664144516,
      "step": 1811
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1584.0246276855469,
      "epoch": 0.541259054588903,
      "grad_norm": 23.514137268066406,
      "kl": 2.82421875,
      "learning_rate": 2.575607814595765e-07,
      "loss": 0.1945,
      "reward": 0.4765625223517418,
      "reward_std": 0.12958474270999432,
      "rewards/accuracy_reward": 0.026785716181620955,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4497768133878708,
      "step": 1812
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1564.1250610351562,
      "epoch": 0.5415577626764244,
      "grad_norm": 12.7684965133667,
      "kl": 3.080078125,
      "learning_rate": 2.5730014030903853e-07,
      "loss": 0.2152,
      "reward": 0.5251116305589676,
      "reward_std": 0.1270734705030918,
      "rewards/accuracy_reward": 0.07366071757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.451450914144516,
      "step": 1813
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1623.2611999511719,
      "epoch": 0.5418564707639459,
      "grad_norm": 6.009121894836426,
      "kl": 2.923828125,
      "learning_rate": 2.570394912166633e-07,
      "loss": 0.1813,
      "reward": 0.4916294813156128,
      "reward_std": 0.12303758598864079,
      "rewards/accuracy_reward": 0.053571431431919336,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4380580633878708,
      "step": 1814
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1545.8683776855469,
      "epoch": 0.5421551788514674,
      "grad_norm": 26.26144027709961,
      "kl": 3.640625,
      "learning_rate": 2.5677883446601154e-07,
      "loss": 0.2689,
      "reward": 0.5535714477300644,
      "reward_std": 0.19417370855808258,
      "rewards/accuracy_reward": 0.10937500186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4441964477300644,
      "step": 1815
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1677.259033203125,
      "epoch": 0.5424538869389889,
      "grad_norm": 71.92900848388672,
      "kl": 1.599609375,
      "learning_rate": 2.565181703406524e-07,
      "loss": 0.1251,
      "reward": 0.4871651977300644,
      "reward_std": 0.14847573451697826,
      "rewards/accuracy_reward": 0.04017857275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4469866305589676,
      "step": 1816
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1644.9688415527344,
      "epoch": 0.5427525950265103,
      "grad_norm": 10.076800346374512,
      "kl": 3.14453125,
      "learning_rate": 2.562574991241627e-07,
      "loss": 0.1983,
      "reward": 0.5295759215950966,
      "reward_std": 0.12321187369525433,
      "rewards/accuracy_reward": 0.0848214328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4447544813156128,
      "step": 1817
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1635.9040832519531,
      "epoch": 0.5430513031140318,
      "grad_norm": 34.617576599121094,
      "kl": 2.5078125,
      "learning_rate": 2.559968211001273e-07,
      "loss": 0.1768,
      "reward": 0.6021205559372902,
      "reward_std": 0.1867583505809307,
      "rewards/accuracy_reward": 0.160714291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4414062723517418,
      "step": 1818
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1648.9397888183594,
      "epoch": 0.5433500112015532,
      "grad_norm": 38.862770080566406,
      "kl": 2.53125,
      "learning_rate": 2.5573613655213827e-07,
      "loss": 0.1806,
      "reward": 0.5033482313156128,
      "reward_std": 0.1319346632808447,
      "rewards/accuracy_reward": 0.058035716880112886,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125223517418,
      "step": 1819
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1646.62060546875,
      "epoch": 0.5436487192890748,
      "grad_norm": 15.36838436126709,
      "kl": 2.6953125,
      "learning_rate": 2.5547544576379486e-07,
      "loss": 0.178,
      "reward": 0.5005580559372902,
      "reward_std": 0.11358188092708588,
      "rewards/accuracy_reward": 0.04687500209547579,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4536830559372902,
      "step": 1820
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1661.5603637695312,
      "epoch": 0.5439474273765962,
      "grad_norm": 27.059324264526367,
      "kl": 2.822265625,
      "learning_rate": 2.5521474901870316e-07,
      "loss": 0.1949,
      "reward": 0.601562537252903,
      "reward_std": 0.1394155267626047,
      "rewards/accuracy_reward": 0.15848215110599995,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4430803805589676,
      "step": 1821
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1695.7925109863281,
      "epoch": 0.5442461354641177,
      "grad_norm": 27.44449806213379,
      "kl": 2.8359375,
      "learning_rate": 2.549540466004756e-07,
      "loss": 0.1814,
      "reward": 0.5318080708384514,
      "reward_std": 0.1326269581913948,
      "rewards/accuracy_reward": 0.08482143259607255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4469866305589676,
      "step": 1822
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1545.0223999023438,
      "epoch": 0.5445448435516391,
      "grad_norm": 17.120256423950195,
      "kl": 2.6953125,
      "learning_rate": 2.546933387927309e-07,
      "loss": 0.1817,
      "reward": 0.5050223469734192,
      "reward_std": 0.12700673565268517,
      "rewards/accuracy_reward": 0.06250000302679837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4425223544239998,
      "step": 1823
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1569.5960388183594,
      "epoch": 0.5448435516391607,
      "grad_norm": 11.614546775817871,
      "kl": 3.50390625,
      "learning_rate": 2.544326258790936e-07,
      "loss": 0.2327,
      "reward": 0.5323660895228386,
      "reward_std": 0.14016331732273102,
      "rewards/accuracy_reward": 0.09598214598372579,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4363839477300644,
      "step": 1824
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1629.2054138183594,
      "epoch": 0.5451422597266821,
      "grad_norm": 24.148929595947266,
      "kl": 3.4921875,
      "learning_rate": 2.541719081431937e-07,
      "loss": 0.203,
      "reward": 0.6060268133878708,
      "reward_std": 0.1510606463998556,
      "rewards/accuracy_reward": 0.1562500069849193,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4497768059372902,
      "step": 1825
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1592.19873046875,
      "epoch": 0.5454409678142036,
      "grad_norm": 61.79855728149414,
      "kl": 4.50390625,
      "learning_rate": 2.539111858686667e-07,
      "loss": 0.2826,
      "reward": 0.4977678805589676,
      "reward_std": 0.1696145497262478,
      "rewards/accuracy_reward": 0.06696429057046771,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035895228386,
      "step": 1826
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1584.0447082519531,
      "epoch": 0.545739675901725,
      "grad_norm": 21.755510330200195,
      "kl": 4.0,
      "learning_rate": 2.536504593391528e-07,
      "loss": 0.2586,
      "reward": 0.541852705180645,
      "reward_std": 0.13907089829444885,
      "rewards/accuracy_reward": 0.0959821492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4458705559372902,
      "step": 1827
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1642.5245971679688,
      "epoch": 0.5460383839892465,
      "grad_norm": 15.096441268920898,
      "kl": 2.96484375,
      "learning_rate": 2.5338972883829695e-07,
      "loss": 0.2013,
      "reward": 0.5959821492433548,
      "reward_std": 0.1694699563086033,
      "rewards/accuracy_reward": 0.14955357927829027,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4464285895228386,
      "step": 1828
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1636.8058471679688,
      "epoch": 0.546337092076768,
      "grad_norm": 53.58095169067383,
      "kl": 3.02734375,
      "learning_rate": 2.531289946497483e-07,
      "loss": 0.2164,
      "reward": 0.4642857313156128,
      "reward_std": 0.14818907342851162,
      "rewards/accuracy_reward": 0.03125000232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357387661934,
      "step": 1829
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1627.1942749023438,
      "epoch": 0.5466358001642895,
      "grad_norm": 16.86935043334961,
      "kl": 3.25390625,
      "learning_rate": 2.528682570571602e-07,
      "loss": 0.2179,
      "reward": 0.5954241380095482,
      "reward_std": 0.15624183230102062,
      "rewards/accuracy_reward": 0.15178571734577417,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443638414144516,
      "step": 1830
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1615.5670471191406,
      "epoch": 0.5469345082518109,
      "grad_norm": 6.735917091369629,
      "kl": 3.109375,
      "learning_rate": 2.5260751634418953e-07,
      "loss": 0.2046,
      "reward": 0.5625000223517418,
      "reward_std": 0.15686860121786594,
      "rewards/accuracy_reward": 0.11830357648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.444196455180645,
      "step": 1831
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1664.9241638183594,
      "epoch": 0.5472332163393324,
      "grad_norm": 10.945684432983398,
      "kl": 3.36328125,
      "learning_rate": 2.523467727944966e-07,
      "loss": 0.2219,
      "reward": 0.5474330484867096,
      "reward_std": 0.1896630898118019,
      "rewards/accuracy_reward": 0.11160714668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4358259066939354,
      "step": 1832
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1601.0157165527344,
      "epoch": 0.5475319244268538,
      "grad_norm": 89.59307098388672,
      "kl": 5.30078125,
      "learning_rate": 2.520860266917449e-07,
      "loss": 0.2999,
      "reward": 0.498325914144516,
      "reward_std": 0.14390374720096588,
      "rewards/accuracy_reward": 0.06026785867288709,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4380580559372902,
      "step": 1833
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1593.9844360351562,
      "epoch": 0.5478306325143754,
      "grad_norm": 16.7200870513916,
      "kl": 3.7734375,
      "learning_rate": 2.518252783196006e-07,
      "loss": 0.2471,
      "reward": 0.5340401977300644,
      "reward_std": 0.14667906612157822,
      "rewards/accuracy_reward": 0.07812500279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.455915205180645,
      "step": 1834
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1592.2857666015625,
      "epoch": 0.5481293406018968,
      "grad_norm": 8.380125999450684,
      "kl": 3.8984375,
      "learning_rate": 2.5156452796173245e-07,
      "loss": 0.2639,
      "reward": 0.5290178805589676,
      "reward_std": 0.1842203363776207,
      "rewards/accuracy_reward": 0.08258928847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4464285969734192,
      "step": 1835
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1547.9688110351562,
      "epoch": 0.5484280486894183,
      "grad_norm": 7.6431474685668945,
      "kl": 2.98828125,
      "learning_rate": 2.513037759018111e-07,
      "loss": 0.1865,
      "reward": 0.6104910969734192,
      "reward_std": 0.14610681496560574,
      "rewards/accuracy_reward": 0.15848214668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.452008955180645,
      "step": 1836
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1620.4933776855469,
      "epoch": 0.5487267567769397,
      "grad_norm": 26.614700317382812,
      "kl": 3.75,
      "learning_rate": 2.5104302242350944e-07,
      "loss": 0.2221,
      "reward": 0.5563616305589676,
      "reward_std": 0.14477082900702953,
      "rewards/accuracy_reward": 0.10937500558793545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4469866305589676,
      "step": 1837
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1706.9486999511719,
      "epoch": 0.5490254648644612,
      "grad_norm": 10.748540878295898,
      "kl": 2.8984375,
      "learning_rate": 2.507822678105016e-07,
      "loss": 0.1694,
      "reward": 0.473772332072258,
      "reward_std": 0.11917665787041187,
      "rewards/accuracy_reward": 0.017857144121080637,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4559151977300644,
      "step": 1838
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1714.2165832519531,
      "epoch": 0.5493241729519827,
      "grad_norm": 26.863054275512695,
      "kl": 1.998046875,
      "learning_rate": 2.505215123464631e-07,
      "loss": 0.1192,
      "reward": 0.5412946715950966,
      "reward_std": 0.13468840345740318,
      "rewards/accuracy_reward": 0.08482143213041127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4564732387661934,
      "step": 1839
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1635.7701721191406,
      "epoch": 0.5496228810395042,
      "grad_norm": 26.842126846313477,
      "kl": 2.609375,
      "learning_rate": 2.5026075631507017e-07,
      "loss": 0.1794,
      "reward": 0.568638414144516,
      "reward_std": 0.1791628785431385,
      "rewards/accuracy_reward": 0.12053572200238705,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.448102705180645,
      "step": 1840
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1669.4397888183594,
      "epoch": 0.5499215891270256,
      "grad_norm": 42.67668914794922,
      "kl": 3.05859375,
      "learning_rate": 2.5e-07,
      "loss": 0.2283,
      "reward": 0.5357143133878708,
      "reward_std": 0.13251677341759205,
      "rewards/accuracy_reward": 0.09821429033763707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000149011612,
      "step": 1841
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1524.57373046875,
      "epoch": 0.5502202972145471,
      "grad_norm": 58.52946090698242,
      "kl": 2.26171875,
      "learning_rate": 2.497392436849298e-07,
      "loss": 0.1894,
      "reward": 0.5608259066939354,
      "reward_std": 0.16248693317174911,
      "rewards/accuracy_reward": 0.10267857578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4581473395228386,
      "step": 1842
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1541.7589721679688,
      "epoch": 0.5505190053020685,
      "grad_norm": 64.95293426513672,
      "kl": 2.392578125,
      "learning_rate": 2.494784876535369e-07,
      "loss": 0.1963,
      "reward": 0.5535714402794838,
      "reward_std": 0.15209194645285606,
      "rewards/accuracy_reward": 0.1049107201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4486607313156128,
      "step": 1843
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1655.0268249511719,
      "epoch": 0.5508177133895901,
      "grad_norm": 40.915008544921875,
      "kl": 2.65625,
      "learning_rate": 2.4921773218949835e-07,
      "loss": 0.1834,
      "reward": 0.513950914144516,
      "reward_std": 0.1428381036967039,
      "rewards/accuracy_reward": 0.07366071664728224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.440290205180645,
      "step": 1844
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1582.3259582519531,
      "epoch": 0.5511164214771115,
      "grad_norm": 25.71390151977539,
      "kl": 2.1171875,
      "learning_rate": 2.4895697757649054e-07,
      "loss": 0.1485,
      "reward": 0.5239955559372902,
      "reward_std": 0.14202952571213245,
      "rewards/accuracy_reward": 0.0691964328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4547991305589676,
      "step": 1845
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1704.4844665527344,
      "epoch": 0.551415129564633,
      "grad_norm": 26.884929656982422,
      "kl": 3.26953125,
      "learning_rate": 2.4869622409818886e-07,
      "loss": 0.189,
      "reward": 0.5301339477300644,
      "reward_std": 0.13444687239825726,
      "rewards/accuracy_reward": 0.08705357322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443080373108387,
      "step": 1846
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1716.9174499511719,
      "epoch": 0.5517138376521544,
      "grad_norm": 28.488710403442383,
      "kl": 2.6875,
      "learning_rate": 2.484354720382676e-07,
      "loss": 0.1728,
      "reward": 0.5747767984867096,
      "reward_std": 0.14004367589950562,
      "rewards/accuracy_reward": 0.1227678656578064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4520089477300644,
      "step": 1847
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1605.57373046875,
      "epoch": 0.552012545739676,
      "grad_norm": 37.16767501831055,
      "kl": 3.12109375,
      "learning_rate": 2.481747216803994e-07,
      "loss": 0.1583,
      "reward": 0.5195312723517418,
      "reward_std": 0.15428761951625347,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4481026977300644,
      "step": 1848
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1595.8996276855469,
      "epoch": 0.5523112538271974,
      "grad_norm": 25.365875244140625,
      "kl": 4.1328125,
      "learning_rate": 2.479139733082551e-07,
      "loss": 0.2891,
      "reward": 0.541294664144516,
      "reward_std": 0.1736100148409605,
      "rewards/accuracy_reward": 0.10491072130389512,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4363839477300644,
      "step": 1849
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1669.8304443359375,
      "epoch": 0.5526099619147189,
      "grad_norm": 87.41926574707031,
      "kl": 5.58203125,
      "learning_rate": 2.476532272055034e-07,
      "loss": 0.3359,
      "reward": 0.5161830708384514,
      "reward_std": 0.17138975113630295,
      "rewards/accuracy_reward": 0.08482143306173384,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.431361623108387,
      "step": 1850
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1650.7433776855469,
      "epoch": 0.5529086700022403,
      "grad_norm": 16.201539993286133,
      "kl": 3.92578125,
      "learning_rate": 2.473924836558105e-07,
      "loss": 0.2465,
      "reward": 0.5440848395228386,
      "reward_std": 0.12630446441471577,
      "rewards/accuracy_reward": 0.113839291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455559372902,
      "step": 1851
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1714.3170776367188,
      "epoch": 0.5532073780897617,
      "grad_norm": 56.88924789428711,
      "kl": 5.2265625,
      "learning_rate": 2.471317429428398e-07,
      "loss": 0.3168,
      "reward": 0.5239955559372902,
      "reward_std": 0.137503856793046,
      "rewards/accuracy_reward": 0.08928571850992739,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098395228386,
      "step": 1852
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1695.5313110351562,
      "epoch": 0.5535060861772833,
      "grad_norm": 61.337310791015625,
      "kl": 5.03125,
      "learning_rate": 2.468710053502517e-07,
      "loss": 0.2693,
      "reward": 0.5234375223517418,
      "reward_std": 0.14771644212305546,
      "rewards/accuracy_reward": 0.10267857648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4207589477300644,
      "step": 1853
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1600.4911499023438,
      "epoch": 0.5538047942648047,
      "grad_norm": 40.9768180847168,
      "kl": 4.53515625,
      "learning_rate": 2.46610271161703e-07,
      "loss": 0.2821,
      "reward": 0.6021205633878708,
      "reward_std": 0.1776778493076563,
      "rewards/accuracy_reward": 0.16517857741564512,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419813156128,
      "step": 1854
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1585.5960388183594,
      "epoch": 0.5541035023523262,
      "grad_norm": 5.329873561859131,
      "kl": 3.0234375,
      "learning_rate": 2.463495406608472e-07,
      "loss": 0.2083,
      "reward": 0.594308078289032,
      "reward_std": 0.16510101221501827,
      "rewards/accuracy_reward": 0.14062500488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4536830559372902,
      "step": 1855
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1721.15185546875,
      "epoch": 0.5544022104398476,
      "grad_norm": 13.018850326538086,
      "kl": 4.27734375,
      "learning_rate": 2.460888141313333e-07,
      "loss": 0.2536,
      "reward": 0.4369419813156128,
      "reward_std": 0.14147493056952953,
      "rewards/accuracy_reward": 0.02008928661234677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4168526977300644,
      "step": 1856
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1619.6541137695312,
      "epoch": 0.5547009185273691,
      "grad_norm": 22.766019821166992,
      "kl": 2.8671875,
      "learning_rate": 2.4582809185680626e-07,
      "loss": 0.2017,
      "reward": 0.4905134215950966,
      "reward_std": 0.176256962120533,
      "rewards/accuracy_reward": 0.046875002793967724,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443638414144516,
      "step": 1857
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1595.5157165527344,
      "epoch": 0.5549996266148906,
      "grad_norm": 46.013431549072266,
      "kl": 2.8125,
      "learning_rate": 2.4556737412090643e-07,
      "loss": 0.2089,
      "reward": 0.6183035969734192,
      "reward_std": 0.15249226242303848,
      "rewards/accuracy_reward": 0.17187500605359674,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4464285969734192,
      "step": 1858
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1652.9420166015625,
      "epoch": 0.5552983347024121,
      "grad_norm": 94.20999908447266,
      "kl": 5.4140625,
      "learning_rate": 2.4530666120726915e-07,
      "loss": 0.3182,
      "reward": 0.5000000298023224,
      "reward_std": 0.18773213773965836,
      "rewards/accuracy_reward": 0.07812500465661287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750149011612,
      "step": 1859
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1569.071533203125,
      "epoch": 0.5555970427899335,
      "grad_norm": 9.277281761169434,
      "kl": 2.81640625,
      "learning_rate": 2.4504595339952445e-07,
      "loss": 0.1925,
      "reward": 0.5396205559372902,
      "reward_std": 0.13220451585948467,
      "rewards/accuracy_reward": 0.0848214328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4547991305589676,
      "step": 1860
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1631.7701416015625,
      "epoch": 0.555895750877455,
      "grad_norm": 5.5020976066589355,
      "kl": 2.76171875,
      "learning_rate": 2.447852509812968e-07,
      "loss": 0.1783,
      "reward": 0.5809152126312256,
      "reward_std": 0.1795587930828333,
      "rewards/accuracy_reward": 0.1361607201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4447544813156128,
      "step": 1861
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1575.9710388183594,
      "epoch": 0.5561944589649764,
      "grad_norm": 36.74385452270508,
      "kl": 2.74609375,
      "learning_rate": 2.445245542362051e-07,
      "loss": 0.1985,
      "reward": 0.575892873108387,
      "reward_std": 0.1431265715509653,
      "rewards/accuracy_reward": 0.1272321513388306,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4486607313156128,
      "step": 1862
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1548.7478332519531,
      "epoch": 0.556493167052498,
      "grad_norm": 54.87440872192383,
      "kl": 2.44921875,
      "learning_rate": 2.442638634478617e-07,
      "loss": 0.1981,
      "reward": 0.4603794887661934,
      "reward_std": 0.11083609610795975,
      "rewards/accuracy_reward": 0.008928572060540318,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.451450914144516,
      "step": 1863
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1744.1027221679688,
      "epoch": 0.5567918751400194,
      "grad_norm": 15.86228084564209,
      "kl": 3.84375,
      "learning_rate": 2.4400317889987266e-07,
      "loss": 0.2302,
      "reward": 0.4988839477300644,
      "reward_std": 0.14138775318861008,
      "rewards/accuracy_reward": 0.08035714668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268059372902,
      "step": 1864
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1692.10498046875,
      "epoch": 0.5570905832275409,
      "grad_norm": 38.201080322265625,
      "kl": 2.3125,
      "learning_rate": 2.4374250087583726e-07,
      "loss": 0.1552,
      "reward": 0.5530134066939354,
      "reward_std": 0.09702224098145962,
      "rewards/accuracy_reward": 0.1093750037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443638414144516,
      "step": 1865
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1616.5558776855469,
      "epoch": 0.5573892913150623,
      "grad_norm": 45.745460510253906,
      "kl": 2.77734375,
      "learning_rate": 2.434818296593476e-07,
      "loss": 0.2003,
      "reward": 0.524553582072258,
      "reward_std": 0.1677049621939659,
      "rewards/accuracy_reward": 0.08928571757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435267873108387,
      "step": 1866
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1596.9509582519531,
      "epoch": 0.5576879994025838,
      "grad_norm": 7.408532619476318,
      "kl": 2.8515625,
      "learning_rate": 2.4322116553398844e-07,
      "loss": 0.1775,
      "reward": 0.5965401828289032,
      "reward_std": 0.12140622176229954,
      "rewards/accuracy_reward": 0.13839286286383867,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4581473395228386,
      "step": 1867
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1650.04248046875,
      "epoch": 0.5579867074901053,
      "grad_norm": 30.674081802368164,
      "kl": 3.0625,
      "learning_rate": 2.429605087833367e-07,
      "loss": 0.2089,
      "reward": 0.4838169813156128,
      "reward_std": 0.11079679429531097,
      "rewards/accuracy_reward": 0.04017857206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443638414144516,
      "step": 1868
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1647.3616943359375,
      "epoch": 0.5582854155776268,
      "grad_norm": 53.42111587524414,
      "kl": 2.8125,
      "learning_rate": 2.426998596909615e-07,
      "loss": 0.2099,
      "reward": 0.5401785895228386,
      "reward_std": 0.1639634370803833,
      "rewards/accuracy_reward": 0.10267857648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000223517418,
      "step": 1869
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1653.4911499023438,
      "epoch": 0.5585841236651482,
      "grad_norm": 20.33791732788086,
      "kl": 3.68359375,
      "learning_rate": 2.424392185404235e-07,
      "loss": 0.2237,
      "reward": 0.511160746216774,
      "reward_std": 0.16897070035338402,
      "rewards/accuracy_reward": 0.07589285937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4352678805589676,
      "step": 1870
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1695.9866943359375,
      "epoch": 0.5588828317526697,
      "grad_norm": 30.807552337646484,
      "kl": 3.861328125,
      "learning_rate": 2.4217858561527464e-07,
      "loss": 0.2145,
      "reward": 0.4877232387661934,
      "reward_std": 0.1683344915509224,
      "rewards/accuracy_reward": 0.049107144586741924,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4386160895228386,
      "step": 1871
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1583.6897888183594,
      "epoch": 0.5591815398401911,
      "grad_norm": 10.859689712524414,
      "kl": 3.171875,
      "learning_rate": 2.419179611990581e-07,
      "loss": 0.2192,
      "reward": 0.5167410895228386,
      "reward_std": 0.12609050795435905,
      "rewards/accuracy_reward": 0.06919643143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.447544664144516,
      "step": 1872
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1580.2701721191406,
      "epoch": 0.5594802479277127,
      "grad_norm": 31.51136589050293,
      "kl": 3.72265625,
      "learning_rate": 2.416573455753076e-07,
      "loss": 0.2579,
      "reward": 0.5703125223517418,
      "reward_std": 0.15775125101208687,
      "rewards/accuracy_reward": 0.11830357694998384,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4520089477300644,
      "step": 1873
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1611.0447082519531,
      "epoch": 0.5597789560152341,
      "grad_norm": 46.38452911376953,
      "kl": 3.76953125,
      "learning_rate": 2.4139673902754753e-07,
      "loss": 0.2359,
      "reward": 0.5797991454601288,
      "reward_std": 0.11784730665385723,
      "rewards/accuracy_reward": 0.12500000419095159,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.454799123108387,
      "step": 1874
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1594.43310546875,
      "epoch": 0.5600776641027556,
      "grad_norm": 10.534722328186035,
      "kl": 2.8671875,
      "learning_rate": 2.4113614183929205e-07,
      "loss": 0.2045,
      "reward": 0.5446428880095482,
      "reward_std": 0.1554849036037922,
      "rewards/accuracy_reward": 0.0937500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4508928805589676,
      "step": 1875
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1729.7590026855469,
      "epoch": 0.560376372190277,
      "grad_norm": 50.84327697753906,
      "kl": 5.37109375,
      "learning_rate": 2.408755542940456e-07,
      "loss": 0.3143,
      "reward": 0.502232164144516,
      "reward_std": 0.1537531465291977,
      "rewards/accuracy_reward": 0.08928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4129464402794838,
      "step": 1876
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1640.15185546875,
      "epoch": 0.5606750802777986,
      "grad_norm": 15.175320625305176,
      "kl": 4.046875,
      "learning_rate": 2.4061497667530176e-07,
      "loss": 0.2442,
      "reward": 0.5636161044239998,
      "reward_std": 0.11764203757047653,
      "rewards/accuracy_reward": 0.1316964365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4319196566939354,
      "step": 1877
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1653.0692443847656,
      "epoch": 0.56097378836532,
      "grad_norm": 27.569849014282227,
      "kl": 3.8828125,
      "learning_rate": 2.4035440926654344e-07,
      "loss": 0.2283,
      "reward": 0.4933035969734192,
      "reward_std": 0.1219912339001894,
      "rewards/accuracy_reward": 0.05357142956927419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.439732164144516,
      "step": 1878
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1604.9353332519531,
      "epoch": 0.5612724964528415,
      "grad_norm": 27.032745361328125,
      "kl": 2.412109375,
      "learning_rate": 2.400938523512425e-07,
      "loss": 0.1639,
      "reward": 0.5452009066939354,
      "reward_std": 0.1399172507226467,
      "rewards/accuracy_reward": 0.08258929289877415,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4626116305589676,
      "step": 1879
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1647.69873046875,
      "epoch": 0.5615712045403629,
      "grad_norm": 22.053356170654297,
      "kl": 3.4453125,
      "learning_rate": 2.3983330621285935e-07,
      "loss": 0.2359,
      "reward": 0.4994419887661934,
      "reward_std": 0.12786435894668102,
      "rewards/accuracy_reward": 0.0625000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419887661934,
      "step": 1880
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1623.5670166015625,
      "epoch": 0.5618699126278844,
      "grad_norm": 22.567678451538086,
      "kl": 3.5078125,
      "learning_rate": 2.395727711348425e-07,
      "loss": 0.2334,
      "reward": 0.5680803805589676,
      "reward_std": 0.1958514228463173,
      "rewards/accuracy_reward": 0.12946429592557251,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4386160895228386,
      "step": 1881
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1675.2858276367188,
      "epoch": 0.5621686207154059,
      "grad_norm": 23.86686134338379,
      "kl": 2.4609375,
      "learning_rate": 2.3931224740062873e-07,
      "loss": 0.1823,
      "reward": 0.5379464477300644,
      "reward_std": 0.1649578120559454,
      "rewards/accuracy_reward": 0.0870535746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4508928805589676,
      "step": 1882
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1668.3036499023438,
      "epoch": 0.5624673288029274,
      "grad_norm": 77.09688568115234,
      "kl": 5.91015625,
      "learning_rate": 2.3905173529364224e-07,
      "loss": 0.3374,
      "reward": 0.5546875223517418,
      "reward_std": 0.1380814090371132,
      "rewards/accuracy_reward": 0.1361607201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268059372902,
      "step": 1883
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1661.0491943359375,
      "epoch": 0.5627660368904488,
      "grad_norm": 9.921865463256836,
      "kl": 4.20703125,
      "learning_rate": 2.3879123509729477e-07,
      "loss": 0.2863,
      "reward": 0.5535714402794838,
      "reward_std": 0.16164697147905827,
      "rewards/accuracy_reward": 0.11607143003493547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000223517418,
      "step": 1884
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1651.3081359863281,
      "epoch": 0.5630647449779703,
      "grad_norm": 46.691959381103516,
      "kl": 4.38671875,
      "learning_rate": 2.385307470949849e-07,
      "loss": 0.2582,
      "reward": 0.4765625149011612,
      "reward_std": 0.12649006955325603,
      "rewards/accuracy_reward": 0.022321430034935474,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4542410895228386,
      "step": 1885
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1651.2411499023438,
      "epoch": 0.5633634530654917,
      "grad_norm": 47.898948669433594,
      "kl": 5.17578125,
      "learning_rate": 2.3827027157009805e-07,
      "loss": 0.3364,
      "reward": 0.5837053954601288,
      "reward_std": 0.17154448479413986,
      "rewards/accuracy_reward": 0.1540178656578064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875223517418,
      "step": 1886
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1614.3906860351562,
      "epoch": 0.5636621611530133,
      "grad_norm": 45.100799560546875,
      "kl": 4.36328125,
      "learning_rate": 2.3800980880600613e-07,
      "loss": 0.2695,
      "reward": 0.5602678954601288,
      "reward_std": 0.1970061995089054,
      "rewards/accuracy_reward": 0.1227678656578064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000223517418,
      "step": 1887
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1646.5179443359375,
      "epoch": 0.5639608692405347,
      "grad_norm": 13.88018798828125,
      "kl": 2.837890625,
      "learning_rate": 2.3774935908606707e-07,
      "loss": 0.19,
      "reward": 0.5206473469734192,
      "reward_std": 0.16583536937832832,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4492187649011612,
      "step": 1888
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1702.0357971191406,
      "epoch": 0.5642595773280562,
      "grad_norm": 17.34386444091797,
      "kl": 3.95703125,
      "learning_rate": 2.3748892269362458e-07,
      "loss": 0.2474,
      "reward": 0.5546875223517418,
      "reward_std": 0.12680058367550373,
      "rewards/accuracy_reward": 0.1250000074505806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875149011612,
      "step": 1889
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1588.2590026855469,
      "epoch": 0.5645582854155776,
      "grad_norm": 18.85053825378418,
      "kl": 3.96484375,
      "learning_rate": 2.3722849991200805e-07,
      "loss": 0.2635,
      "reward": 0.6021205559372902,
      "reward_std": 0.12867296021431684,
      "rewards/accuracy_reward": 0.16071428917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4414062798023224,
      "step": 1890
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1646.6719360351562,
      "epoch": 0.5648569935030991,
      "grad_norm": 25.355182647705078,
      "kl": 3.392578125,
      "learning_rate": 2.3696809102453187e-07,
      "loss": 0.2294,
      "reward": 0.4905134215950966,
      "reward_std": 0.15294067189097404,
      "rewards/accuracy_reward": 0.05580357392318547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098469734192,
      "step": 1891
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1713.1428833007812,
      "epoch": 0.5651557015906206,
      "grad_norm": 30.410537719726562,
      "kl": 3.423828125,
      "learning_rate": 2.3670769631449535e-07,
      "loss": 0.2261,
      "reward": 0.5491071715950966,
      "reward_std": 0.13489943742752075,
      "rewards/accuracy_reward": 0.12276785937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263392984867096,
      "step": 1892
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1600.0201721191406,
      "epoch": 0.5654544096781421,
      "grad_norm": 15.669499397277832,
      "kl": 3.30859375,
      "learning_rate": 2.3644731606518245e-07,
      "loss": 0.2067,
      "reward": 0.5072544813156128,
      "reward_std": 0.16281880252063274,
      "rewards/accuracy_reward": 0.0937500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4135044813156128,
      "step": 1893
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1646.8036499023438,
      "epoch": 0.5657531177656635,
      "grad_norm": 22.78024673461914,
      "kl": 2.205078125,
      "learning_rate": 2.361869505598614e-07,
      "loss": 0.1481,
      "reward": 0.5887277126312256,
      "reward_std": 0.17063764110207558,
      "rewards/accuracy_reward": 0.129464291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.459263414144516,
      "step": 1894
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1550.07373046875,
      "epoch": 0.5660518258531849,
      "grad_norm": 29.20393943786621,
      "kl": 2.9609375,
      "learning_rate": 2.359266000817843e-07,
      "loss": 0.2152,
      "reward": 0.5111607313156128,
      "reward_std": 0.14274494908750057,
      "rewards/accuracy_reward": 0.06473214668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4464285895228386,
      "step": 1895
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1705.4308776855469,
      "epoch": 0.5663505339407064,
      "grad_norm": 32.4789924621582,
      "kl": 3.3046875,
      "learning_rate": 2.3566626491418696e-07,
      "loss": 0.227,
      "reward": 0.5725446790456772,
      "reward_std": 0.1053739283233881,
      "rewards/accuracy_reward": 0.1428571492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875149011612,
      "step": 1896
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1617.9420471191406,
      "epoch": 0.5666492420282279,
      "grad_norm": 63.8911247253418,
      "kl": 2.65625,
      "learning_rate": 2.3540594534028855e-07,
      "loss": 0.2239,
      "reward": 0.5050223395228386,
      "reward_std": 0.14275668375194073,
      "rewards/accuracy_reward": 0.05580357415601611,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4492187798023224,
      "step": 1897
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1601.8817443847656,
      "epoch": 0.5669479501157494,
      "grad_norm": 10.655900955200195,
      "kl": 3.31640625,
      "learning_rate": 2.3514564164329127e-07,
      "loss": 0.2153,
      "reward": 0.5474330708384514,
      "reward_std": 0.1622856967151165,
      "rewards/accuracy_reward": 0.09598214644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.451450914144516,
      "step": 1898
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1598.7433776855469,
      "epoch": 0.5672466582032708,
      "grad_norm": 77.06262969970703,
      "kl": 5.03125,
      "learning_rate": 2.3488535410637997e-07,
      "loss": 0.3045,
      "reward": 0.4743303805589676,
      "reward_std": 0.14830541796982288,
      "rewards/accuracy_reward": 0.0357142873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4386160895228386,
      "step": 1899
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1606.6607666015625,
      "epoch": 0.5675453662907923,
      "grad_norm": 63.15616989135742,
      "kl": 4.56640625,
      "learning_rate": 2.3462508301272197e-07,
      "loss": 0.2786,
      "reward": 0.5206473469734192,
      "reward_std": 0.1900695003569126,
      "rewards/accuracy_reward": 0.07589285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4447544887661934,
      "step": 1900
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1702.01123046875,
      "epoch": 0.5678440743783137,
      "grad_norm": 35.832733154296875,
      "kl": 4.4140625,
      "learning_rate": 2.3436482864546677e-07,
      "loss": 0.2672,
      "reward": 0.4827009215950966,
      "reward_std": 0.14082352630794048,
      "rewards/accuracy_reward": 0.06026786006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330559372902,
      "step": 1901
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1588.3259582519531,
      "epoch": 0.5681427824658353,
      "grad_norm": 37.67543411254883,
      "kl": 4.17578125,
      "learning_rate": 2.3410459128774563e-07,
      "loss": 0.2606,
      "reward": 0.483258955180645,
      "reward_std": 0.15673963725566864,
      "rewards/accuracy_reward": 0.04017857229337096,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4430803805589676,
      "step": 1902
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1635.3550109863281,
      "epoch": 0.5684414905533567,
      "grad_norm": 12.764564514160156,
      "kl": 3.9296875,
      "learning_rate": 2.338443712226712e-07,
      "loss": 0.2406,
      "reward": 0.5061384215950966,
      "reward_std": 0.12712224572896957,
      "rewards/accuracy_reward": 0.05803571757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.448102705180645,
      "step": 1903
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1621.8728332519531,
      "epoch": 0.5687401986408782,
      "grad_norm": 12.215208053588867,
      "kl": 2.244140625,
      "learning_rate": 2.335841687333375e-07,
      "loss": 0.1545,
      "reward": 0.5982143133878708,
      "reward_std": 0.15142563544213772,
      "rewards/accuracy_reward": 0.1406250074505806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4575893133878708,
      "step": 1904
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1615.4554138183594,
      "epoch": 0.5690389067283996,
      "grad_norm": 43.20653533935547,
      "kl": 3.859375,
      "learning_rate": 2.333239841028193e-07,
      "loss": 0.2524,
      "reward": 0.5457589477300644,
      "reward_std": 0.15469874814152718,
      "rewards/accuracy_reward": 0.09821429057046771,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.447544664144516,
      "step": 1905
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1675.9911499023438,
      "epoch": 0.5693376148159212,
      "grad_norm": 41.28318405151367,
      "kl": 2.546875,
      "learning_rate": 2.3306381761417194e-07,
      "loss": 0.1785,
      "reward": 0.5424107313156128,
      "reward_std": 0.1523641124367714,
      "rewards/accuracy_reward": 0.09375000116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4486607387661934,
      "step": 1906
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1614.1964721679688,
      "epoch": 0.5696363229034426,
      "grad_norm": 16.117908477783203,
      "kl": 3.0390625,
      "learning_rate": 2.3280366955043106e-07,
      "loss": 0.2163,
      "reward": 0.534040205180645,
      "reward_std": 0.16804299876093864,
      "rewards/accuracy_reward": 0.09151786146685481,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4425223395228386,
      "step": 1907
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1674.8527526855469,
      "epoch": 0.5699350309909641,
      "grad_norm": 6.39769172668457,
      "kl": 2.421875,
      "learning_rate": 2.3254354019461232e-07,
      "loss": 0.1464,
      "reward": 0.583147332072258,
      "reward_std": 0.16184914484620094,
      "rewards/accuracy_reward": 0.1250000020954758,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4581473395228386,
      "step": 1908
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1655.1964721679688,
      "epoch": 0.5702337390784855,
      "grad_norm": 18.632488250732422,
      "kl": 2.69921875,
      "learning_rate": 2.32283429829711e-07,
      "loss": 0.1854,
      "reward": 0.5111607387661934,
      "reward_std": 0.18763187527656555,
      "rewards/accuracy_reward": 0.066964291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.444196455180645,
      "step": 1909
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1578.9063415527344,
      "epoch": 0.570532447166007,
      "grad_norm": 22.70128631591797,
      "kl": 2.953125,
      "learning_rate": 2.320233387387015e-07,
      "loss": 0.2137,
      "reward": 0.565290205180645,
      "reward_std": 0.16703174635767937,
      "rewards/accuracy_reward": 0.1183035783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4469866305589676,
      "step": 1910
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1616.5536804199219,
      "epoch": 0.5708311552535285,
      "grad_norm": 14.844437599182129,
      "kl": 2.99609375,
      "learning_rate": 2.3176326720453762e-07,
      "loss": 0.2071,
      "reward": 0.5915178954601288,
      "reward_std": 0.14543218351900578,
      "rewards/accuracy_reward": 0.1473214365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.444196455180645,
      "step": 1911
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1708.7031860351562,
      "epoch": 0.57112986334105,
      "grad_norm": 32.28428649902344,
      "kl": 3.84765625,
      "learning_rate": 2.3150321551015163e-07,
      "loss": 0.2221,
      "reward": 0.4966518133878708,
      "reward_std": 0.1747417338192463,
      "rewards/accuracy_reward": 0.060267860535532236,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4363839477300644,
      "step": 1912
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1600.2969055175781,
      "epoch": 0.5714285714285714,
      "grad_norm": 12.04436206817627,
      "kl": 2.56640625,
      "learning_rate": 2.3124318393845428e-07,
      "loss": 0.1553,
      "reward": 0.5334821715950966,
      "reward_std": 0.1719003077596426,
      "rewards/accuracy_reward": 0.07142857694998384,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4620535895228386,
      "step": 1913
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1606.3951416015625,
      "epoch": 0.5717272795160929,
      "grad_norm": 28.22039031982422,
      "kl": 2.2744140625,
      "learning_rate": 2.309831727723344e-07,
      "loss": 0.1584,
      "reward": 0.5825892984867096,
      "reward_std": 0.16422081738710403,
      "rewards/accuracy_reward": 0.1272321492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.455357164144516,
      "step": 1914
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1628.5781860351562,
      "epoch": 0.5720259876036143,
      "grad_norm": 14.920714378356934,
      "kl": 3.66796875,
      "learning_rate": 2.3072318229465872e-07,
      "loss": 0.2328,
      "reward": 0.5976562798023224,
      "reward_std": 0.10308068804442883,
      "rewards/accuracy_reward": 0.14732143515720963,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4503348395228386,
      "step": 1915
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1633.6697082519531,
      "epoch": 0.5723246956911359,
      "grad_norm": 58.2711067199707,
      "kl": 4.59375,
      "learning_rate": 2.3046321278827138e-07,
      "loss": 0.2715,
      "reward": 0.5608259215950966,
      "reward_std": 0.14429756440222263,
      "rewards/accuracy_reward": 0.12276786286383867,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4380580559372902,
      "step": 1916
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1638.8728637695312,
      "epoch": 0.5726234037786573,
      "grad_norm": 92.38995361328125,
      "kl": 5.22265625,
      "learning_rate": 2.3020326453599368e-07,
      "loss": 0.2991,
      "reward": 0.5714286044239998,
      "reward_std": 0.14556533098220825,
      "rewards/accuracy_reward": 0.12946429406292737,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4419643059372902,
      "step": 1917
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1660.5536499023438,
      "epoch": 0.5729221118661788,
      "grad_norm": 14.821087837219238,
      "kl": 3.19140625,
      "learning_rate": 2.2994333782062393e-07,
      "loss": 0.2136,
      "reward": 0.4676339477300644,
      "reward_std": 0.11946978233754635,
      "rewards/accuracy_reward": 0.017857144121080637,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4497768059372902,
      "step": 1918
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1639.9755249023438,
      "epoch": 0.5732208199537002,
      "grad_norm": 20.257143020629883,
      "kl": 3.9375,
      "learning_rate": 2.2968343292493686e-07,
      "loss": 0.2339,
      "reward": 0.5379464477300644,
      "reward_std": 0.17739084362983704,
      "rewards/accuracy_reward": 0.09598214738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4419643059372902,
      "step": 1919
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1625.2098693847656,
      "epoch": 0.5735195280412217,
      "grad_norm": 27.95660972595215,
      "kl": 3.349609375,
      "learning_rate": 2.294235501316835e-07,
      "loss": 0.1923,
      "reward": 0.5747768059372902,
      "reward_std": 0.16299426183104515,
      "rewards/accuracy_reward": 0.12276786239817739,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4520089477300644,
      "step": 1920
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1618.4219665527344,
      "epoch": 0.5738182361287432,
      "grad_norm": 12.885087013244629,
      "kl": 3.71875,
      "learning_rate": 2.2916368972359088e-07,
      "loss": 0.24,
      "reward": 0.5106026977300644,
      "reward_std": 0.14280694164335728,
      "rewards/accuracy_reward": 0.06696429080329835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443638414144516,
      "step": 1921
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1690.2545471191406,
      "epoch": 0.5741169442162647,
      "grad_norm": 56.94000244140625,
      "kl": 4.5859375,
      "learning_rate": 2.289038519833617e-07,
      "loss": 0.261,
      "reward": 0.5530134215950966,
      "reward_std": 0.1930195726454258,
      "rewards/accuracy_reward": 0.12723214644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812723517418,
      "step": 1922
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1652.1607971191406,
      "epoch": 0.5744156523037861,
      "grad_norm": 19.354734420776367,
      "kl": 4.14453125,
      "learning_rate": 2.2864403719367392e-07,
      "loss": 0.2532,
      "reward": 0.6484375298023224,
      "reward_std": 0.11191197670996189,
      "rewards/accuracy_reward": 0.2142857238650322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341517984867096,
      "step": 1923
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1708.7232971191406,
      "epoch": 0.5747143603913076,
      "grad_norm": 36.38756561279297,
      "kl": 3.953125,
      "learning_rate": 2.283842456371805e-07,
      "loss": 0.2562,
      "reward": 0.5993303805589676,
      "reward_std": 0.16229806654155254,
      "rewards/accuracy_reward": 0.16294643399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4363839402794838,
      "step": 1924
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1608.51123046875,
      "epoch": 0.575013068478829,
      "grad_norm": 38.94938659667969,
      "kl": 2.984375,
      "learning_rate": 2.2812447759650932e-07,
      "loss": 0.2164,
      "reward": 0.5825893133878708,
      "reward_std": 0.1651261616498232,
      "rewards/accuracy_reward": 0.1339285783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4486607313156128,
      "step": 1925
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1581.3460388183594,
      "epoch": 0.5753117765663506,
      "grad_norm": 12.798613548278809,
      "kl": 2.189453125,
      "learning_rate": 2.2786473335426247e-07,
      "loss": 0.1596,
      "reward": 0.5502232238650322,
      "reward_std": 0.16101041063666344,
      "rewards/accuracy_reward": 0.09821428847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4520089477300644,
      "step": 1926
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1584.3594665527344,
      "epoch": 0.575610484653872,
      "grad_norm": 21.631610870361328,
      "kl": 2.765625,
      "learning_rate": 2.2760501319301626e-07,
      "loss": 0.1933,
      "reward": 0.5195312723517418,
      "reward_std": 0.18138942122459412,
      "rewards/accuracy_reward": 0.0758928619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443638414144516,
      "step": 1927
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1614.2835693359375,
      "epoch": 0.5759091927413935,
      "grad_norm": 27.47783660888672,
      "kl": 2.181640625,
      "learning_rate": 2.2734531739532067e-07,
      "loss": 0.1576,
      "reward": 0.4994419887661934,
      "reward_std": 0.14626258052885532,
      "rewards/accuracy_reward": 0.035714287078008056,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4637276977300644,
      "step": 1928
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1647.8996276855469,
      "epoch": 0.5762079008289149,
      "grad_norm": 16.487390518188477,
      "kl": 2.796875,
      "learning_rate": 2.2708564624369944e-07,
      "loss": 0.1675,
      "reward": 0.5223214477300644,
      "reward_std": 0.13632718473672867,
      "rewards/accuracy_reward": 0.07366071757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4486607313156128,
      "step": 1929
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1624.2567749023438,
      "epoch": 0.5765066089164365,
      "grad_norm": 15.081119537353516,
      "kl": 2.59375,
      "learning_rate": 2.2682600002064923e-07,
      "loss": 0.1629,
      "reward": 0.5150669813156128,
      "reward_std": 0.15225842595100403,
      "rewards/accuracy_reward": 0.0535714291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4614955633878708,
      "step": 1930
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1712.6429443359375,
      "epoch": 0.5768053170039579,
      "grad_norm": 14.658039093017578,
      "kl": 3.63671875,
      "learning_rate": 2.2656637900863965e-07,
      "loss": 0.2079,
      "reward": 0.4760044813156128,
      "reward_std": 0.16634920425713062,
      "rewards/accuracy_reward": 0.04017857275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435825914144516,
      "step": 1931
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1715.3527221679688,
      "epoch": 0.5771040250914794,
      "grad_norm": 17.052610397338867,
      "kl": 3.146484375,
      "learning_rate": 2.2630678349011303e-07,
      "loss": 0.2064,
      "reward": 0.6049107313156128,
      "reward_std": 0.17495453171432018,
      "rewards/accuracy_reward": 0.1607142984867096,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4441964477300644,
      "step": 1932
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1617.3996276855469,
      "epoch": 0.5774027331790008,
      "grad_norm": 36.0367546081543,
      "kl": 3.17578125,
      "learning_rate": 2.2604721374748378e-07,
      "loss": 0.2334,
      "reward": 0.5424107536673546,
      "reward_std": 0.15594821237027645,
      "rewards/accuracy_reward": 0.1071428619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435267873108387,
      "step": 1933
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1568.4308776855469,
      "epoch": 0.5777014412665223,
      "grad_norm": 8.084633827209473,
      "kl": 2.474609375,
      "learning_rate": 2.2578767006313834e-07,
      "loss": 0.1614,
      "reward": 0.5111607313156128,
      "reward_std": 0.14678932167589664,
      "rewards/accuracy_reward": 0.04910714668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4620535969734192,
      "step": 1934
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1562.6629943847656,
      "epoch": 0.5780001493540438,
      "grad_norm": 5.367437839508057,
      "kl": 2.765625,
      "learning_rate": 2.2552815271943477e-07,
      "loss": 0.1906,
      "reward": 0.5435268133878708,
      "reward_std": 0.19687253050506115,
      "rewards/accuracy_reward": 0.08705357555299997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4564732387661934,
      "step": 1935
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1640.3348999023438,
      "epoch": 0.5782988574415653,
      "grad_norm": 9.087560653686523,
      "kl": 3.60546875,
      "learning_rate": 2.2526866199870258e-07,
      "loss": 0.2308,
      "reward": 0.5457589626312256,
      "reward_std": 0.14839855581521988,
      "rewards/accuracy_reward": 0.10714286379516125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4386160969734192,
      "step": 1936
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1720.5067749023438,
      "epoch": 0.5785975655290867,
      "grad_norm": 7.7881083488464355,
      "kl": 4.20703125,
      "learning_rate": 2.2500919818324224e-07,
      "loss": 0.2535,
      "reward": 0.4375000223517418,
      "reward_std": 0.12629659287631512,
      "rewards/accuracy_reward": 0.006696428870782256,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035895228386,
      "step": 1937
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1582.8996276855469,
      "epoch": 0.5788962736166081,
      "grad_norm": 22.43412208557129,
      "kl": 3.591796875,
      "learning_rate": 2.2474976155532484e-07,
      "loss": 0.2247,
      "reward": 0.5591518133878708,
      "reward_std": 0.13764260709285736,
      "rewards/accuracy_reward": 0.10714286402799189,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.452008955180645,
      "step": 1938
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1569.1675109863281,
      "epoch": 0.5791949817041296,
      "grad_norm": 27.67903709411621,
      "kl": 3.599609375,
      "learning_rate": 2.2449035239719212e-07,
      "loss": 0.229,
      "reward": 0.547991082072258,
      "reward_std": 0.15824110619723797,
      "rewards/accuracy_reward": 0.10267857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125223517418,
      "step": 1939
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1570.15185546875,
      "epoch": 0.5794936897916511,
      "grad_norm": 7.029219150543213,
      "kl": 2.568359375,
      "learning_rate": 2.2423097099105577e-07,
      "loss": 0.1706,
      "reward": 0.5373883992433548,
      "reward_std": 0.15089070796966553,
      "rewards/accuracy_reward": 0.0758928619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4614955559372902,
      "step": 1940
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1610.3170166015625,
      "epoch": 0.5797923978791726,
      "grad_norm": 14.871225357055664,
      "kl": 2.96875,
      "learning_rate": 2.2397161761909736e-07,
      "loss": 0.2055,
      "reward": 0.5133928805589676,
      "reward_std": 0.15376004949212074,
      "rewards/accuracy_reward": 0.058035716880112886,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.455357164144516,
      "step": 1941
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1643.8929443359375,
      "epoch": 0.580091105966694,
      "grad_norm": 54.981807708740234,
      "kl": 4.47265625,
      "learning_rate": 2.2371229256346787e-07,
      "loss": 0.2707,
      "reward": 0.5753348544239998,
      "reward_std": 0.17521991953253746,
      "rewards/accuracy_reward": 0.1361607238650322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4391741305589676,
      "step": 1942
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1640.9509582519531,
      "epoch": 0.5803898140542155,
      "grad_norm": 15.395828247070312,
      "kl": 3.701171875,
      "learning_rate": 2.2345299610628765e-07,
      "loss": 0.2389,
      "reward": 0.5747768133878708,
      "reward_std": 0.11841774545609951,
      "rewards/accuracy_reward": 0.12276786286383867,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4520089477300644,
      "step": 1943
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1674.3661499023438,
      "epoch": 0.5806885221417369,
      "grad_norm": 44.160858154296875,
      "kl": 4.046875,
      "learning_rate": 2.231937285296458e-07,
      "loss": 0.242,
      "reward": 0.5033482313156128,
      "reward_std": 0.17699915170669556,
      "rewards/accuracy_reward": 0.0535714291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4497768059372902,
      "step": 1944
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1661.4688110351562,
      "epoch": 0.5809872302292585,
      "grad_norm": 6.402981758117676,
      "kl": 3.3203125,
      "learning_rate": 2.2293449011559994e-07,
      "loss": 0.2177,
      "reward": 0.5306919887661934,
      "reward_std": 0.13995127193629742,
      "rewards/accuracy_reward": 0.0758928619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4547991305589676,
      "step": 1945
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1575.7165832519531,
      "epoch": 0.5812859383167799,
      "grad_norm": 34.00381851196289,
      "kl": 3.58203125,
      "learning_rate": 2.2267528114617617e-07,
      "loss": 0.2141,
      "reward": 0.5613839477300644,
      "reward_std": 0.16338125616312027,
      "rewards/accuracy_reward": 0.10937500605359674,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4520089477300644,
      "step": 1946
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1647.66748046875,
      "epoch": 0.5815846464043014,
      "grad_norm": 12.491554260253906,
      "kl": 2.810546875,
      "learning_rate": 2.2241610190336844e-07,
      "loss": 0.177,
      "reward": 0.581473246216774,
      "reward_std": 0.14199474826455116,
      "rewards/accuracy_reward": 0.13169643748551607,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4497768059372902,
      "step": 1947
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1514.4264526367188,
      "epoch": 0.5818833544918228,
      "grad_norm": 77.83522033691406,
      "kl": 1.572265625,
      "learning_rate": 2.221569526691383e-07,
      "loss": 0.1352,
      "reward": 0.5887276977300644,
      "reward_std": 0.14795255847275257,
      "rewards/accuracy_reward": 0.12946429220028222,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.459263414144516,
      "step": 1948
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1569.5603332519531,
      "epoch": 0.5821820625793444,
      "grad_norm": 37.207584381103516,
      "kl": 1.900390625,
      "learning_rate": 2.218978337254147e-07,
      "loss": 0.1545,
      "reward": 0.5452008992433548,
      "reward_std": 0.13090214040130377,
      "rewards/accuracy_reward": 0.09375000558793545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4514509066939354,
      "step": 1949
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1516.9576416015625,
      "epoch": 0.5824807706668658,
      "grad_norm": 12.996857643127441,
      "kl": 3.08203125,
      "learning_rate": 2.2163874535409375e-07,
      "loss": 0.1997,
      "reward": 0.6088169887661934,
      "reward_std": 0.1362476423382759,
      "rewards/accuracy_reward": 0.15625000465661287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4525669813156128,
      "step": 1950
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1547.5491638183594,
      "epoch": 0.5827794787543873,
      "grad_norm": 20.087648391723633,
      "kl": 2.0,
      "learning_rate": 2.2137968783703813e-07,
      "loss": 0.1781,
      "reward": 0.6328125149011612,
      "reward_std": 0.20250336825847626,
      "rewards/accuracy_reward": 0.1830357201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4497768133878708,
      "step": 1951
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1592.8460388183594,
      "epoch": 0.5830781868419087,
      "grad_norm": 25.490694046020508,
      "kl": 1.53173828125,
      "learning_rate": 2.2112066145607703e-07,
      "loss": 0.0938,
      "reward": 0.5781250298023224,
      "reward_std": 0.1565748080611229,
      "rewards/accuracy_reward": 0.11607143469154835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4620535895228386,
      "step": 1952
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1643.3661499023438,
      "epoch": 0.5833768949294302,
      "grad_norm": 14.498685836791992,
      "kl": 2.8515625,
      "learning_rate": 2.2086166649300583e-07,
      "loss": 0.186,
      "reward": 0.4782366156578064,
      "reward_std": 0.17088854685425758,
      "rewards/accuracy_reward": 0.0424107164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4358259066939354,
      "step": 1953
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1580.732177734375,
      "epoch": 0.5836756030169516,
      "grad_norm": 16.9406681060791,
      "kl": 3.4453125,
      "learning_rate": 2.2060270322958562e-07,
      "loss": 0.2124,
      "reward": 0.5987723395228386,
      "reward_std": 0.11908787302672863,
      "rewards/accuracy_reward": 0.1584821492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4402901977300644,
      "step": 1954
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1684.3170776367188,
      "epoch": 0.5839743111044732,
      "grad_norm": 21.546798706054688,
      "kl": 3.57421875,
      "learning_rate": 2.2034377194754306e-07,
      "loss": 0.2313,
      "reward": 0.5357143208384514,
      "reward_std": 0.14970465376973152,
      "rewards/accuracy_reward": 0.098214291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000223517418,
      "step": 1955
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1680.4442749023438,
      "epoch": 0.5842730191919946,
      "grad_norm": 20.07666015625,
      "kl": 3.5234375,
      "learning_rate": 2.2008487292856991e-07,
      "loss": 0.2285,
      "reward": 0.5005580559372902,
      "reward_std": 0.11579084023833275,
      "rewards/accuracy_reward": 0.0625000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4380580559372902,
      "step": 1956
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1652.5603637695312,
      "epoch": 0.5845717272795161,
      "grad_norm": 39.14363479614258,
      "kl": 3.98828125,
      "learning_rate": 2.198260064543231e-07,
      "loss": 0.2412,
      "reward": 0.616629496216774,
      "reward_std": 0.19253928028047085,
      "rewards/accuracy_reward": 0.1852678656578064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4313616305589676,
      "step": 1957
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1629.5915832519531,
      "epoch": 0.5848704353670375,
      "grad_norm": 23.324373245239258,
      "kl": 3.51171875,
      "learning_rate": 2.1956717280642392e-07,
      "loss": 0.2232,
      "reward": 0.5680803805589676,
      "reward_std": 0.17788539454340935,
      "rewards/accuracy_reward": 0.13169643515720963,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.436383955180645,
      "step": 1958
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1646.5581359863281,
      "epoch": 0.5851691434545591,
      "grad_norm": 17.148895263671875,
      "kl": 3.0625,
      "learning_rate": 2.193083722664579e-07,
      "loss": 0.183,
      "reward": 0.5558035969734192,
      "reward_std": 0.168367438018322,
      "rewards/accuracy_reward": 0.10714286006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4486607313156128,
      "step": 1959
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1651.4911804199219,
      "epoch": 0.5854678515420805,
      "grad_norm": 22.90903091430664,
      "kl": 3.5703125,
      "learning_rate": 2.1904960511597481e-07,
      "loss": 0.2218,
      "reward": 0.4972098469734192,
      "reward_std": 0.11686134897172451,
      "rewards/accuracy_reward": 0.051339289639145136,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4458705559372902,
      "step": 1960
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1638.32373046875,
      "epoch": 0.585766559629602,
      "grad_norm": 28.854951858520508,
      "kl": 2.5390625,
      "learning_rate": 2.187908716364879e-07,
      "loss": 0.1865,
      "reward": 0.5641741380095482,
      "reward_std": 0.16427361592650414,
      "rewards/accuracy_reward": 0.11830357951112092,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4458705633878708,
      "step": 1961
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1613.8192749023438,
      "epoch": 0.5860652677171234,
      "grad_norm": 30.84535026550293,
      "kl": 2.236328125,
      "learning_rate": 2.1853217210947382e-07,
      "loss": 0.1756,
      "reward": 0.5998884066939354,
      "reward_std": 0.1821010746061802,
      "rewards/accuracy_reward": 0.15401786658912897,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4458705559372902,
      "step": 1962
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1622.3170471191406,
      "epoch": 0.5863639758046449,
      "grad_norm": 11.063496589660645,
      "kl": 2.953125,
      "learning_rate": 2.1827350681637225e-07,
      "loss": 0.1999,
      "reward": 0.5234375223517418,
      "reward_std": 0.14028967916965485,
      "rewards/accuracy_reward": 0.08035714738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4430803805589676,
      "step": 1963
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1602.3282165527344,
      "epoch": 0.5866626838921664,
      "grad_norm": 18.20419692993164,
      "kl": 3.09765625,
      "learning_rate": 2.1801487603858578e-07,
      "loss": 0.1993,
      "reward": 0.4810268059372902,
      "reward_std": 0.14484633691608906,
      "rewards/accuracy_reward": 0.03571428684517741,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125223517418,
      "step": 1964
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1637.2813110351562,
      "epoch": 0.5869613919796879,
      "grad_norm": 38.480751037597656,
      "kl": 2.513671875,
      "learning_rate": 2.177562800574793e-07,
      "loss": 0.1714,
      "reward": 0.5044642984867096,
      "reward_std": 0.1829574517905712,
      "rewards/accuracy_reward": 0.06250000139698386,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4419643059372902,
      "step": 1965
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1553.6027526855469,
      "epoch": 0.5872601000672093,
      "grad_norm": 46.01110076904297,
      "kl": 1.93359375,
      "learning_rate": 2.1749771915437987e-07,
      "loss": 0.1441,
      "reward": 0.5753348395228386,
      "reward_std": 0.13641639985144138,
      "rewards/accuracy_reward": 0.11160714970901608,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4637276977300644,
      "step": 1966
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1593.8818054199219,
      "epoch": 0.5875588081547308,
      "grad_norm": 25.398975372314453,
      "kl": 2.013671875,
      "learning_rate": 2.1723919361057646e-07,
      "loss": 0.1484,
      "reward": 0.5937500298023224,
      "reward_std": 0.14600478298962116,
      "rewards/accuracy_reward": 0.1294642947614193,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4642857313156128,
      "step": 1967
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1653.6094665527344,
      "epoch": 0.5878575162422522,
      "grad_norm": 41.52937316894531,
      "kl": 1.9814453125,
      "learning_rate": 2.1698070370731953e-07,
      "loss": 0.1432,
      "reward": 0.4732143133878708,
      "reward_std": 0.11274466384202242,
      "rewards/accuracy_reward": 0.01562500069849193,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4575893059372902,
      "step": 1968
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1674.76123046875,
      "epoch": 0.5881562243297738,
      "grad_norm": 38.19734573364258,
      "kl": 2.775390625,
      "learning_rate": 2.167222497258207e-07,
      "loss": 0.1936,
      "reward": 0.5033482387661934,
      "reward_std": 0.15683877281844616,
      "rewards/accuracy_reward": 0.07142857392318547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.431919664144516,
      "step": 1969
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1682.72998046875,
      "epoch": 0.5884549324172952,
      "grad_norm": 6.867143630981445,
      "kl": 3.19921875,
      "learning_rate": 2.1646383194725253e-07,
      "loss": 0.1959,
      "reward": 0.5128348395228386,
      "reward_std": 0.10380838066339493,
      "rewards/accuracy_reward": 0.07589286053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419813156128,
      "step": 1970
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1713.0201416015625,
      "epoch": 0.5887536405048167,
      "grad_norm": 12.690998077392578,
      "kl": 3.142578125,
      "learning_rate": 2.162054506527484e-07,
      "loss": 0.1879,
      "reward": 0.5396205559372902,
      "reward_std": 0.13667017593979836,
      "rewards/accuracy_reward": 0.09375000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4458705559372902,
      "step": 1971
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1629.6875610351562,
      "epoch": 0.5890523485923381,
      "grad_norm": 33.08745574951172,
      "kl": 2.451171875,
      "learning_rate": 2.159471061234017e-07,
      "loss": 0.1754,
      "reward": 0.7036830633878708,
      "reward_std": 0.1653738860040903,
      "rewards/accuracy_reward": 0.2522321492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.451450914144516,
      "step": 1972
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1708.5357971191406,
      "epoch": 0.5893510566798597,
      "grad_norm": 46.91404724121094,
      "kl": 4.734375,
      "learning_rate": 2.15688798640266e-07,
      "loss": 0.2664,
      "reward": 0.5429687649011612,
      "reward_std": 0.19622131809592247,
      "rewards/accuracy_reward": 0.10714286146685481,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435825914144516,
      "step": 1973
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1531.7723999023438,
      "epoch": 0.5896497647673811,
      "grad_norm": 21.30575942993164,
      "kl": 2.240234375,
      "learning_rate": 2.154305284843545e-07,
      "loss": 0.1571,
      "reward": 0.5440848618745804,
      "reward_std": 0.14968426525592804,
      "rewards/accuracy_reward": 0.09151785937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4525669813156128,
      "step": 1974
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1658.5067749023438,
      "epoch": 0.5899484728549026,
      "grad_norm": 88.57762908935547,
      "kl": 4.6328125,
      "learning_rate": 2.1517229593663985e-07,
      "loss": 0.2536,
      "reward": 0.5976562649011612,
      "reward_std": 0.16410492360591888,
      "rewards/accuracy_reward": 0.14955357881262898,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4481026977300644,
      "step": 1975
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1658.4107971191406,
      "epoch": 0.590247180942424,
      "grad_norm": 166.81121826171875,
      "kl": 7.125,
      "learning_rate": 2.1491410127805377e-07,
      "loss": 0.3638,
      "reward": 0.5530134066939354,
      "reward_std": 0.1174120381474495,
      "rewards/accuracy_reward": 0.1183035746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098395228386,
      "step": 1976
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1601.9063415527344,
      "epoch": 0.5905458890299455,
      "grad_norm": 73.37601470947266,
      "kl": 6.1953125,
      "learning_rate": 2.146559447894866e-07,
      "loss": 0.3785,
      "reward": 0.5251116305589676,
      "reward_std": 0.1434258446097374,
      "rewards/accuracy_reward": 0.098214291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4268973395228386,
      "step": 1977
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1685.2076721191406,
      "epoch": 0.590844597117467,
      "grad_norm": 10.409059524536133,
      "kl": 4.07421875,
      "learning_rate": 2.1439782675178752e-07,
      "loss": 0.2518,
      "reward": 0.5719866454601288,
      "reward_std": 0.15977041609585285,
      "rewards/accuracy_reward": 0.1428571492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294813156128,
      "step": 1978
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1649.2969360351562,
      "epoch": 0.5911433052049885,
      "grad_norm": 64.11497497558594,
      "kl": 3.93359375,
      "learning_rate": 2.1413974744576355e-07,
      "loss": 0.2074,
      "reward": 0.5228794887661934,
      "reward_std": 0.1752008181065321,
      "rewards/accuracy_reward": 0.07812500302679837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4447544887661934,
      "step": 1979
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1621.4107666015625,
      "epoch": 0.5914420132925099,
      "grad_norm": 15.70301342010498,
      "kl": 3.025390625,
      "learning_rate": 2.1388170715217966e-07,
      "loss": 0.1899,
      "reward": 0.5156250223517418,
      "reward_std": 0.13348249904811382,
      "rewards/accuracy_reward": 0.06250000419095159,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4531250223517418,
      "step": 1980
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1663.8170471191406,
      "epoch": 0.5917407213800313,
      "grad_norm": 10.870034217834473,
      "kl": 2.86328125,
      "learning_rate": 2.1362370615175834e-07,
      "loss": 0.1746,
      "reward": 0.5848214700818062,
      "reward_std": 0.16083652526140213,
      "rewards/accuracy_reward": 0.13392857369035482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4508928805589676,
      "step": 1981
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1634.7411193847656,
      "epoch": 0.5920394294675528,
      "grad_norm": 51.173702239990234,
      "kl": 3.93359375,
      "learning_rate": 2.1336574472517953e-07,
      "loss": 0.2281,
      "reward": 0.5117187723517418,
      "reward_std": 0.1355637740343809,
      "rewards/accuracy_reward": 0.06026786030270159,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.451450914144516,
      "step": 1982
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1641.3371276855469,
      "epoch": 0.5923381375550743,
      "grad_norm": 30.83989143371582,
      "kl": 3.3125,
      "learning_rate": 2.1310782315307995e-07,
      "loss": 0.2157,
      "reward": 0.5189732387661934,
      "reward_std": 0.1435831133276224,
      "rewards/accuracy_reward": 0.06696428754366934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4520089477300644,
      "step": 1983
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1723.9732971191406,
      "epoch": 0.5926368456425958,
      "grad_norm": 5.076685905456543,
      "kl": 3.005859375,
      "learning_rate": 2.1284994171605288e-07,
      "loss": 0.1816,
      "reward": 0.4787946715950966,
      "reward_std": 0.11630864068865776,
      "rewards/accuracy_reward": 0.029017859371379018,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4497768133878708,
      "step": 1984
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1708.4933776855469,
      "epoch": 0.5929355537301172,
      "grad_norm": 5.019188404083252,
      "kl": 3.35546875,
      "learning_rate": 2.125921006946482e-07,
      "loss": 0.2124,
      "reward": 0.647879496216774,
      "reward_std": 0.19821508787572384,
      "rewards/accuracy_reward": 0.2098214402794838,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4380580559372902,
      "step": 1985
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1566.1920471191406,
      "epoch": 0.5932342618176387,
      "grad_norm": 24.845048904418945,
      "kl": 2.396484375,
      "learning_rate": 2.1233430036937156e-07,
      "loss": 0.211,
      "reward": 0.5267857313156128,
      "reward_std": 0.12628904543817043,
      "rewards/accuracy_reward": 0.06696428824216127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4598214477300644,
      "step": 1986
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1715.0915832519531,
      "epoch": 0.5935329699051601,
      "grad_norm": 22.309974670410156,
      "kl": 2.78515625,
      "learning_rate": 2.120765410206845e-07,
      "loss": 0.1725,
      "reward": 0.5005580484867096,
      "reward_std": 0.12635646015405655,
      "rewards/accuracy_reward": 0.053571430733427405,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.446986623108387,
      "step": 1987
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1579.6340026855469,
      "epoch": 0.5938316779926817,
      "grad_norm": 13.213762283325195,
      "kl": 2.375,
      "learning_rate": 2.1181882292900386e-07,
      "loss": 0.1692,
      "reward": 0.5524553805589676,
      "reward_std": 0.16034020483493805,
      "rewards/accuracy_reward": 0.10044643585570157,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4520089477300644,
      "step": 1988
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1570.2277221679688,
      "epoch": 0.5941303860802031,
      "grad_norm": 25.873594284057617,
      "kl": 2.1796875,
      "learning_rate": 2.1156114637470178e-07,
      "loss": 0.1642,
      "reward": 0.606584832072258,
      "reward_std": 0.13057277165353298,
      "rewards/accuracy_reward": 0.1450892947614193,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4614955559372902,
      "step": 1989
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1673.118408203125,
      "epoch": 0.5944290941677246,
      "grad_norm": 20.207181930541992,
      "kl": 2.33984375,
      "learning_rate": 2.1130351163810502e-07,
      "loss": 0.1407,
      "reward": 0.584263414144516,
      "reward_std": 0.18858056142926216,
      "rewards/accuracy_reward": 0.12723215110599995,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4570312723517418,
      "step": 1990
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1631.0357971191406,
      "epoch": 0.594727802255246,
      "grad_norm": 30.100889205932617,
      "kl": 3.125,
      "learning_rate": 2.1104591899949487e-07,
      "loss": 0.2222,
      "reward": 0.4575893133878708,
      "reward_std": 0.10735885240137577,
      "rewards/accuracy_reward": 0.006696428870782256,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4508928880095482,
      "step": 1991
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1589.4085693359375,
      "epoch": 0.5950265103427675,
      "grad_norm": 21.332429885864258,
      "kl": 2.6484375,
      "learning_rate": 2.1078836873910698e-07,
      "loss": 0.1565,
      "reward": 0.6004464477300644,
      "reward_std": 0.1508875098079443,
      "rewards/accuracy_reward": 0.13616072200238705,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4642857387661934,
      "step": 1992
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1675.1719665527344,
      "epoch": 0.595325218430289,
      "grad_norm": 19.49724006652832,
      "kl": 3.78515625,
      "learning_rate": 2.105308611371308e-07,
      "loss": 0.2449,
      "reward": 0.474330373108387,
      "reward_std": 0.15705344267189503,
      "rewards/accuracy_reward": 0.029017859138548374,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125298023224,
      "step": 1993
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1657.3906860351562,
      "epoch": 0.5956239265178105,
      "grad_norm": 32.924652099609375,
      "kl": 3.5234375,
      "learning_rate": 2.1027339647370926e-07,
      "loss": 0.1961,
      "reward": 0.4693080484867096,
      "reward_std": 0.12829893082380295,
      "rewards/accuracy_reward": 0.02008928661234677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4492187649011612,
      "step": 1994
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1736.8237609863281,
      "epoch": 0.5959226346053319,
      "grad_norm": 83.11372375488281,
      "kl": 5.58203125,
      "learning_rate": 2.100159750289387e-07,
      "loss": 0.3079,
      "reward": 0.5318080633878708,
      "reward_std": 0.17447687312960625,
      "rewards/accuracy_reward": 0.09375000465661287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4380580559372902,
      "step": 1995
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1739.4911804199219,
      "epoch": 0.5962213426928534,
      "grad_norm": 22.46120834350586,
      "kl": 4.15234375,
      "learning_rate": 2.097585970828685e-07,
      "loss": 0.2448,
      "reward": 0.5641741454601288,
      "reward_std": 0.1256833542138338,
      "rewards/accuracy_reward": 0.1205357201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4436384215950966,
      "step": 1996
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1659.4554443359375,
      "epoch": 0.5965200507803748,
      "grad_norm": 20.094161987304688,
      "kl": 3.474609375,
      "learning_rate": 2.095012629155006e-07,
      "loss": 0.2151,
      "reward": 0.5597098618745804,
      "reward_std": 0.1981860287487507,
      "rewards/accuracy_reward": 0.11383928824216127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4458705559372902,
      "step": 1997
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1695.29248046875,
      "epoch": 0.5968187588678964,
      "grad_norm": 16.4444522857666,
      "kl": 3.83203125,
      "learning_rate": 2.092439728067893e-07,
      "loss": 0.2343,
      "reward": 0.5619419887661934,
      "reward_std": 0.15152296982705593,
      "rewards/accuracy_reward": 0.1316964365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455559372902,
      "step": 1998
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1652.2702026367188,
      "epoch": 0.5971174669554178,
      "grad_norm": 6.168159008026123,
      "kl": 3.74609375,
      "learning_rate": 2.0898672703664112e-07,
      "loss": 0.2429,
      "reward": 0.4581473469734192,
      "reward_std": 0.14216610044240952,
      "rewards/accuracy_reward": 0.029017859371379018,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294813156128,
      "step": 1999
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1673.3416137695312,
      "epoch": 0.5974161750429393,
      "grad_norm": 29.924734115600586,
      "kl": 1.763671875,
      "learning_rate": 2.0872952588491415e-07,
      "loss": 0.1207,
      "reward": 0.5764509290456772,
      "reward_std": 0.1512145884335041,
      "rewards/accuracy_reward": 0.11607143399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4603794887661934,
      "step": 2000
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2007.0134582519531,
      "epoch": 0.5977148831304607,
      "grad_norm": 19.01376724243164,
      "kl": 1.60546875,
      "learning_rate": 2.0847236963141812e-07,
      "loss": 0.1022,
      "reward": 0.2494419738650322,
      "reward_std": 0.10328670870512724,
      "rewards/accuracy_reward": 0.008928572060540318,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2405133992433548,
      "step": 2001
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1958.4598999023438,
      "epoch": 0.5980135912179823,
      "grad_norm": 22.44007110595703,
      "kl": 1.5703125,
      "learning_rate": 2.082152585559137e-07,
      "loss": 0.1331,
      "reward": 0.3777901977300644,
      "reward_std": 0.10609771031886339,
      "rewards/accuracy_reward": 0.1250000074505806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2527901940047741,
      "step": 2002
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2026.2835388183594,
      "epoch": 0.5983122993055037,
      "grad_norm": 17.095685958862305,
      "kl": 1.673828125,
      "learning_rate": 2.079581929381127e-07,
      "loss": 0.0902,
      "reward": 0.3867187649011612,
      "reward_std": 0.09345119819045067,
      "rewards/accuracy_reward": 0.1517857201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2349330484867096,
      "step": 2003
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2010.7277526855469,
      "epoch": 0.5986110073930252,
      "grad_norm": 20.289731979370117,
      "kl": 1.697265625,
      "learning_rate": 2.0770117305767726e-07,
      "loss": 0.0975,
      "reward": 0.2449776865541935,
      "reward_std": 0.07792892586439848,
      "rewards/accuracy_reward": 0.0022321429569274187,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2427455447614193,
      "step": 2004
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1989.1385192871094,
      "epoch": 0.5989097154805466,
      "grad_norm": 22.676464080810547,
      "kl": 1.71875,
      "learning_rate": 2.074441991942197e-07,
      "loss": 0.1194,
      "reward": 0.3219866193830967,
      "reward_std": 0.1178138442337513,
      "rewards/accuracy_reward": 0.08482143399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2371651902794838,
      "step": 2005
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2013.8550109863281,
      "epoch": 0.5992084235680681,
      "grad_norm": 29.393930435180664,
      "kl": 1.734375,
      "learning_rate": 2.0718727162730259e-07,
      "loss": 0.0967,
      "reward": 0.3476562798023224,
      "reward_std": 0.08774643391370773,
      "rewards/accuracy_reward": 0.11160714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2360491156578064,
      "step": 2006
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2016.9019165039062,
      "epoch": 0.5995071316555896,
      "grad_norm": 22.131059646606445,
      "kl": 1.76171875,
      "learning_rate": 2.0693039063643784e-07,
      "loss": 0.0932,
      "reward": 0.2444196529686451,
      "reward_std": 0.07599498424679041,
      "rewards/accuracy_reward": 0.004464285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2399553693830967,
      "step": 2007
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1983.3639221191406,
      "epoch": 0.5998058397431111,
      "grad_norm": 27.39740753173828,
      "kl": 1.734375,
      "learning_rate": 2.0667355650108682e-07,
      "loss": 0.1162,
      "reward": 0.2572544738650322,
      "reward_std": 0.07956382818520069,
      "rewards/accuracy_reward": 0.0133928582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2438616156578064,
      "step": 2008
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2018.8773193359375,
      "epoch": 0.6001045478306325,
      "grad_norm": 27.842668533325195,
      "kl": 1.82421875,
      "learning_rate": 2.064167695006598e-07,
      "loss": 0.1015,
      "reward": 0.2700892984867096,
      "reward_std": 0.06435556150972843,
      "rewards/accuracy_reward": 0.0357142873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2343750111758709,
      "step": 2009
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1995.9130249023438,
      "epoch": 0.600403255918154,
      "grad_norm": 30.926021575927734,
      "kl": 1.82421875,
      "learning_rate": 2.0616002991451602e-07,
      "loss": 0.1094,
      "reward": 0.2851562649011612,
      "reward_std": 0.07530072797089815,
      "rewards/accuracy_reward": 0.03794643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.247209832072258,
      "step": 2010
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1957.8884887695312,
      "epoch": 0.6007019640056754,
      "grad_norm": 35.56877899169922,
      "kl": 1.734375,
      "learning_rate": 2.0590333802196293e-07,
      "loss": 0.128,
      "reward": 0.279017873108387,
      "reward_std": 0.09334464650601149,
      "rewards/accuracy_reward": 0.0223214291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2566964402794838,
      "step": 2011
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1993.5223999023438,
      "epoch": 0.601000672093197,
      "grad_norm": 35.18021011352539,
      "kl": 1.998046875,
      "learning_rate": 2.0564669410225606e-07,
      "loss": 0.0972,
      "reward": 0.3526785895228386,
      "reward_std": 0.060251260176301,
      "rewards/accuracy_reward": 0.1116071492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2410714365541935,
      "step": 2012
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1958.9353637695312,
      "epoch": 0.6012993801807184,
      "grad_norm": 38.1002311706543,
      "kl": 1.8515625,
      "learning_rate": 2.0539009843459895e-07,
      "loss": 0.1373,
      "reward": 0.301897332072258,
      "reward_std": 0.10135246813297272,
      "rewards/accuracy_reward": 0.0446428582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2572544738650322,
      "step": 2013
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1948.46435546875,
      "epoch": 0.6015980882682399,
      "grad_norm": 41.63920211791992,
      "kl": 1.94921875,
      "learning_rate": 2.0513355129814252e-07,
      "loss": 0.1188,
      "reward": 0.3035714477300644,
      "reward_std": 0.09025296848267317,
      "rewards/accuracy_reward": 0.04687500232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2566964402794838,
      "step": 2014
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1944.9353637695312,
      "epoch": 0.6018967963557613,
      "grad_norm": 75.75386047363281,
      "kl": 1.83984375,
      "learning_rate": 2.048770529719849e-07,
      "loss": 0.1295,
      "reward": 0.313058041036129,
      "reward_std": 0.10595975816249847,
      "rewards/accuracy_reward": 0.051339288242161274,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2617187574505806,
      "step": 2015
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1971.0625610351562,
      "epoch": 0.6021955044432828,
      "grad_norm": 88.37937927246094,
      "kl": 1.90625,
      "learning_rate": 2.0462060373517106e-07,
      "loss": 0.1249,
      "reward": 0.3041294738650322,
      "reward_std": 0.08976395009085536,
      "rewards/accuracy_reward": 0.049107146449387074,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2550223246216774,
      "step": 2016
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1994.0827026367188,
      "epoch": 0.6024942125308043,
      "grad_norm": 76.98424530029297,
      "kl": 1.95703125,
      "learning_rate": 2.0436420386669275e-07,
      "loss": 0.1096,
      "reward": 0.247209832072258,
      "reward_std": 0.04999090917408466,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.247209832072258,
      "step": 2017
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1954.884033203125,
      "epoch": 0.6027929206183258,
      "grad_norm": 185.7693634033203,
      "kl": 1.51953125,
      "learning_rate": 2.0410785364548785e-07,
      "loss": 0.1068,
      "reward": 0.290178582072258,
      "reward_std": 0.08779922686517239,
      "rewards/accuracy_reward": 0.0379464291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2522321529686451,
      "step": 2018
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1966.5357971191406,
      "epoch": 0.6030916287058472,
      "grad_norm": 93.97416687011719,
      "kl": 1.88671875,
      "learning_rate": 2.0385155335044023e-07,
      "loss": 0.1024,
      "reward": 0.3191964514553547,
      "reward_std": 0.07120133191347122,
      "rewards/accuracy_reward": 0.07589286053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2433035783469677,
      "step": 2019
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1930.4598999023438,
      "epoch": 0.6033903367933687,
      "grad_norm": 138.41859436035156,
      "kl": 1.78125,
      "learning_rate": 2.0359530326037955e-07,
      "loss": 0.1427,
      "reward": 0.3638393133878708,
      "reward_std": 0.08370322547852993,
      "rewards/accuracy_reward": 0.11160714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2522321492433548,
      "step": 2020
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1942.587158203125,
      "epoch": 0.6036890448808901,
      "grad_norm": 164.16201782226562,
      "kl": 1.666015625,
      "learning_rate": 2.033391036540808e-07,
      "loss": 0.1273,
      "reward": 0.352120541036129,
      "reward_std": 0.09485107287764549,
      "rewards/accuracy_reward": 0.0892857164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2628348395228386,
      "step": 2021
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1941.7300109863281,
      "epoch": 0.6039877529684117,
      "grad_norm": 156.23495483398438,
      "kl": 1.669921875,
      "learning_rate": 2.03082954810264e-07,
      "loss": 0.1224,
      "reward": 0.286830373108387,
      "reward_std": 0.06754599511623383,
      "rewards/accuracy_reward": 0.0357142873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.251116082072258,
      "step": 2022
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1953.6139221191406,
      "epoch": 0.6042864610559331,
      "grad_norm": 163.2364044189453,
      "kl": 1.533203125,
      "learning_rate": 2.0282685700759397e-07,
      "loss": 0.1152,
      "reward": 0.3297991268336773,
      "reward_std": 0.06811636220663786,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2583705447614193,
      "step": 2023
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1930.8281860351562,
      "epoch": 0.6045851691434545,
      "grad_norm": 130.30120849609375,
      "kl": 1.75390625,
      "learning_rate": 2.0257081052468007e-07,
      "loss": 0.1284,
      "reward": 0.2734375223517418,
      "reward_std": 0.08655294217169285,
      "rewards/accuracy_reward": 0.013392857741564512,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2600446566939354,
      "step": 2024
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1946.7702026367188,
      "epoch": 0.604883877230976,
      "grad_norm": 132.37033081054688,
      "kl": 1.76171875,
      "learning_rate": 2.0231481564007579e-07,
      "loss": 0.1151,
      "reward": 0.3387276902794838,
      "reward_std": 0.0722337681800127,
      "rewards/accuracy_reward": 0.0781250037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2606026828289032,
      "step": 2025
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1922.9264526367188,
      "epoch": 0.6051825853184974,
      "grad_norm": 202.0115966796875,
      "kl": 1.666015625,
      "learning_rate": 2.020588726322784e-07,
      "loss": 0.1389,
      "reward": 0.3085937649011612,
      "reward_std": 0.09983004443347454,
      "rewards/accuracy_reward": 0.04017857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2684151902794838,
      "step": 2026
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1940.4331359863281,
      "epoch": 0.605481293406019,
      "grad_norm": 113.05741119384766,
      "kl": 1.4765625,
      "learning_rate": 2.0180298177972888e-07,
      "loss": 0.1023,
      "reward": 0.3074776902794838,
      "reward_std": 0.08548478037118912,
      "rewards/accuracy_reward": 0.04687500116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2606026828289032,
      "step": 2027
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1916.3973999023438,
      "epoch": 0.6057800014935404,
      "grad_norm": 172.66815185546875,
      "kl": 1.373046875,
      "learning_rate": 2.0154714336081135e-07,
      "loss": 0.1127,
      "reward": 0.3437500149011612,
      "reward_std": 0.10583369247615337,
      "rewards/accuracy_reward": 0.07812500116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2656250223517418,
      "step": 2028
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1864.9576416015625,
      "epoch": 0.6060787095810619,
      "grad_norm": 196.3884735107422,
      "kl": 1.265625,
      "learning_rate": 2.0129135765385297e-07,
      "loss": 0.1341,
      "reward": 0.3152901902794838,
      "reward_std": 0.11865677125751972,
      "rewards/accuracy_reward": 0.04687500116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2684151828289032,
      "step": 2029
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1896.5156860351562,
      "epoch": 0.6063774176685833,
      "grad_norm": 96.03591918945312,
      "kl": 1.05078125,
      "learning_rate": 2.010356249371234e-07,
      "loss": 0.1336,
      "reward": 0.3164062574505806,
      "reward_std": 0.1234282087534666,
      "rewards/accuracy_reward": 0.04687500232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2695312574505806,
      "step": 2030
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1825.9197082519531,
      "epoch": 0.6066761257561049,
      "grad_norm": 81.84765625,
      "kl": 0.9638671875,
      "learning_rate": 2.0077994548883493e-07,
      "loss": 0.1172,
      "reward": 0.3895089402794838,
      "reward_std": 0.15263395570218563,
      "rewards/accuracy_reward": 0.11160714668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2779017984867096,
      "step": 2031
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1894.4175109863281,
      "epoch": 0.6069748338436263,
      "grad_norm": 94.24299621582031,
      "kl": 1.181640625,
      "learning_rate": 2.0052431958714174e-07,
      "loss": 0.1031,
      "reward": 0.3069196566939354,
      "reward_std": 0.0905121136456728,
      "rewards/accuracy_reward": 0.04241071757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2645089402794838,
      "step": 2032
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1932.4978637695312,
      "epoch": 0.6072735419311478,
      "grad_norm": 100.99922943115234,
      "kl": 1.212890625,
      "learning_rate": 2.002687475101397e-07,
      "loss": 0.1005,
      "reward": 0.3052455484867096,
      "reward_std": 0.07347106747329235,
      "rewards/accuracy_reward": 0.0424107164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.262834832072258,
      "step": 2033
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1916.0246276855469,
      "epoch": 0.6075722500186692,
      "grad_norm": 82.82646942138672,
      "kl": 1.1640625,
      "learning_rate": 2.000132295358662e-07,
      "loss": 0.0869,
      "reward": 0.282366082072258,
      "reward_std": 0.08921909797936678,
      "rewards/accuracy_reward": 0.011160715017467737,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.271205373108387,
      "step": 2034
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1923.6563415527344,
      "epoch": 0.6078709581061907,
      "grad_norm": 87.85645294189453,
      "kl": 1.103515625,
      "learning_rate": 1.9975776594229965e-07,
      "loss": 0.1112,
      "reward": 0.3750000074505806,
      "reward_std": 0.08628001902252436,
      "rewards/accuracy_reward": 0.113839291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2611607238650322,
      "step": 2035
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1911.1675415039062,
      "epoch": 0.6081696661937122,
      "grad_norm": 87.55707550048828,
      "kl": 1.091796875,
      "learning_rate": 1.9950235700735953e-07,
      "loss": 0.0917,
      "reward": 0.2851562574505806,
      "reward_std": 0.10026535391807556,
      "rewards/accuracy_reward": 0.01562500116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2695312574505806,
      "step": 2036
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1900.3281860351562,
      "epoch": 0.6084683742812337,
      "grad_norm": 95.55062866210938,
      "kl": 1.154296875,
      "learning_rate": 1.992470030089057e-07,
      "loss": 0.1061,
      "reward": 0.3236607313156128,
      "reward_std": 0.10007515829056501,
      "rewards/accuracy_reward": 0.05357143096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2700892984867096,
      "step": 2037
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1925.9665832519531,
      "epoch": 0.6087670823687551,
      "grad_norm": 69.78916931152344,
      "kl": 1.0185546875,
      "learning_rate": 1.9899170422473825e-07,
      "loss": 0.0856,
      "reward": 0.322544664144516,
      "reward_std": 0.10184823349118233,
      "rewards/accuracy_reward": 0.0513392873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.271205373108387,
      "step": 2038
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1872.0112609863281,
      "epoch": 0.6090657904562766,
      "grad_norm": 90.38148498535156,
      "kl": 1.095703125,
      "learning_rate": 1.9873646093259727e-07,
      "loss": 0.099,
      "reward": 0.4034598395228386,
      "reward_std": 0.13785859383642673,
      "rewards/accuracy_reward": 0.1316964365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2717633992433548,
      "step": 2039
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1861.9465026855469,
      "epoch": 0.609364498543798,
      "grad_norm": 68.22972869873047,
      "kl": 0.9384765625,
      "learning_rate": 1.9848127341016244e-07,
      "loss": 0.1023,
      "reward": 0.4012277014553547,
      "reward_std": 0.12528756260871887,
      "rewards/accuracy_reward": 0.12276786682195961,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2784598357975483,
      "step": 2040
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1892.946533203125,
      "epoch": 0.6096632066313196,
      "grad_norm": 62.6482048034668,
      "kl": 0.9833984375,
      "learning_rate": 1.9822614193505275e-07,
      "loss": 0.1073,
      "reward": 0.2806919738650322,
      "reward_std": 0.10459551215171814,
      "rewards/accuracy_reward": 0.006696428870782256,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2739955484867096,
      "step": 2041
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1854.4509887695312,
      "epoch": 0.609961914718841,
      "grad_norm": 60.31330490112305,
      "kl": 0.8564453125,
      "learning_rate": 1.979710667848262e-07,
      "loss": 0.1283,
      "reward": 0.3314732238650322,
      "reward_std": 0.12542915157973766,
      "rewards/accuracy_reward": 0.04687500186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2845982238650322,
      "step": 2042
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1887.2567749023438,
      "epoch": 0.6102606228063625,
      "grad_norm": 62.87117385864258,
      "kl": 0.9365234375,
      "learning_rate": 1.9771604823697966e-07,
      "loss": 0.0959,
      "reward": 0.297991082072258,
      "reward_std": 0.1309618316590786,
      "rewards/accuracy_reward": 0.0223214291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2756696566939354,
      "step": 2043
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1903.8572692871094,
      "epoch": 0.6105593308938839,
      "grad_norm": 33.77011489868164,
      "kl": 0.82421875,
      "learning_rate": 1.9746108656894826e-07,
      "loss": 0.0797,
      "reward": 0.3521205484867096,
      "reward_std": 0.10524630546569824,
      "rewards/accuracy_reward": 0.0736607164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.278459832072258,
      "step": 2044
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1872.1540832519531,
      "epoch": 0.6108580389814054,
      "grad_norm": 30.331754684448242,
      "kl": 0.7509765625,
      "learning_rate": 1.9720618205810527e-07,
      "loss": 0.0974,
      "reward": 0.3253348395228386,
      "reward_std": 0.1536751687526703,
      "rewards/accuracy_reward": 0.024553572991862893,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3007812649011612,
      "step": 2045
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1883.1942749023438,
      "epoch": 0.6111567470689269,
      "grad_norm": 64.93643188476562,
      "kl": 0.845703125,
      "learning_rate": 1.9695133498176187e-07,
      "loss": 0.1039,
      "reward": 0.2952009066939354,
      "reward_std": 0.14137830026447773,
      "rewards/accuracy_reward": 0.011160715017467737,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2840401902794838,
      "step": 2046
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1883.3505554199219,
      "epoch": 0.6114554551564484,
      "grad_norm": 27.461055755615234,
      "kl": 0.7763671875,
      "learning_rate": 1.966965456171667e-07,
      "loss": 0.0923,
      "reward": 0.3537946566939354,
      "reward_std": 0.11342634446918964,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.282366082072258,
      "step": 2047
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1896.0513916015625,
      "epoch": 0.6117541632439698,
      "grad_norm": 26.486169815063477,
      "kl": 0.7763671875,
      "learning_rate": 1.964418142415056e-07,
      "loss": 0.0729,
      "reward": 0.3247767984867096,
      "reward_std": 0.12322649359703064,
      "rewards/accuracy_reward": 0.04241071501746774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2823660895228386,
      "step": 2048
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1924.8415832519531,
      "epoch": 0.6120528713314913,
      "grad_norm": 21.21870994567871,
      "kl": 0.7314453125,
      "learning_rate": 1.9618714113190128e-07,
      "loss": 0.0743,
      "reward": 0.321986623108387,
      "reward_std": 0.09982621483504772,
      "rewards/accuracy_reward": 0.0379464291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2840401902794838,
      "step": 2049
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1888.7366943359375,
      "epoch": 0.6123515794190127,
      "grad_norm": 31.675918579101562,
      "kl": 0.724609375,
      "learning_rate": 1.9593252656541325e-07,
      "loss": 0.0982,
      "reward": 0.3783482313156128,
      "reward_std": 0.14071377366781235,
      "rewards/accuracy_reward": 0.08258928963914514,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2957589402794838,
      "step": 2050
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1837.0603332519531,
      "epoch": 0.6126502875065343,
      "grad_norm": 19.6038875579834,
      "kl": 0.6416015625,
      "learning_rate": 1.9567797081903714e-07,
      "loss": 0.116,
      "reward": 0.3805803805589676,
      "reward_std": 0.16843822970986366,
      "rewards/accuracy_reward": 0.07366071734577417,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3069196566939354,
      "step": 2051
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1881.3482971191406,
      "epoch": 0.6129489955940557,
      "grad_norm": 21.44715118408203,
      "kl": 0.6943359375,
      "learning_rate": 1.9542347416970463e-07,
      "loss": 0.1001,
      "reward": 0.4095982313156128,
      "reward_std": 0.14218728058040142,
      "rewards/accuracy_reward": 0.1183035746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2912946566939354,
      "step": 2052
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1919.774658203125,
      "epoch": 0.6132477036815772,
      "grad_norm": 14.921602249145508,
      "kl": 0.6875,
      "learning_rate": 1.951690368942831e-07,
      "loss": 0.0824,
      "reward": 0.3292410895228386,
      "reward_std": 0.12462631799280643,
      "rewards/accuracy_reward": 0.04017857206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2890625149011612,
      "step": 2053
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1904.2523193359375,
      "epoch": 0.6135464117690986,
      "grad_norm": 8.613449096679688,
      "kl": 0.6416015625,
      "learning_rate": 1.9491465926957546e-07,
      "loss": 0.0999,
      "reward": 0.3945312723517418,
      "reward_std": 0.11939899995923042,
      "rewards/accuracy_reward": 0.1071428619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2873883992433548,
      "step": 2054
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1891.8705749511719,
      "epoch": 0.6138451198566202,
      "grad_norm": 7.6890082359313965,
      "kl": 0.630859375,
      "learning_rate": 1.9466034157231959e-07,
      "loss": 0.0801,
      "reward": 0.3041294738650322,
      "reward_std": 0.13820013962686062,
      "rewards/accuracy_reward": 0.011160714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2929687649011612,
      "step": 2055
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1915.8058471679688,
      "epoch": 0.6141438279441416,
      "grad_norm": 6.97948694229126,
      "kl": 0.59765625,
      "learning_rate": 1.9440608407918817e-07,
      "loss": 0.0809,
      "reward": 0.333705373108387,
      "reward_std": 0.1400212198495865,
      "rewards/accuracy_reward": 0.046875003492459655,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.286830373108387,
      "step": 2056
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1887.3661499023438,
      "epoch": 0.6144425360316631,
      "grad_norm": 9.253793716430664,
      "kl": 0.5830078125,
      "learning_rate": 1.9415188706678857e-07,
      "loss": 0.1031,
      "reward": 0.3275669813156128,
      "reward_std": 0.17872971296310425,
      "rewards/accuracy_reward": 0.03571428684517741,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2918526828289032,
      "step": 2057
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1910.5202026367188,
      "epoch": 0.6147412441191845,
      "grad_norm": 7.67112398147583,
      "kl": 0.5751953125,
      "learning_rate": 1.938977508116621e-07,
      "loss": 0.0811,
      "reward": 0.3822544813156128,
      "reward_std": 0.16895470768213272,
      "rewards/accuracy_reward": 0.09598214388825,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.286272332072258,
      "step": 2058
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1829.6652526855469,
      "epoch": 0.615039952206706,
      "grad_norm": 14.717704772949219,
      "kl": 0.53515625,
      "learning_rate": 1.9364367559028415e-07,
      "loss": 0.1143,
      "reward": 0.4101562723517418,
      "reward_std": 0.15780089795589447,
      "rewards/accuracy_reward": 0.09375000232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3164062649011612,
      "step": 2059
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1880.62060546875,
      "epoch": 0.6153386602942275,
      "grad_norm": 15.343208312988281,
      "kl": 0.51123046875,
      "learning_rate": 1.933896616790636e-07,
      "loss": 0.0817,
      "reward": 0.3231026902794838,
      "reward_std": 0.1713215596973896,
      "rewards/accuracy_reward": 0.01562500069849193,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3074776902794838,
      "step": 2060
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1866.5000915527344,
      "epoch": 0.615637368381749,
      "grad_norm": 9.42824935913086,
      "kl": 0.5029296875,
      "learning_rate": 1.9313570935434275e-07,
      "loss": 0.0881,
      "reward": 0.3130580484867096,
      "reward_std": 0.1517094001173973,
      "rewards/accuracy_reward": 0.008928572060540318,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3041294738650322,
      "step": 2061
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1908.1585388183594,
      "epoch": 0.6159360764692704,
      "grad_norm": 8.376147270202637,
      "kl": 0.5029296875,
      "learning_rate": 1.9288181889239685e-07,
      "loss": 0.0718,
      "reward": 0.4107142984867096,
      "reward_std": 0.1813274547457695,
      "rewards/accuracy_reward": 0.1227678582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2879464402794838,
      "step": 2062
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1828.9197082519531,
      "epoch": 0.6162347845567919,
      "grad_norm": 8.778471946716309,
      "kl": 0.466796875,
      "learning_rate": 1.9262799056943377e-07,
      "loss": 0.1016,
      "reward": 0.411830373108387,
      "reward_std": 0.1879202388226986,
      "rewards/accuracy_reward": 0.09598214668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3158482313156128,
      "step": 2063
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1781.7880249023438,
      "epoch": 0.6165334926443133,
      "grad_norm": 6.207269191741943,
      "kl": 0.43359375,
      "learning_rate": 1.9237422466159392e-07,
      "loss": 0.1151,
      "reward": 0.3616071566939354,
      "reward_std": 0.1996200494468212,
      "rewards/accuracy_reward": 0.03348214505240321,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3281250149011612,
      "step": 2064
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1896.0201416015625,
      "epoch": 0.6168322007318349,
      "grad_norm": 6.0737104415893555,
      "kl": 0.49951171875,
      "learning_rate": 1.9212052144494973e-07,
      "loss": 0.0689,
      "reward": 0.4174107313156128,
      "reward_std": 0.18051912635564804,
      "rewards/accuracy_reward": 0.12053572130389512,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2968750149011612,
      "step": 2065
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1884.4107971191406,
      "epoch": 0.6171309088193563,
      "grad_norm": 7.498432159423828,
      "kl": 0.455078125,
      "learning_rate": 1.9186688119550537e-07,
      "loss": 0.0778,
      "reward": 0.2924107164144516,
      "reward_std": 0.16048693284392357,
      "rewards/accuracy_reward": 0.004464285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2879464328289032,
      "step": 2066
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1878.3862609863281,
      "epoch": 0.6174296169068777,
      "grad_norm": 6.3126983642578125,
      "kl": 0.42333984375,
      "learning_rate": 1.916133041891966e-07,
      "loss": 0.0871,
      "reward": 0.325892873108387,
      "reward_std": 0.20067593455314636,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2946428656578064,
      "step": 2067
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1874.4353637695312,
      "epoch": 0.6177283249943992,
      "grad_norm": 6.604072093963623,
      "kl": 0.408203125,
      "learning_rate": 1.9135979070189038e-07,
      "loss": 0.0921,
      "reward": 0.4179687574505806,
      "reward_std": 0.16319535300135612,
      "rewards/accuracy_reward": 0.113839291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3041294738650322,
      "step": 2068
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1804.5826721191406,
      "epoch": 0.6180270330819206,
      "grad_norm": 7.38966178894043,
      "kl": 0.37939453125,
      "learning_rate": 1.9110634100938453e-07,
      "loss": 0.1164,
      "reward": 0.428571455180645,
      "reward_std": 0.22308167815208435,
      "rewards/accuracy_reward": 0.12723214831203222,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.301339291036129,
      "step": 2069
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1885.0067749023438,
      "epoch": 0.6183257411694422,
      "grad_norm": 2.2965540885925293,
      "kl": 0.42919921875,
      "learning_rate": 1.9085295538740742e-07,
      "loss": 0.083,
      "reward": 0.356584832072258,
      "reward_std": 0.1893683262169361,
      "rewards/accuracy_reward": 0.08705357578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2695312574505806,
      "step": 2070
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1804.2344665527344,
      "epoch": 0.6186244492569636,
      "grad_norm": 3.3730814456939697,
      "kl": 0.35693359375,
      "learning_rate": 1.9059963411161787e-07,
      "loss": 0.1204,
      "reward": 0.4531250223517418,
      "reward_std": 0.17145071178674698,
      "rewards/accuracy_reward": 0.1473214328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3058035746216774,
      "step": 2071
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1877.2790832519531,
      "epoch": 0.6189231573444851,
      "grad_norm": 2.264995574951172,
      "kl": 0.37255859375,
      "learning_rate": 1.9034637745760451e-07,
      "loss": 0.0932,
      "reward": 0.3292410895228386,
      "reward_std": 0.1646197885274887,
      "rewards/accuracy_reward": 0.03794643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2912946566939354,
      "step": 2072
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1857.0067749023438,
      "epoch": 0.6192218654320065,
      "grad_norm": 3.5510640144348145,
      "kl": 0.3671875,
      "learning_rate": 1.9009318570088585e-07,
      "loss": 0.0898,
      "reward": 0.3046875149011612,
      "reward_std": 0.185846209526062,
      "rewards/accuracy_reward": 0.013392857974395156,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2912946566939354,
      "step": 2073
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1866.1898498535156,
      "epoch": 0.619520573519528,
      "grad_norm": 1.8919751644134521,
      "kl": 0.3564453125,
      "learning_rate": 1.8984005911690954e-07,
      "loss": 0.0727,
      "reward": 0.3297991305589676,
      "reward_std": 0.1632586196064949,
      "rewards/accuracy_reward": 0.04017857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2896205484867096,
      "step": 2074
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1790.6139221191406,
      "epoch": 0.6198192816070495,
      "grad_norm": 2.4560365676879883,
      "kl": 0.34033203125,
      "learning_rate": 1.8958699798105268e-07,
      "loss": 0.1238,
      "reward": 0.3242187649011612,
      "reward_std": 0.21345408260822296,
      "rewards/accuracy_reward": 0.026785716181620955,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2974330559372902,
      "step": 2075
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1823.35498046875,
      "epoch": 0.620117989694571,
      "grad_norm": 1.6656031608581543,
      "kl": 0.34326171875,
      "learning_rate": 1.8933400256862092e-07,
      "loss": 0.0942,
      "reward": 0.475446455180645,
      "reward_std": 0.19583533704280853,
      "rewards/accuracy_reward": 0.1674107238650322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3080357313156128,
      "step": 2076
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1856.97998046875,
      "epoch": 0.6204166977820924,
      "grad_norm": 2.081491470336914,
      "kl": 0.34716796875,
      "learning_rate": 1.890810731548484e-07,
      "loss": 0.088,
      "reward": 0.3733259066939354,
      "reward_std": 0.1813489869236946,
      "rewards/accuracy_reward": 0.0915178619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.281808041036129,
      "step": 2077
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1871.9598999023438,
      "epoch": 0.6207154058696139,
      "grad_norm": 2.64741587638855,
      "kl": 0.3427734375,
      "learning_rate": 1.8882821001489764e-07,
      "loss": 0.0889,
      "reward": 0.3314732238650322,
      "reward_std": 0.20153598487377167,
      "rewards/accuracy_reward": 0.05357143236324191,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2779017984867096,
      "step": 2078
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1888.7567749023438,
      "epoch": 0.6210141139571353,
      "grad_norm": 2.3889336585998535,
      "kl": 0.3154296875,
      "learning_rate": 1.8857541342385888e-07,
      "loss": 0.0924,
      "reward": 0.2968750149011612,
      "reward_std": 0.20180265977978706,
      "rewards/accuracy_reward": 0.01785714365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.279017873108387,
      "step": 2079
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1889.4777526855469,
      "epoch": 0.6213128220446569,
      "grad_norm": 1.319521427154541,
      "kl": 0.3251953125,
      "learning_rate": 1.883226836567501e-07,
      "loss": 0.0774,
      "reward": 0.329241082072258,
      "reward_std": 0.19447574391961098,
      "rewards/accuracy_reward": 0.04910714388824999,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2801339402794838,
      "step": 2080
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1810.5179443359375,
      "epoch": 0.6216115301321783,
      "grad_norm": 1.205213189125061,
      "kl": 0.296875,
      "learning_rate": 1.8807002098851636e-07,
      "loss": 0.1122,
      "reward": 0.333705373108387,
      "reward_std": 0.22254730015993118,
      "rewards/accuracy_reward": 0.03348214505240321,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3002232238650322,
      "step": 2081
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1881.915283203125,
      "epoch": 0.6219102382196998,
      "grad_norm": 0.7983621954917908,
      "kl": 0.3154296875,
      "learning_rate": 1.8781742569403008e-07,
      "loss": 0.0988,
      "reward": 0.423549123108387,
      "reward_std": 0.1816374883055687,
      "rewards/accuracy_reward": 0.1517857201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2717634066939354,
      "step": 2082
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1816.7389526367188,
      "epoch": 0.6222089463072212,
      "grad_norm": 0.8310419321060181,
      "kl": 0.2890625,
      "learning_rate": 1.875648980480901e-07,
      "loss": 0.103,
      "reward": 0.3878348395228386,
      "reward_std": 0.21784145012497902,
      "rewards/accuracy_reward": 0.07812500116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.309709832072258,
      "step": 2083
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1836.32373046875,
      "epoch": 0.6225076543947428,
      "grad_norm": 1.8516379594802856,
      "kl": 0.30078125,
      "learning_rate": 1.873124383254217e-07,
      "loss": 0.1052,
      "reward": 0.408482164144516,
      "reward_std": 0.1849219687283039,
      "rewards/accuracy_reward": 0.1093750037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2991071566939354,
      "step": 2084
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1867.2322387695312,
      "epoch": 0.6228063624822642,
      "grad_norm": 1.6333746910095215,
      "kl": 0.3291015625,
      "learning_rate": 1.8706004680067644e-07,
      "loss": 0.1025,
      "reward": 0.4190848395228386,
      "reward_std": 0.17525043338537216,
      "rewards/accuracy_reward": 0.12723214668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2918526977300644,
      "step": 2085
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1808.4019165039062,
      "epoch": 0.6231050705697857,
      "grad_norm": 1.0347784757614136,
      "kl": 0.294921875,
      "learning_rate": 1.8680772374843145e-07,
      "loss": 0.1268,
      "reward": 0.3632812649011612,
      "reward_std": 0.203450296074152,
      "rewards/accuracy_reward": 0.06473214668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2985491305589676,
      "step": 2086
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1822.8304443359375,
      "epoch": 0.6234037786573071,
      "grad_norm": 1.7334294319152832,
      "kl": 0.296875,
      "learning_rate": 1.865554694431895e-07,
      "loss": 0.1078,
      "reward": 0.3800223395228386,
      "reward_std": 0.20891457423567772,
      "rewards/accuracy_reward": 0.06696429057046771,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3130580484867096,
      "step": 2087
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1807.9822387695312,
      "epoch": 0.6237024867448286,
      "grad_norm": 1.623219609260559,
      "kl": 0.3056640625,
      "learning_rate": 1.8630328415937854e-07,
      "loss": 0.1221,
      "reward": 0.3900669813156128,
      "reward_std": 0.18368515372276306,
      "rewards/accuracy_reward": 0.07366071734577417,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3164062649011612,
      "step": 2088
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1761.2322387695312,
      "epoch": 0.6240011948323501,
      "grad_norm": 1.5474106073379517,
      "kl": 0.279052734375,
      "learning_rate": 1.8605116817135149e-07,
      "loss": 0.1032,
      "reward": 0.3939732313156128,
      "reward_std": 0.1731840819120407,
      "rewards/accuracy_reward": 0.05803571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3359375223517418,
      "step": 2089
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1857.1072387695312,
      "epoch": 0.6242999029198716,
      "grad_norm": 1.4291529655456543,
      "kl": 0.3056640625,
      "learning_rate": 1.8579912175338586e-07,
      "loss": 0.1094,
      "reward": 0.3638392984867096,
      "reward_std": 0.17072179168462753,
      "rewards/accuracy_reward": 0.049107145285233855,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3147321566939354,
      "step": 2090
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1780.3728637695312,
      "epoch": 0.624598611007393,
      "grad_norm": 1.7529492378234863,
      "kl": 0.28759765625,
      "learning_rate": 1.8554714517968328e-07,
      "loss": 0.1381,
      "reward": 0.3967634066939354,
      "reward_std": 0.1704740896821022,
      "rewards/accuracy_reward": 0.06250000232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3342634066939354,
      "step": 2091
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1817.4576416015625,
      "epoch": 0.6248973190949145,
      "grad_norm": 1.4974085092544556,
      "kl": 0.3212890625,
      "learning_rate": 1.8529523872436977e-07,
      "loss": 0.1111,
      "reward": 0.4229910895228386,
      "reward_std": 0.1995687335729599,
      "rewards/accuracy_reward": 0.1071428619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3158482238650322,
      "step": 2092
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1762.3728332519531,
      "epoch": 0.6251960271824359,
      "grad_norm": 1.1088848114013672,
      "kl": 0.30078125,
      "learning_rate": 1.850434026614948e-07,
      "loss": 0.1091,
      "reward": 0.3906250149011612,
      "reward_std": 0.15429268591105938,
      "rewards/accuracy_reward": 0.0580357164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3325893059372902,
      "step": 2093
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1827.7947387695312,
      "epoch": 0.6254947352699575,
      "grad_norm": 1.5202471017837524,
      "kl": 0.31103515625,
      "learning_rate": 1.8479163726503127e-07,
      "loss": 0.1084,
      "reward": 0.4196428805589676,
      "reward_std": 0.18249313905835152,
      "rewards/accuracy_reward": 0.09151785937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3281250149011612,
      "step": 2094
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1721.10498046875,
      "epoch": 0.6257934433574789,
      "grad_norm": 1.0438306331634521,
      "kl": 0.27880859375,
      "learning_rate": 1.8453994280887524e-07,
      "loss": 0.1678,
      "reward": 0.3750000149011612,
      "reward_std": 0.16945624724030495,
      "rewards/accuracy_reward": 0.022321429569274187,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3526785895228386,
      "step": 2095
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1826.57373046875,
      "epoch": 0.6260921514450004,
      "grad_norm": 0.7396443486213684,
      "kl": 0.3291015625,
      "learning_rate": 1.8428831956684576e-07,
      "loss": 0.1198,
      "reward": 0.400111623108387,
      "reward_std": 0.17511969432234764,
      "rewards/accuracy_reward": 0.08035714598372579,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3197544813156128,
      "step": 2096
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1833.0045471191406,
      "epoch": 0.6263908595325218,
      "grad_norm": 1.3237217664718628,
      "kl": 0.35302734375,
      "learning_rate": 1.840367678126841e-07,
      "loss": 0.1208,
      "reward": 0.475446455180645,
      "reward_std": 0.14890214428305626,
      "rewards/accuracy_reward": 0.15848214784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3169643059372902,
      "step": 2097
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1817.6027526855469,
      "epoch": 0.6266895676200434,
      "grad_norm": 0.9102877974510193,
      "kl": 0.330810546875,
      "learning_rate": 1.8378528782005393e-07,
      "loss": 0.1034,
      "reward": 0.3666294813156128,
      "reward_std": 0.16900014504790306,
      "rewards/accuracy_reward": 0.0424107164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3242187723517418,
      "step": 2098
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1729.2433471679688,
      "epoch": 0.6269882757075648,
      "grad_norm": 0.8752507567405701,
      "kl": 0.31982421875,
      "learning_rate": 1.8353387986254092e-07,
      "loss": 0.1534,
      "reward": 0.3984375149011612,
      "reward_std": 0.19472495838999748,
      "rewards/accuracy_reward": 0.055803575087338686,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3426339477300644,
      "step": 2099
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1773.6741943359375,
      "epoch": 0.6272869837950863,
      "grad_norm": 1.2365474700927734,
      "kl": 0.30224609375,
      "learning_rate": 1.8328254421365226e-07,
      "loss": 0.1339,
      "reward": 0.4525669887661934,
      "reward_std": 0.17674745246767998,
      "rewards/accuracy_reward": 0.0959821455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.356584832072258,
      "step": 2100
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1798.4643859863281,
      "epoch": 0.6275856918826077,
      "grad_norm": 1.1902239322662354,
      "kl": 0.33642578125,
      "learning_rate": 1.830312811468165e-07,
      "loss": 0.1414,
      "reward": 0.404017873108387,
      "reward_std": 0.15046238899230957,
      "rewards/accuracy_reward": 0.05580357206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3482143059372902,
      "step": 2101
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1749.0670471191406,
      "epoch": 0.6278843999701292,
      "grad_norm": 0.9776829481124878,
      "kl": 0.30517578125,
      "learning_rate": 1.827800909353831e-07,
      "loss": 0.1478,
      "reward": 0.4570312723517418,
      "reward_std": 0.19961456209421158,
      "rewards/accuracy_reward": 0.0959821492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.361049123108387,
      "step": 2102
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1696.3683776855469,
      "epoch": 0.6281831080576507,
      "grad_norm": 0.7158123850822449,
      "kl": 0.3125,
      "learning_rate": 1.8252897385262258e-07,
      "loss": 0.1189,
      "reward": 0.4123884215950966,
      "reward_std": 0.1254323273897171,
      "rewards/accuracy_reward": 0.05357143026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3588169887661934,
      "step": 2103
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1788.5447082519531,
      "epoch": 0.6284818161451722,
      "grad_norm": 0.7134464979171753,
      "kl": 0.32666015625,
      "learning_rate": 1.8227793017172566e-07,
      "loss": 0.1378,
      "reward": 0.4073660969734192,
      "reward_std": 0.1486248429864645,
      "rewards/accuracy_reward": 0.05357143026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3537946566939354,
      "step": 2104
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1743.8081359863281,
      "epoch": 0.6287805242326936,
      "grad_norm": 0.6041935086250305,
      "kl": 0.30126953125,
      "learning_rate": 1.8202696016580316e-07,
      "loss": 0.1267,
      "reward": 0.5050223469734192,
      "reward_std": 0.18869824893772602,
      "rewards/accuracy_reward": 0.13169643143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.373325914144516,
      "step": 2105
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1734.3572387695312,
      "epoch": 0.6290792323202151,
      "grad_norm": 0.5518330931663513,
      "kl": 0.3251953125,
      "learning_rate": 1.81776064107886e-07,
      "loss": 0.1403,
      "reward": 0.4921875149011612,
      "reward_std": 0.17287111282348633,
      "rewards/accuracy_reward": 0.12946428824216127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3627232387661934,
      "step": 2106
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1681.6139221191406,
      "epoch": 0.6293779404077365,
      "grad_norm": 0.5544466376304626,
      "kl": 0.273681640625,
      "learning_rate": 1.815252422709244e-07,
      "loss": 0.1501,
      "reward": 0.5301339477300644,
      "reward_std": 0.1689399555325508,
      "rewards/accuracy_reward": 0.14285715110599995,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3872767984867096,
      "step": 2107
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1680.368408203125,
      "epoch": 0.6296766484952581,
      "grad_norm": 0.8801741600036621,
      "kl": 0.279541015625,
      "learning_rate": 1.8127449492778796e-07,
      "loss": 0.1538,
      "reward": 0.4994419887661934,
      "reward_std": 0.16287441924214363,
      "rewards/accuracy_reward": 0.1160714328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3833705484867096,
      "step": 2108
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1713.9889221191406,
      "epoch": 0.6299753565827795,
      "grad_norm": 0.5388920307159424,
      "kl": 0.291259765625,
      "learning_rate": 1.8102382235126518e-07,
      "loss": 0.1479,
      "reward": 0.5122768133878708,
      "reward_std": 0.16867785528302193,
      "rewards/accuracy_reward": 0.13392857578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3783482313156128,
      "step": 2109
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1750.76123046875,
      "epoch": 0.6302740646703009,
      "grad_norm": 0.6623998284339905,
      "kl": 0.3154296875,
      "learning_rate": 1.8077322481406332e-07,
      "loss": 0.1265,
      "reward": 0.3995535895228386,
      "reward_std": 0.1532442942261696,
      "rewards/accuracy_reward": 0.026785714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.372767873108387,
      "step": 2110
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1654.4822082519531,
      "epoch": 0.6305727727578224,
      "grad_norm": 0.4257897734642029,
      "kl": 0.26318359375,
      "learning_rate": 1.8052270258880795e-07,
      "loss": 0.1225,
      "reward": 0.4771205633878708,
      "reward_std": 0.17281510308384895,
      "rewards/accuracy_reward": 0.07589286286383867,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4012276902794838,
      "step": 2111
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1777.5804138183594,
      "epoch": 0.6308714808453438,
      "grad_norm": 0.725608229637146,
      "kl": 0.34912109375,
      "learning_rate": 1.8027225594804257e-07,
      "loss": 0.1419,
      "reward": 0.3995535895228386,
      "reward_std": 0.13435711525380611,
      "rewards/accuracy_reward": 0.04017857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3593750074505806,
      "step": 2112
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1617.80810546875,
      "epoch": 0.6311701889328654,
      "grad_norm": 0.4285762906074524,
      "kl": 0.27099609375,
      "learning_rate": 1.8002188516422873e-07,
      "loss": 0.1523,
      "reward": 0.4994419887661934,
      "reward_std": 0.149072527885437,
      "rewards/accuracy_reward": 0.10044643213041127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3989955633878708,
      "step": 2113
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1762.5804443359375,
      "epoch": 0.6314688970203868,
      "grad_norm": 0.6978349685668945,
      "kl": 0.3232421875,
      "learning_rate": 1.7977159050974522e-07,
      "loss": 0.1341,
      "reward": 0.4559151902794838,
      "reward_std": 0.1532731056213379,
      "rewards/accuracy_reward": 0.08482143399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3710937649011612,
      "step": 2114
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1732.7389221191406,
      "epoch": 0.6317676051079083,
      "grad_norm": 0.38722729682922363,
      "kl": 0.31640625,
      "learning_rate": 1.7952137225688816e-07,
      "loss": 0.0963,
      "reward": 0.526785746216774,
      "reward_std": 0.23611834272742271,
      "rewards/accuracy_reward": 0.14732143748551607,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3794643059372902,
      "step": 2115
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1719.0960693359375,
      "epoch": 0.6320663131954297,
      "grad_norm": 0.4838818907737732,
      "kl": 0.3466796875,
      "learning_rate": 1.7927123067787033e-07,
      "loss": 0.1491,
      "reward": 0.5000000298023224,
      "reward_std": 0.18269427865743637,
      "rewards/accuracy_reward": 0.1272321492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.372767873108387,
      "step": 2116
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1824.3259887695312,
      "epoch": 0.6323650212829512,
      "grad_norm": 0.4241102337837219,
      "kl": 0.3349609375,
      "learning_rate": 1.7902116604482143e-07,
      "loss": 0.1143,
      "reward": 0.4587053805589676,
      "reward_std": 0.15574504621326923,
      "rewards/accuracy_reward": 0.08928571734577417,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3694196566939354,
      "step": 2117
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1785.5759582519531,
      "epoch": 0.6326637293704727,
      "grad_norm": 0.4569185972213745,
      "kl": 0.32763671875,
      "learning_rate": 1.7877117862978722e-07,
      "loss": 0.1126,
      "reward": 0.3816964477300644,
      "reward_std": 0.14352201856672764,
      "rewards/accuracy_reward": 0.0133928582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3683035895228386,
      "step": 2118
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1776.9532165527344,
      "epoch": 0.6329624374579942,
      "grad_norm": 0.39548131823539734,
      "kl": 0.32763671875,
      "learning_rate": 1.785212687047294e-07,
      "loss": 0.1134,
      "reward": 0.4056919813156128,
      "reward_std": 0.17617937549948692,
      "rewards/accuracy_reward": 0.03348214435391128,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3722098395228386,
      "step": 2119
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1696.8014221191406,
      "epoch": 0.6332611455455156,
      "grad_norm": 0.34707358479499817,
      "kl": 0.2841796875,
      "learning_rate": 1.7827143654152565e-07,
      "loss": 0.1235,
      "reward": 0.427455373108387,
      "reward_std": 0.13176509737968445,
      "rewards/accuracy_reward": 0.0290178582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375223517418,
      "step": 2120
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1735.0915832519531,
      "epoch": 0.6335598536330371,
      "grad_norm": 0.38854965567588806,
      "kl": 0.266845703125,
      "learning_rate": 1.7802168241196876e-07,
      "loss": 0.1088,
      "reward": 0.4905134066939354,
      "reward_std": 0.1862284429371357,
      "rewards/accuracy_reward": 0.0915178619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3989955559372902,
      "step": 2121
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1694.2925109863281,
      "epoch": 0.6338585617205585,
      "grad_norm": 1371.439208984375,
      "kl": 4.068359375,
      "learning_rate": 1.7777200658776678e-07,
      "loss": 0.2675,
      "reward": 0.4419643059372902,
      "reward_std": 0.16847360506653786,
      "rewards/accuracy_reward": 0.06473214644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3772321566939354,
      "step": 2122
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1685.0179138183594,
      "epoch": 0.6341572698080801,
      "grad_norm": 0.5019664764404297,
      "kl": 0.290283203125,
      "learning_rate": 1.7752240934054247e-07,
      "loss": 0.1359,
      "reward": 0.5440848395228386,
      "reward_std": 0.19859091006219387,
      "rewards/accuracy_reward": 0.1473214365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3967634066939354,
      "step": 2123
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1720.4509582519531,
      "epoch": 0.6344559778956015,
      "grad_norm": 0.6113325953483582,
      "kl": 0.31396484375,
      "learning_rate": 1.772728909418333e-07,
      "loss": 0.128,
      "reward": 0.5106026977300644,
      "reward_std": 0.1420088279992342,
      "rewards/accuracy_reward": 0.1183035783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.392299123108387,
      "step": 2124
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1743.2612609863281,
      "epoch": 0.634754685983123,
      "grad_norm": 0.622826874256134,
      "kl": 0.2978515625,
      "learning_rate": 1.7702345166309075e-07,
      "loss": 0.1133,
      "reward": 0.464285746216774,
      "reward_std": 0.126892926171422,
      "rewards/accuracy_reward": 0.07812500488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3861607387661934,
      "step": 2125
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1780.6786804199219,
      "epoch": 0.6350533940706444,
      "grad_norm": 0.9348044991493225,
      "kl": 0.3115234375,
      "learning_rate": 1.7677409177568025e-07,
      "loss": 0.1073,
      "reward": 0.4492187798023224,
      "reward_std": 0.16591059044003487,
      "rewards/accuracy_reward": 0.066964291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3822544813156128,
      "step": 2126
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1764.5402221679688,
      "epoch": 0.635352102158166,
      "grad_norm": 0.40611472725868225,
      "kl": 0.30126953125,
      "learning_rate": 1.7652481155088106e-07,
      "loss": 0.1094,
      "reward": 0.467075914144516,
      "reward_std": 0.1750699169933796,
      "rewards/accuracy_reward": 0.07812500232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.388950914144516,
      "step": 2127
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1822.5782165527344,
      "epoch": 0.6356508102456874,
      "grad_norm": 1.2373063564300537,
      "kl": 0.35888671875,
      "learning_rate": 1.7627561125988557e-07,
      "loss": 0.0972,
      "reward": 0.513950914144516,
      "reward_std": 0.16292742639780045,
      "rewards/accuracy_reward": 0.1361607185099274,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3777901902794838,
      "step": 2128
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1713.8438415527344,
      "epoch": 0.6359495183332089,
      "grad_norm": 0.4926605224609375,
      "kl": 0.265869140625,
      "learning_rate": 1.7602649117379925e-07,
      "loss": 0.1156,
      "reward": 0.5033482387661934,
      "reward_std": 0.13990241661667824,
      "rewards/accuracy_reward": 0.098214291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4051339402794838,
      "step": 2129
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1745.6139221191406,
      "epoch": 0.6362482264207303,
      "grad_norm": 0.4288490414619446,
      "kl": 0.292236328125,
      "learning_rate": 1.7577745156364033e-07,
      "loss": 0.0938,
      "reward": 0.4302455484867096,
      "reward_std": 0.178131565451622,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3989955559372902,
      "step": 2130
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1725.0849304199219,
      "epoch": 0.6365469345082518,
      "grad_norm": 0.41701826453208923,
      "kl": 0.2763671875,
      "learning_rate": 1.7552849270033954e-07,
      "loss": 0.0956,
      "reward": 0.4799107238650322,
      "reward_std": 0.16452564299106598,
      "rewards/accuracy_reward": 0.08258929033763707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3973214477300644,
      "step": 2131
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1793.4420471191406,
      "epoch": 0.6368456425957733,
      "grad_norm": 0.46655797958374023,
      "kl": 0.296630859375,
      "learning_rate": 1.752796148547397e-07,
      "loss": 0.0929,
      "reward": 0.5563616454601288,
      "reward_std": 0.2164175733923912,
      "rewards/accuracy_reward": 0.16071429289877415,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3956473469734192,
      "step": 2132
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1730.8773193359375,
      "epoch": 0.6371443506832948,
      "grad_norm": 0.4649956226348877,
      "kl": 0.30029296875,
      "learning_rate": 1.7503081829759545e-07,
      "loss": 0.0957,
      "reward": 0.5290178880095482,
      "reward_std": 0.16916153579950333,
      "rewards/accuracy_reward": 0.14062500116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.388392873108387,
      "step": 2133
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1745.0022888183594,
      "epoch": 0.6374430587708162,
      "grad_norm": 8.007674217224121,
      "kl": 0.343505859375,
      "learning_rate": 1.7478210329957315e-07,
      "loss": 0.1052,
      "reward": 0.5200893059372902,
      "reward_std": 0.17915472760796547,
      "rewards/accuracy_reward": 0.11607143469154835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4040178656578064,
      "step": 2134
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1755.0625915527344,
      "epoch": 0.6377417668583377,
      "grad_norm": 0.5245476961135864,
      "kl": 0.2451171875,
      "learning_rate": 1.7453347013125035e-07,
      "loss": 0.0868,
      "reward": 0.5273437723517418,
      "reward_std": 0.20810982212424278,
      "rewards/accuracy_reward": 0.12500000465661287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4023437723517418,
      "step": 2135
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1714.6697082519531,
      "epoch": 0.6380404749458591,
      "grad_norm": 0.5301141142845154,
      "kl": 0.22265625,
      "learning_rate": 1.7428491906311555e-07,
      "loss": 0.08,
      "reward": 0.554129496216774,
      "reward_std": 0.18122979626059532,
      "rewards/accuracy_reward": 0.1272321492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4268973395228386,
      "step": 2136
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1786.2969360351562,
      "epoch": 0.6383391830333807,
      "grad_norm": 0.3667783737182617,
      "kl": 0.290771484375,
      "learning_rate": 1.7403645036556785e-07,
      "loss": 0.0903,
      "reward": 0.491071455180645,
      "reward_std": 0.15836292132735252,
      "rewards/accuracy_reward": 0.09375000349245965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.397321455180645,
      "step": 2137
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1705.6540832519531,
      "epoch": 0.6386378911209021,
      "grad_norm": 0.3673921823501587,
      "kl": 0.260498046875,
      "learning_rate": 1.73788064308917e-07,
      "loss": 0.0957,
      "reward": 0.5613839700818062,
      "reward_std": 0.2142045982182026,
      "rewards/accuracy_reward": 0.15848214644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4029018133878708,
      "step": 2138
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1711.5067749023438,
      "epoch": 0.6389365992084236,
      "grad_norm": 0.45877182483673096,
      "kl": 0.23193359375,
      "learning_rate": 1.7353976116338258e-07,
      "loss": 0.105,
      "reward": 0.4771205559372902,
      "reward_std": 0.1556875165551901,
      "rewards/accuracy_reward": 0.05803571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4190848395228386,
      "step": 2139
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1701.8482666015625,
      "epoch": 0.639235307295945,
      "grad_norm": 0.4578523635864258,
      "kl": 0.236328125,
      "learning_rate": 1.7329154119909413e-07,
      "loss": 0.0999,
      "reward": 0.4921875223517418,
      "reward_std": 0.11706082336604595,
      "rewards/accuracy_reward": 0.07589286053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4162946566939354,
      "step": 2140
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1719.2478332519531,
      "epoch": 0.6395340153834665,
      "grad_norm": 0.453637957572937,
      "kl": 0.264892578125,
      "learning_rate": 1.7304340468609058e-07,
      "loss": 0.0868,
      "reward": 0.521205373108387,
      "reward_std": 0.1501566246151924,
      "rewards/accuracy_reward": 0.11383929220028222,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4073660895228386,
      "step": 2141
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1718.2813110351562,
      "epoch": 0.639832723470988,
      "grad_norm": 0.4856603443622589,
      "kl": 0.242919921875,
      "learning_rate": 1.727953518943202e-07,
      "loss": 0.1056,
      "reward": 0.5814732387661934,
      "reward_std": 0.1565337087959051,
      "rewards/accuracy_reward": 0.16741072502918541,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625223517418,
      "step": 2142
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1794.7656860351562,
      "epoch": 0.6401314315585095,
      "grad_norm": 0.9509595036506653,
      "kl": 0.264892578125,
      "learning_rate": 1.7254738309364008e-07,
      "loss": 0.0839,
      "reward": 0.444196455180645,
      "reward_std": 0.17128043435513973,
      "rewards/accuracy_reward": 0.03794643096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500223517418,
      "step": 2143
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1666.4509887695312,
      "epoch": 0.6404301396460309,
      "grad_norm": 0.4770304560661316,
      "kl": 0.206298828125,
      "learning_rate": 1.7229949855381598e-07,
      "loss": 0.1051,
      "reward": 0.5368303954601288,
      "reward_std": 0.12840250879526138,
      "rewards/accuracy_reward": 0.1026785746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341518059372902,
      "step": 2144
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1731.5692749023438,
      "epoch": 0.6407288477335524,
      "grad_norm": 0.4486764669418335,
      "kl": 0.223876953125,
      "learning_rate": 1.72051698544522e-07,
      "loss": 0.0949,
      "reward": 0.4497768133878708,
      "reward_std": 0.15864156931638718,
      "rewards/accuracy_reward": 0.03125000046566129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268059372902,
      "step": 2145
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1743.7433776855469,
      "epoch": 0.6410275558210738,
      "grad_norm": 0.41825202107429504,
      "kl": 0.213134765625,
      "learning_rate": 1.7180398333534025e-07,
      "loss": 0.0813,
      "reward": 0.466517873108387,
      "reward_std": 0.11457138508558273,
      "rewards/accuracy_reward": 0.0401785746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263393059372902,
      "step": 2146
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1713.2991638183594,
      "epoch": 0.6413262639085954,
      "grad_norm": 0.31682437658309937,
      "kl": 0.243408203125,
      "learning_rate": 1.715563531957606e-07,
      "loss": 0.068,
      "reward": 0.5245535969734192,
      "reward_std": 0.14568622782826424,
      "rewards/accuracy_reward": 0.1026785746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 2147
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1757.3192749023438,
      "epoch": 0.6416249719961168,
      "grad_norm": 1.4602437019348145,
      "kl": 0.24853515625,
      "learning_rate": 1.7130880839518032e-07,
      "loss": 0.0658,
      "reward": 0.4871652126312256,
      "reward_std": 0.14345146343111992,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4157366305589676,
      "step": 2148
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1757.8550109863281,
      "epoch": 0.6419236800836383,
      "grad_norm": 0.45338594913482666,
      "kl": 0.2587890625,
      "learning_rate": 1.7106134920290398e-07,
      "loss": 0.0911,
      "reward": 0.5228794813156128,
      "reward_std": 0.19778629764914513,
      "rewards/accuracy_reward": 0.1205357201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4023437649011612,
      "step": 2149
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1822.2947387695312,
      "epoch": 0.6422223881711597,
      "grad_norm": 0.6706060767173767,
      "kl": 0.259033203125,
      "learning_rate": 1.708139758881429e-07,
      "loss": 0.0899,
      "reward": 0.4854910895228386,
      "reward_std": 0.15445445477962494,
      "rewards/accuracy_reward": 0.09151786006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3939732238650322,
      "step": 2150
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1787.3259887695312,
      "epoch": 0.6425210962586813,
      "grad_norm": 0.483112633228302,
      "kl": 0.28759765625,
      "learning_rate": 1.7056668872001496e-07,
      "loss": 0.0995,
      "reward": 0.423549123108387,
      "reward_std": 0.17098619788885117,
      "rewards/accuracy_reward": 0.03125000116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.392299123108387,
      "step": 2151
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1807.5469665527344,
      "epoch": 0.6428198043462027,
      "grad_norm": 0.4360291659832001,
      "kl": 0.256591796875,
      "learning_rate": 1.7031948796754443e-07,
      "loss": 0.0561,
      "reward": 0.4960937723517418,
      "reward_std": 0.17814015597105026,
      "rewards/accuracy_reward": 0.08705357508733869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4090401977300644,
      "step": 2152
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1734.9933776855469,
      "epoch": 0.6431185124337241,
      "grad_norm": 0.5238285660743713,
      "kl": 0.2197265625,
      "learning_rate": 1.7007237389966145e-07,
      "loss": 0.0648,
      "reward": 0.4977678880095482,
      "reward_std": 0.134385802783072,
      "rewards/accuracy_reward": 0.07589286053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750149011612,
      "step": 2153
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1830.7210693359375,
      "epoch": 0.6434172205212456,
      "grad_norm": 0.758245587348938,
      "kl": 0.29296875,
      "learning_rate": 1.6982534678520188e-07,
      "loss": 0.0838,
      "reward": 0.450334832072258,
      "reward_std": 0.1505792960524559,
      "rewards/accuracy_reward": 0.058035716181620955,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.392299123108387,
      "step": 2154
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1716.5915832519531,
      "epoch": 0.643715928608767,
      "grad_norm": 0.620316743850708,
      "kl": 0.20068359375,
      "learning_rate": 1.6957840689290706e-07,
      "loss": 0.0693,
      "reward": 0.538504496216774,
      "reward_std": 0.19045366533100605,
      "rewards/accuracy_reward": 0.10937500325962901,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294813156128,
      "step": 2155
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1808.3081359863281,
      "epoch": 0.6440146366962886,
      "grad_norm": 1.0322672128677368,
      "kl": 0.240478515625,
      "learning_rate": 1.693315544914234e-07,
      "loss": 0.0666,
      "reward": 0.5440848618745804,
      "reward_std": 0.19268004596233368,
      "rewards/accuracy_reward": 0.1361607201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.407924123108387,
      "step": 2156
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1758.8482666015625,
      "epoch": 0.64431334478381,
      "grad_norm": 1.4129730463027954,
      "kl": 0.257080078125,
      "learning_rate": 1.6908478984930208e-07,
      "loss": 0.0938,
      "reward": 0.4654018059372902,
      "reward_std": 0.15272756479680538,
      "rewards/accuracy_reward": 0.060267859138548374,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4051339402794838,
      "step": 2157
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1771.3505249023438,
      "epoch": 0.6446120528713315,
      "grad_norm": 2.950673818588257,
      "kl": 0.29931640625,
      "learning_rate": 1.6883811323499887e-07,
      "loss": 0.0948,
      "reward": 0.4486607387661934,
      "reward_std": 0.14786637760698795,
      "rewards/accuracy_reward": 0.05580357392318547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.392857164144516,
      "step": 2158
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1727.5603332519531,
      "epoch": 0.6449107609588529,
      "grad_norm": 3.739187479019165,
      "kl": 0.223876953125,
      "learning_rate": 1.6859152491687372e-07,
      "loss": 0.088,
      "reward": 0.5541294887661934,
      "reward_std": 0.1490048784762621,
      "rewards/accuracy_reward": 0.13169643888249993,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330559372902,
      "step": 2159
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1792.5134582519531,
      "epoch": 0.6452094690463744,
      "grad_norm": 4.240115642547607,
      "kl": 0.2724609375,
      "learning_rate": 1.683450251631906e-07,
      "loss": 0.0861,
      "reward": 0.4927455633878708,
      "reward_std": 0.16888438537716866,
      "rewards/accuracy_reward": 0.06696428824216127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812723517418,
      "step": 2160
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.6451721191406,
      "epoch": 0.6455081771338959,
      "grad_norm": 5.114086151123047,
      "kl": 0.35498046875,
      "learning_rate": 1.6809861424211707e-07,
      "loss": 0.0774,
      "reward": 0.5083705633878708,
      "reward_std": 0.12216413952410221,
      "rewards/accuracy_reward": 0.10267857951112092,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4056919887661934,
      "step": 2161
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1795.5089721679688,
      "epoch": 0.6458068852214174,
      "grad_norm": 7.29588508605957,
      "kl": 0.376953125,
      "learning_rate": 1.67852292421724e-07,
      "loss": 0.0681,
      "reward": 0.4960937649011612,
      "reward_std": 0.127341253682971,
      "rewards/accuracy_reward": 0.0848214328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.411272332072258,
      "step": 2162
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1829.1808776855469,
      "epoch": 0.6461055933089388,
      "grad_norm": 6.904472351074219,
      "kl": 0.4560546875,
      "learning_rate": 1.6760605996998545e-07,
      "loss": 0.0937,
      "reward": 0.431919664144516,
      "reward_std": 0.14569793827831745,
      "rewards/accuracy_reward": 0.04687500116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3850446566939354,
      "step": 2163
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1758.8282165527344,
      "epoch": 0.6464043013964603,
      "grad_norm": 6.7539591789245605,
      "kl": 0.41064453125,
      "learning_rate": 1.673599171547782e-07,
      "loss": 0.0877,
      "reward": 0.4620535969734192,
      "reward_std": 0.13200645707547665,
      "rewards/accuracy_reward": 0.051339288242161274,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4107143059372902,
      "step": 2164
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1789.8505249023438,
      "epoch": 0.6467030094839817,
      "grad_norm": 9.889727592468262,
      "kl": 0.4228515625,
      "learning_rate": 1.6711386424388144e-07,
      "loss": 0.0708,
      "reward": 0.4542410895228386,
      "reward_std": 0.15225978195667267,
      "rewards/accuracy_reward": 0.04241071455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.411830373108387,
      "step": 2165
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1794.8906860351562,
      "epoch": 0.6470017175715033,
      "grad_norm": 7.766774654388428,
      "kl": 0.609375,
      "learning_rate": 1.668679015049767e-07,
      "loss": 0.0808,
      "reward": 0.4843750223517418,
      "reward_std": 0.17038054764270782,
      "rewards/accuracy_reward": 0.07589285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.408482164144516,
      "step": 2166
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1784.8036499023438,
      "epoch": 0.6473004256590247,
      "grad_norm": 10.321952819824219,
      "kl": 0.55810546875,
      "learning_rate": 1.6662202920564734e-07,
      "loss": 0.089,
      "reward": 0.6183036118745804,
      "reward_std": 0.17240991070866585,
      "rewards/accuracy_reward": 0.19419643399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.424107164144516,
      "step": 2167
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1759.4866943359375,
      "epoch": 0.6475991337465462,
      "grad_norm": 11.509559631347656,
      "kl": 0.677734375,
      "learning_rate": 1.6637624761337825e-07,
      "loss": 0.1015,
      "reward": 0.5228794887661934,
      "reward_std": 0.1483767181634903,
      "rewards/accuracy_reward": 0.09151786426082253,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.431361623108387,
      "step": 2168
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1733.0536499023438,
      "epoch": 0.6478978418340676,
      "grad_norm": 14.783576011657715,
      "kl": 0.896484375,
      "learning_rate": 1.661305569955557e-07,
      "loss": 0.1343,
      "reward": 0.467075914144516,
      "reward_std": 0.13916569203138351,
      "rewards/accuracy_reward": 0.058035718742758036,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4090401977300644,
      "step": 2169
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1822.3438415527344,
      "epoch": 0.6481965499215891,
      "grad_norm": 18.20441246032715,
      "kl": 0.884765625,
      "learning_rate": 1.658849576194671e-07,
      "loss": 0.0967,
      "reward": 0.4670759215950966,
      "reward_std": 0.1291603073477745,
      "rewards/accuracy_reward": 0.06026786006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4068080484867096,
      "step": 2170
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1747.0715026855469,
      "epoch": 0.6484952580091106,
      "grad_norm": 16.15131187438965,
      "kl": 0.67724609375,
      "learning_rate": 1.6563944975230047e-07,
      "loss": 0.0829,
      "reward": 0.5368303805589676,
      "reward_std": 0.15096243284642696,
      "rewards/accuracy_reward": 0.1049107201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.431919664144516,
      "step": 2171
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1802.1831359863281,
      "epoch": 0.6487939660966321,
      "grad_norm": 13.471420288085938,
      "kl": 1.05859375,
      "learning_rate": 1.653940336611442e-07,
      "loss": 0.1081,
      "reward": 0.4486607313156128,
      "reward_std": 0.11494387127459049,
      "rewards/accuracy_reward": 0.0379464291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4107143059372902,
      "step": 2172
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1733.8995971679688,
      "epoch": 0.6490926741841535,
      "grad_norm": 16.804391860961914,
      "kl": 0.904296875,
      "learning_rate": 1.651487096129871e-07,
      "loss": 0.1198,
      "reward": 0.4575893059372902,
      "reward_std": 0.14100381731987,
      "rewards/accuracy_reward": 0.03571428661234677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 2173
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1820.7500915527344,
      "epoch": 0.649391382271675,
      "grad_norm": 220.9008026123047,
      "kl": 2.4931640625,
      "learning_rate": 1.6490347787471765e-07,
      "loss": 0.1621,
      "reward": 0.4313616305589676,
      "reward_std": 0.1305142119526863,
      "rewards/accuracy_reward": 0.0379464291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3934151977300644,
      "step": 2174
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1836.2077026367188,
      "epoch": 0.6496900903591964,
      "grad_norm": 152.51202392578125,
      "kl": 1.8408203125,
      "learning_rate": 1.646583387131239e-07,
      "loss": 0.1248,
      "reward": 0.5016741305589676,
      "reward_std": 0.19546056166291237,
      "rewards/accuracy_reward": 0.09151786286383867,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4101562649011612,
      "step": 2175
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1843.8817443847656,
      "epoch": 0.649988798446718,
      "grad_norm": 54.85565948486328,
      "kl": 0.6376953125,
      "learning_rate": 1.6441329239489314e-07,
      "loss": 0.0802,
      "reward": 0.4654018133878708,
      "reward_std": 0.173141997307539,
      "rewards/accuracy_reward": 0.05357143026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4118303805589676,
      "step": 2176
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1842.1898193359375,
      "epoch": 0.6502875065342394,
      "grad_norm": 44.84663772583008,
      "kl": 0.6259765625,
      "learning_rate": 1.64168339186612e-07,
      "loss": 0.0892,
      "reward": 0.4464285895228386,
      "reward_std": 0.1605691984295845,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.415178582072258,
      "step": 2177
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1856.4398193359375,
      "epoch": 0.6505862146217609,
      "grad_norm": 45.75059127807617,
      "kl": 0.830078125,
      "learning_rate": 1.6392347935476537e-07,
      "loss": 0.082,
      "reward": 0.4793526902794838,
      "reward_std": 0.16352969780564308,
      "rewards/accuracy_reward": 0.0758928619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4034598395228386,
      "step": 2178
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1811.52685546875,
      "epoch": 0.6508849227092823,
      "grad_norm": 37.970272064208984,
      "kl": 0.955078125,
      "learning_rate": 1.6367871316573672e-07,
      "loss": 0.0993,
      "reward": 0.5256696566939354,
      "reward_std": 0.192394794896245,
      "rewards/accuracy_reward": 0.10491071874275804,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4207589477300644,
      "step": 2179
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1829.3817749023438,
      "epoch": 0.6511836307968039,
      "grad_norm": 27.142709732055664,
      "kl": 1.58203125,
      "learning_rate": 1.6343404088580786e-07,
      "loss": 0.1375,
      "reward": 0.4330357238650322,
      "reward_std": 0.13291746377944946,
      "rewards/accuracy_reward": 0.044642859138548374,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3883928656578064,
      "step": 2180
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1915.9353332519531,
      "epoch": 0.6514823388843253,
      "grad_norm": 94.56024169921875,
      "kl": 3.009765625,
      "learning_rate": 1.631894627811581e-07,
      "loss": 0.1664,
      "reward": 0.4564732313156128,
      "reward_std": 0.15921980701386929,
      "rewards/accuracy_reward": 0.06250000232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3939732313156128,
      "step": 2181
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1791.46435546875,
      "epoch": 0.6517810469718468,
      "grad_norm": 18.58778953552246,
      "kl": 0.962890625,
      "learning_rate": 1.629449791178645e-07,
      "loss": 0.0827,
      "reward": 0.5535714626312256,
      "reward_std": 0.16586069762706757,
      "rewards/accuracy_reward": 0.1272321459837258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263393059372902,
      "step": 2182
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1784.7165832519531,
      "epoch": 0.6520797550593682,
      "grad_norm": 16.2475528717041,
      "kl": 1.94140625,
      "learning_rate": 1.6270059016190126e-07,
      "loss": 0.1528,
      "reward": 0.4771205633878708,
      "reward_std": 0.13889878802001476,
      "rewards/accuracy_reward": 0.0736607164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4034598395228386,
      "step": 2183
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1828.3996276855469,
      "epoch": 0.6523784631468897,
      "grad_norm": 432.28411865234375,
      "kl": 6.80078125,
      "learning_rate": 1.6245629617913968e-07,
      "loss": 0.2558,
      "reward": 0.4760044887661934,
      "reward_std": 0.16295186430215836,
      "rewards/accuracy_reward": 0.06250000232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4135044887661934,
      "step": 2184
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1882.7835693359375,
      "epoch": 0.6526771712344112,
      "grad_norm": 35.409759521484375,
      "kl": 3.23046875,
      "learning_rate": 1.6221209743534773e-07,
      "loss": 0.1713,
      "reward": 0.4380580633878708,
      "reward_std": 0.14730950631201267,
      "rewards/accuracy_reward": 0.055803573690354824,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3822544887661934,
      "step": 2185
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1782.0648193359375,
      "epoch": 0.6529758793219327,
      "grad_norm": 18.855337142944336,
      "kl": 1.251953125,
      "learning_rate": 1.619679941961895e-07,
      "loss": 0.1013,
      "reward": 0.5217634066939354,
      "reward_std": 0.13952495530247688,
      "rewards/accuracy_reward": 0.09151785937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455484867096,
      "step": 2186
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1852.3393859863281,
      "epoch": 0.6532745874094541,
      "grad_norm": 13.165804862976074,
      "kl": 1.74609375,
      "learning_rate": 1.6172398672722562e-07,
      "loss": 0.1097,
      "reward": 0.4966518208384514,
      "reward_std": 0.14323166385293007,
      "rewards/accuracy_reward": 0.09151785937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4051339477300644,
      "step": 2187
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1771.9777526855469,
      "epoch": 0.6535732954969756,
      "grad_norm": 31.907453536987305,
      "kl": 1.2939453125,
      "learning_rate": 1.614800752939122e-07,
      "loss": 0.1357,
      "reward": 0.4960937798023224,
      "reward_std": 0.16885284520685673,
      "rewards/accuracy_reward": 0.0848214328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4112723395228386,
      "step": 2188
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1804.4063415527344,
      "epoch": 0.653872003584497,
      "grad_norm": 30.073810577392578,
      "kl": 2.70703125,
      "learning_rate": 1.6123626016160096e-07,
      "loss": 0.1913,
      "reward": 0.4854910969734192,
      "reward_std": 0.16061488725245,
      "rewards/accuracy_reward": 0.06696428847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268059372902,
      "step": 2189
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1810.6250610351562,
      "epoch": 0.6541707116720186,
      "grad_norm": 14.673182487487793,
      "kl": 2.46875,
      "learning_rate": 1.6099254159553883e-07,
      "loss": 0.1488,
      "reward": 0.475446455180645,
      "reward_std": 0.18951418064534664,
      "rewards/accuracy_reward": 0.06026786006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4151785895228386,
      "step": 2190
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1879.9822387695312,
      "epoch": 0.65446941975954,
      "grad_norm": 21.033632278442383,
      "kl": 1.779296875,
      "learning_rate": 1.6074891986086783e-07,
      "loss": 0.1174,
      "reward": 0.4414062649011612,
      "reward_std": 0.15120753087103367,
      "rewards/accuracy_reward": 0.02901785750873387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4123884066939354,
      "step": 2191
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1866.6362609863281,
      "epoch": 0.6547681278470615,
      "grad_norm": 19.368297576904297,
      "kl": 1.576171875,
      "learning_rate": 1.6050539522262448e-07,
      "loss": 0.1043,
      "reward": 0.4737723469734192,
      "reward_std": 0.1611400619149208,
      "rewards/accuracy_reward": 0.06696428847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4068080559372902,
      "step": 2192
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1802.8862609863281,
      "epoch": 0.6550668359345829,
      "grad_norm": 16.645069122314453,
      "kl": 2.197265625,
      "learning_rate": 1.6026196794573965e-07,
      "loss": 0.1657,
      "reward": 0.513950914144516,
      "reward_std": 0.18210257589817047,
      "rewards/accuracy_reward": 0.10714286426082253,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4068080484867096,
      "step": 2193
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1836.3996276855469,
      "epoch": 0.6553655440221045,
      "grad_norm": 56.880218505859375,
      "kl": 3.7578125,
      "learning_rate": 1.6001863829503847e-07,
      "loss": 0.2208,
      "reward": 0.6110491305589676,
      "reward_std": 0.18799839727580547,
      "rewards/accuracy_reward": 0.2053571529686451,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4056919813156128,
      "step": 2194
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1817.149658203125,
      "epoch": 0.6556642521096259,
      "grad_norm": 38.62685012817383,
      "kl": 3.294921875,
      "learning_rate": 1.5977540653523975e-07,
      "loss": 0.1887,
      "reward": 0.502232164144516,
      "reward_std": 0.19252017885446548,
      "rewards/accuracy_reward": 0.09151785937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4107142984867096,
      "step": 2195
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1862.2500610351562,
      "epoch": 0.6559629601971473,
      "grad_norm": 21.49437141418457,
      "kl": 1.771484375,
      "learning_rate": 1.5953227293095568e-07,
      "loss": 0.128,
      "reward": 0.4581473395228386,
      "reward_std": 0.12906410917639732,
      "rewards/accuracy_reward": 0.04910714481957257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.409040205180645,
      "step": 2196
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1861.0625915527344,
      "epoch": 0.6562616682846688,
      "grad_norm": 22.62534523010254,
      "kl": 1.90234375,
      "learning_rate": 1.592892377466919e-07,
      "loss": 0.1182,
      "reward": 0.487165205180645,
      "reward_std": 0.16745640523731709,
      "rewards/accuracy_reward": 0.0781250037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4090401977300644,
      "step": 2197
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1847.6563110351562,
      "epoch": 0.6565603763721902,
      "grad_norm": 10.633485794067383,
      "kl": 2.142578125,
      "learning_rate": 1.5904630124684685e-07,
      "loss": 0.1384,
      "reward": 0.474330373108387,
      "reward_std": 0.134651405736804,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.411830373108387,
      "step": 2198
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1810.618408203125,
      "epoch": 0.6568590844597118,
      "grad_norm": 45.121280670166016,
      "kl": 3.83203125,
      "learning_rate": 1.5880346369571163e-07,
      "loss": 0.2187,
      "reward": 0.492187537252903,
      "reward_std": 0.23960452899336815,
      "rewards/accuracy_reward": 0.07589286006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.416294664144516,
      "step": 2199
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1823.0379943847656,
      "epoch": 0.6571577925472332,
      "grad_norm": 41.59630584716797,
      "kl": 3.8046875,
      "learning_rate": 1.585607253574696e-07,
      "loss": 0.2071,
      "reward": 0.4291294813156128,
      "reward_std": 0.1582975797355175,
      "rewards/accuracy_reward": 0.031250001629814506,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3978794813156128,
      "step": 2200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1858.71435546875,
      "epoch": 0.6574565006347547,
      "grad_norm": 50.36886978149414,
      "kl": 1.4921875,
      "learning_rate": 1.583180864961965e-07,
      "loss": 0.107,
      "reward": 0.4787946566939354,
      "reward_std": 0.11614236608147621,
      "rewards/accuracy_reward": 0.0736607164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4051339477300644,
      "step": 2201
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1714.7344665527344,
      "epoch": 0.6577552087222761,
      "grad_norm": 30.95758819580078,
      "kl": 1.0771484375,
      "learning_rate": 1.5807554737585943e-07,
      "loss": 0.0973,
      "reward": 0.5145089402794838,
      "reward_std": 0.1429465338587761,
      "rewards/accuracy_reward": 0.09375000558793545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4207589402794838,
      "step": 2202
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1822.4241943359375,
      "epoch": 0.6580539168097976,
      "grad_norm": 49.198951721191406,
      "kl": 1.419921875,
      "learning_rate": 1.578331082603173e-07,
      "loss": 0.1048,
      "reward": 0.4787946715950966,
      "reward_std": 0.16841862350702286,
      "rewards/accuracy_reward": 0.05357143236324191,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232313156128,
      "step": 2203
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1734.1161804199219,
      "epoch": 0.658352624897319,
      "grad_norm": 16.387285232543945,
      "kl": 2.59375,
      "learning_rate": 1.5759076941331995e-07,
      "loss": 0.1882,
      "reward": 0.5318080559372902,
      "reward_std": 0.14454777166247368,
      "rewards/accuracy_reward": 0.10714286006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4246651977300644,
      "step": 2204
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1853.4443054199219,
      "epoch": 0.6586513329848406,
      "grad_norm": 44.269378662109375,
      "kl": 4.7109375,
      "learning_rate": 1.5734853109850848e-07,
      "loss": 0.2651,
      "reward": 0.4190848469734192,
      "reward_std": 0.1389799825847149,
      "rewards/accuracy_reward": 0.0200892873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3989955559372902,
      "step": 2205
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1844.524658203125,
      "epoch": 0.658950041072362,
      "grad_norm": 14.374629974365234,
      "kl": 2.439453125,
      "learning_rate": 1.5710639357941437e-07,
      "loss": 0.1494,
      "reward": 0.5055803805589676,
      "reward_std": 0.15176177397370338,
      "rewards/accuracy_reward": 0.08258928940631449,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 2206
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1849.821533203125,
      "epoch": 0.6592487491598835,
      "grad_norm": 17.543710708618164,
      "kl": 2.673828125,
      "learning_rate": 1.5686435711945946e-07,
      "loss": 0.1578,
      "reward": 0.452008955180645,
      "reward_std": 0.16376896388828754,
      "rewards/accuracy_reward": 0.0424107164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4095982387661934,
      "step": 2207
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1736.1585693359375,
      "epoch": 0.6595474572474049,
      "grad_norm": 14.489166259765625,
      "kl": 1.99609375,
      "learning_rate": 1.5662242198195574e-07,
      "loss": 0.1368,
      "reward": 0.494977705180645,
      "reward_std": 0.1917777769267559,
      "rewards/accuracy_reward": 0.07589286286383867,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4190848395228386,
      "step": 2208
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1787.99560546875,
      "epoch": 0.6598461653349265,
      "grad_norm": 18.134517669677734,
      "kl": 2.3671875,
      "learning_rate": 1.5638058843010492e-07,
      "loss": 0.1563,
      "reward": 0.5089285969734192,
      "reward_std": 0.13993413373827934,
      "rewards/accuracy_reward": 0.10044643469154835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.408482164144516,
      "step": 2209
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1825.7166137695312,
      "epoch": 0.6601448734224479,
      "grad_norm": 16.48717498779297,
      "kl": 2.630859375,
      "learning_rate": 1.5613885672699821e-07,
      "loss": 0.1683,
      "reward": 0.550781287252903,
      "reward_std": 0.1636018007993698,
      "rewards/accuracy_reward": 0.13392857578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4168526977300644,
      "step": 2210
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1848.7880554199219,
      "epoch": 0.6604435815099694,
      "grad_norm": 10.177712440490723,
      "kl": 2.810546875,
      "learning_rate": 1.5589722713561592e-07,
      "loss": 0.1602,
      "reward": 0.4815848469734192,
      "reward_std": 0.15495957620441914,
      "rewards/accuracy_reward": 0.06696428847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205484867096,
      "step": 2211
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1819.0893859863281,
      "epoch": 0.6607422895974908,
      "grad_norm": 84.11796569824219,
      "kl": 3.57421875,
      "learning_rate": 1.5565569991882753e-07,
      "loss": 0.1919,
      "reward": 0.5379464477300644,
      "reward_std": 0.1255164686590433,
      "rewards/accuracy_reward": 0.1205357201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4174107313156128,
      "step": 2212
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1733.2076416015625,
      "epoch": 0.6610409976850123,
      "grad_norm": 13.702842712402344,
      "kl": 2.60546875,
      "learning_rate": 1.554142753393909e-07,
      "loss": 0.1806,
      "reward": 0.6021205559372902,
      "reward_std": 0.14941392093896866,
      "rewards/accuracy_reward": 0.19196429220028222,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4101562723517418,
      "step": 2213
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1824.5849304199219,
      "epoch": 0.6613397057725338,
      "grad_norm": 48.1385383605957,
      "kl": 2.044921875,
      "learning_rate": 1.5517295365995225e-07,
      "loss": 0.129,
      "reward": 0.5212053805589676,
      "reward_std": 0.163785882294178,
      "rewards/accuracy_reward": 0.10491071827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.416294664144516,
      "step": 2214
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1876.3326721191406,
      "epoch": 0.6616384138600553,
      "grad_norm": 13.208414077758789,
      "kl": 3.453125,
      "learning_rate": 1.54931735143046e-07,
      "loss": 0.1959,
      "reward": 0.5708705559372902,
      "reward_std": 0.16594872437417507,
      "rewards/accuracy_reward": 0.17633929220028222,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3945312649011612,
      "step": 2215
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1877.8014221191406,
      "epoch": 0.6619371219475767,
      "grad_norm": 12.01906681060791,
      "kl": 3.1953125,
      "learning_rate": 1.5469062005109423e-07,
      "loss": 0.1928,
      "reward": 0.561941996216774,
      "reward_std": 0.16843882016837597,
      "rewards/accuracy_reward": 0.1562500111758709,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4056919813156128,
      "step": 2216
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1776.3393859863281,
      "epoch": 0.6622358300350982,
      "grad_norm": 34.699825286865234,
      "kl": 1.857421875,
      "learning_rate": 1.5444960864640651e-07,
      "loss": 0.1194,
      "reward": 0.4799107313156128,
      "reward_std": 0.19737237319350243,
      "rewards/accuracy_reward": 0.05133928917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 2217
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1822.2813415527344,
      "epoch": 0.6625345381226196,
      "grad_norm": 13.85798168182373,
      "kl": 2.771484375,
      "learning_rate": 1.542087011911796e-07,
      "loss": 0.1774,
      "reward": 0.5044643059372902,
      "reward_std": 0.15126293525099754,
      "rewards/accuracy_reward": 0.09598214877769351,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4084821566939354,
      "step": 2218
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1830.1987609863281,
      "epoch": 0.6628332462101412,
      "grad_norm": 31.499422073364258,
      "kl": 3.76171875,
      "learning_rate": 1.539678979474973e-07,
      "loss": 0.2186,
      "reward": 0.5016741454601288,
      "reward_std": 0.17718639224767685,
      "rewards/accuracy_reward": 0.0870535746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205633878708,
      "step": 2219
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1852.2590026855469,
      "epoch": 0.6631319542976626,
      "grad_norm": 16.087862014770508,
      "kl": 2.578125,
      "learning_rate": 1.5372719917732986e-07,
      "loss": 0.1572,
      "reward": 0.5016741305589676,
      "reward_std": 0.16891697607934475,
      "rewards/accuracy_reward": 0.10044643469154835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4012276977300644,
      "step": 2220
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1833.6719665527344,
      "epoch": 0.6634306623851841,
      "grad_norm": 37.167659759521484,
      "kl": 2.279296875,
      "learning_rate": 1.5348660514253397e-07,
      "loss": 0.1585,
      "reward": 0.548549123108387,
      "reward_std": 0.15856679901480675,
      "rewards/accuracy_reward": 0.13616072107106447,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.412388414144516,
      "step": 2221
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1885.4643859863281,
      "epoch": 0.6637293704727055,
      "grad_norm": 53.50740051269531,
      "kl": 4.953125,
      "learning_rate": 1.5324611610485227e-07,
      "loss": 0.2605,
      "reward": 0.4481026902794838,
      "reward_std": 0.1638985201716423,
      "rewards/accuracy_reward": 0.06919643143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3789062723517418,
      "step": 2222
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1815.0156860351562,
      "epoch": 0.664028078560227,
      "grad_norm": 53.27512741088867,
      "kl": 2.48046875,
      "learning_rate": 1.5300573232591347e-07,
      "loss": 0.1716,
      "reward": 0.5558036044239998,
      "reward_std": 0.13264968432486057,
      "rewards/accuracy_reward": 0.1450892947614193,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4107143133878708,
      "step": 2223
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1805.5134582519531,
      "epoch": 0.6643267866477485,
      "grad_norm": 14.959280967712402,
      "kl": 4.41796875,
      "learning_rate": 1.527654540672314e-07,
      "loss": 0.2547,
      "reward": 0.5117187723517418,
      "reward_std": 0.14256246387958527,
      "rewards/accuracy_reward": 0.12500000488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3867187723517418,
      "step": 2224
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1835.7969665527344,
      "epoch": 0.66462549473527,
      "grad_norm": 23.359363555908203,
      "kl": 3.00390625,
      "learning_rate": 1.5252528159020524e-07,
      "loss": 0.1958,
      "reward": 0.4905134215950966,
      "reward_std": 0.1470353864133358,
      "rewards/accuracy_reward": 0.08928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4012276977300644,
      "step": 2225
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1880.8907165527344,
      "epoch": 0.6649242028227914,
      "grad_norm": 7.5873332023620605,
      "kl": 3.29296875,
      "learning_rate": 1.5228521515611919e-07,
      "loss": 0.181,
      "reward": 0.4905134215950966,
      "reward_std": 0.2158062420785427,
      "rewards/accuracy_reward": 0.07589286006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205559372902,
      "step": 2226
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1733.2165832519531,
      "epoch": 0.6652229109103129,
      "grad_norm": 29.001022338867188,
      "kl": 2.75390625,
      "learning_rate": 1.52045255026142e-07,
      "loss": 0.1869,
      "reward": 0.5083705559372902,
      "reward_std": 0.13273309543728828,
      "rewards/accuracy_reward": 0.09151786053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4168526902794838,
      "step": 2227
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1825.2300109863281,
      "epoch": 0.6655216189978344,
      "grad_norm": 5.926249980926514,
      "kl": 2.294921875,
      "learning_rate": 1.5180540146132675e-07,
      "loss": 0.1414,
      "reward": 0.5424107313156128,
      "reward_std": 0.19186085276305676,
      "rewards/accuracy_reward": 0.1160714365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263393059372902,
      "step": 2228
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1860.5871276855469,
      "epoch": 0.6658203270853559,
      "grad_norm": 27.400875091552734,
      "kl": 4.40625,
      "learning_rate": 1.5156565472261057e-07,
      "loss": 0.2476,
      "reward": 0.4877232387661934,
      "reward_std": 0.12675922363996506,
      "rewards/accuracy_reward": 0.07812500349245965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4095982313156128,
      "step": 2229
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1854.2300109863281,
      "epoch": 0.6661190351728773,
      "grad_norm": 12.79924201965332,
      "kl": 3.3359375,
      "learning_rate": 1.5132601507081448e-07,
      "loss": 0.2002,
      "reward": 0.4609375223517418,
      "reward_std": 0.1683448702096939,
      "rewards/accuracy_reward": 0.044642860535532236,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.416294664144516,
      "step": 2230
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1782.5090026855469,
      "epoch": 0.6664177432603988,
      "grad_norm": 5.868471145629883,
      "kl": 3.009765625,
      "learning_rate": 1.5108648276664289e-07,
      "loss": 0.1713,
      "reward": 0.4972098395228386,
      "reward_std": 0.15552951395511627,
      "rewards/accuracy_reward": 0.09598215017467737,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4012276977300644,
      "step": 2231
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1770.5179443359375,
      "epoch": 0.6667164513479202,
      "grad_norm": 54.690853118896484,
      "kl": 1.42578125,
      "learning_rate": 1.5084705807068344e-07,
      "loss": 0.0856,
      "reward": 0.6155133992433548,
      "reward_std": 0.1778074074536562,
      "rewards/accuracy_reward": 0.180803582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.434709832072258,
      "step": 2232
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1815.8594970703125,
      "epoch": 0.6670151594354418,
      "grad_norm": 105.45365142822266,
      "kl": 1.4423828125,
      "learning_rate": 1.5060774124340686e-07,
      "loss": 0.1208,
      "reward": 0.4229910969734192,
      "reward_std": 0.12866713665425777,
      "rewards/accuracy_reward": 0.0200892873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4029018059372902,
      "step": 2233
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1914.2835693359375,
      "epoch": 0.6673138675229632,
      "grad_norm": 47.46696853637695,
      "kl": 1.8515625,
      "learning_rate": 1.5036853254516627e-07,
      "loss": 0.1218,
      "reward": 0.4056919813156128,
      "reward_std": 0.13845831714570522,
      "rewards/accuracy_reward": 0.008928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.396763414144516,
      "step": 2234
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1779.1920471191406,
      "epoch": 0.6676125756104847,
      "grad_norm": 25.844369888305664,
      "kl": 2.416015625,
      "learning_rate": 1.5012943223619735e-07,
      "loss": 0.1574,
      "reward": 0.5128348544239998,
      "reward_std": 0.15217616595327854,
      "rewards/accuracy_reward": 0.09151786286383867,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4213169738650322,
      "step": 2235
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1844.99560546875,
      "epoch": 0.6679112836980061,
      "grad_norm": 88.18907928466797,
      "kl": 5.453125,
      "learning_rate": 1.4989044057661777e-07,
      "loss": 0.2741,
      "reward": 0.4765625223517418,
      "reward_std": 0.1489609144628048,
      "rewards/accuracy_reward": 0.05803571571595967,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268059372902,
      "step": 2236
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1832.6384887695312,
      "epoch": 0.6682099917855276,
      "grad_norm": 65.8905258178711,
      "kl": 4.990234375,
      "learning_rate": 1.4965155782642707e-07,
      "loss": 0.2609,
      "reward": 0.498883955180645,
      "reward_std": 0.19453697465360165,
      "rewards/accuracy_reward": 0.07812500488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4207589477300644,
      "step": 2237
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1846.1563110351562,
      "epoch": 0.6685086998730491,
      "grad_norm": 94.49633026123047,
      "kl": 7.3203125,
      "learning_rate": 1.4941278424550625e-07,
      "loss": 0.3648,
      "reward": 0.5117187798023224,
      "reward_std": 0.14260359480977058,
      "rewards/accuracy_reward": 0.12276786379516125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.388950914144516,
      "step": 2238
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1800.9130249023438,
      "epoch": 0.6688074079605706,
      "grad_norm": 17.300281524658203,
      "kl": 3.78515625,
      "learning_rate": 1.4917412009361752e-07,
      "loss": 0.2243,
      "reward": 0.4737723395228386,
      "reward_std": 0.1392871793359518,
      "rewards/accuracy_reward": 0.07142857811413705,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4023437649011612,
      "step": 2239
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1837.7567749023438,
      "epoch": 0.669106116048092,
      "grad_norm": 36.67346954345703,
      "kl": 2.173828125,
      "learning_rate": 1.4893556563040418e-07,
      "loss": 0.1368,
      "reward": 0.4899553880095482,
      "reward_std": 0.1452622301876545,
      "rewards/accuracy_reward": 0.07589286239817739,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625149011612,
      "step": 2240
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1834.8728637695312,
      "epoch": 0.6694048241356134,
      "grad_norm": 41.0516357421875,
      "kl": 2.21875,
      "learning_rate": 1.4869712111539007e-07,
      "loss": 0.1517,
      "reward": 0.4659598395228386,
      "reward_std": 0.20707625150680542,
      "rewards/accuracy_reward": 0.05357143096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.412388414144516,
      "step": 2241
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1811.5983276367188,
      "epoch": 0.669703532223135,
      "grad_norm": 33.667640686035156,
      "kl": 2.177734375,
      "learning_rate": 1.4845878680797942e-07,
      "loss": 0.16,
      "reward": 0.4866071566939354,
      "reward_std": 0.18808109126985073,
      "rewards/accuracy_reward": 0.0915178619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3950893059372902,
      "step": 2242
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1738.4286499023438,
      "epoch": 0.6700022403106564,
      "grad_norm": 16.0098876953125,
      "kl": 2.5703125,
      "learning_rate": 1.4822056296745656e-07,
      "loss": 0.1775,
      "reward": 0.667410746216774,
      "reward_std": 0.1542766820639372,
      "rewards/accuracy_reward": 0.2455357275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 2243
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1805.7657165527344,
      "epoch": 0.6703009483981779,
      "grad_norm": 29.059484481811523,
      "kl": 4.48046875,
      "learning_rate": 1.479824498529858e-07,
      "loss": 0.2575,
      "reward": 0.4838169738650322,
      "reward_std": 0.1428676601499319,
      "rewards/accuracy_reward": 0.0647321455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.419084832072258,
      "step": 2244
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1771.5804443359375,
      "epoch": 0.6705996564856993,
      "grad_norm": 10.385119438171387,
      "kl": 3.74609375,
      "learning_rate": 1.477444477236108e-07,
      "loss": 0.2244,
      "reward": 0.5803571715950966,
      "reward_std": 0.1499339733272791,
      "rewards/accuracy_reward": 0.15848215157166123,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750298023224,
      "step": 2245
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1836.9219970703125,
      "epoch": 0.6708983645732208,
      "grad_norm": 8.37000846862793,
      "kl": 4.10546875,
      "learning_rate": 1.4750655683825454e-07,
      "loss": 0.2264,
      "reward": 0.5256696715950966,
      "reward_std": 0.13385805487632751,
      "rewards/accuracy_reward": 0.12946429220028222,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3962053805589676,
      "step": 2246
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1826.3371276855469,
      "epoch": 0.6711970726607422,
      "grad_norm": 30.40422821044922,
      "kl": 2.45703125,
      "learning_rate": 1.4726877745571898e-07,
      "loss": 0.1438,
      "reward": 0.5546875298023224,
      "reward_std": 0.13541391491889954,
      "rewards/accuracy_reward": 0.1316964365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910969734192,
      "step": 2247
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1827.3572387695312,
      "epoch": 0.6714957807482638,
      "grad_norm": 22.97214126586914,
      "kl": 3.10546875,
      "learning_rate": 1.4703110983468486e-07,
      "loss": 0.1934,
      "reward": 0.435267873108387,
      "reward_std": 0.1530088633298874,
      "rewards/accuracy_reward": 0.017857143888249993,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4174107387661934,
      "step": 2248
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1756.3817749023438,
      "epoch": 0.6717944888357852,
      "grad_norm": 8.939291000366211,
      "kl": 4.15234375,
      "learning_rate": 1.4679355423371117e-07,
      "loss": 0.2441,
      "reward": 0.4441964402794838,
      "reward_std": 0.14485556446015835,
      "rewards/accuracy_reward": 0.03571428591385484,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4084821566939354,
      "step": 2249
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1857.5313415527344,
      "epoch": 0.6720931969233067,
      "grad_norm": 15.77121353149414,
      "kl": 3.6328125,
      "learning_rate": 1.4655611091123511e-07,
      "loss": 0.1884,
      "reward": 0.4709821715950966,
      "reward_std": 0.14920120686292648,
      "rewards/accuracy_reward": 0.05580357322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4151785969734192,
      "step": 2250
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1831.8438110351562,
      "epoch": 0.6723919050108281,
      "grad_norm": 33.26713180541992,
      "kl": 2.212890625,
      "learning_rate": 1.4631878012557186e-07,
      "loss": 0.1527,
      "reward": 0.5284598395228386,
      "reward_std": 0.18534547463059425,
      "rewards/accuracy_reward": 0.11607143399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.412388414144516,
      "step": 2251
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1808.8505554199219,
      "epoch": 0.6726906130983497,
      "grad_norm": 28.90888786315918,
      "kl": 2.154296875,
      "learning_rate": 1.4608156213491398e-07,
      "loss": 0.1386,
      "reward": 0.4843750298023224,
      "reward_std": 0.1605367213487625,
      "rewards/accuracy_reward": 0.06696428591385484,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4174107238650322,
      "step": 2252
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1803.6875610351562,
      "epoch": 0.6729893211858711,
      "grad_norm": 7.708353519439697,
      "kl": 3.3203125,
      "learning_rate": 1.458444571973314e-07,
      "loss": 0.181,
      "reward": 0.5206473544239998,
      "reward_std": 0.18021929264068604,
      "rewards/accuracy_reward": 0.11383929220028222,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4068080559372902,
      "step": 2253
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1780.5022888183594,
      "epoch": 0.6732880292733926,
      "grad_norm": 15.127179145812988,
      "kl": 3.34765625,
      "learning_rate": 1.4560746557077096e-07,
      "loss": 0.2062,
      "reward": 0.4475446715950966,
      "reward_std": 0.13529558666050434,
      "rewards/accuracy_reward": 0.029017858672887087,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268059372902,
      "step": 2254
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1814.1451721191406,
      "epoch": 0.673586737360914,
      "grad_norm": 37.160343170166016,
      "kl": 4.55859375,
      "learning_rate": 1.4537058751305648e-07,
      "loss": 0.2656,
      "reward": 0.4447544813156128,
      "reward_std": 0.19166382774710655,
      "rewards/accuracy_reward": 0.0535714328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3911830633878708,
      "step": 2255
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1860.4041137695312,
      "epoch": 0.6738854454484355,
      "grad_norm": 11.697124481201172,
      "kl": 3.73828125,
      "learning_rate": 1.4513382328188804e-07,
      "loss": 0.1929,
      "reward": 0.466517873108387,
      "reward_std": 0.13220770843327045,
      "rewards/accuracy_reward": 0.0691964328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3973214477300644,
      "step": 2256
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1845.3862609863281,
      "epoch": 0.674184153535957,
      "grad_norm": 19.31157875061035,
      "kl": 3.38671875,
      "learning_rate": 1.448971731348418e-07,
      "loss": 0.1885,
      "reward": 0.4553571566939354,
      "reward_std": 0.13384700566530228,
      "rewards/accuracy_reward": 0.0691964328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3861607313156128,
      "step": 2257
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1885.2478637695312,
      "epoch": 0.6744828616234785,
      "grad_norm": 7.32114839553833,
      "kl": 4.31640625,
      "learning_rate": 1.4466063732937e-07,
      "loss": 0.2278,
      "reward": 0.4815848469734192,
      "reward_std": 0.12960257567465305,
      "rewards/accuracy_reward": 0.082589291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3989955559372902,
      "step": 2258
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1809.1742248535156,
      "epoch": 0.6747815697109999,
      "grad_norm": 39.921119689941406,
      "kl": 3.09375,
      "learning_rate": 1.4442421612280055e-07,
      "loss": 0.2065,
      "reward": 0.5016741380095482,
      "reward_std": 0.13272346556186676,
      "rewards/accuracy_reward": 0.08035714668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4213169887661934,
      "step": 2259
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1777.9554443359375,
      "epoch": 0.6750802777985214,
      "grad_norm": 26.055233001708984,
      "kl": 4.2109375,
      "learning_rate": 1.4418790977233624e-07,
      "loss": 0.2353,
      "reward": 0.5039062798023224,
      "reward_std": 0.1640316154807806,
      "rewards/accuracy_reward": 0.08035714644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.423549123108387,
      "step": 2260
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1790.3237609863281,
      "epoch": 0.6753789858860428,
      "grad_norm": 19.48446273803711,
      "kl": 4.158203125,
      "learning_rate": 1.4395171853505555e-07,
      "loss": 0.2264,
      "reward": 0.5161830633878708,
      "reward_std": 0.16254903934895992,
      "rewards/accuracy_reward": 0.10044643376022577,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.415736623108387,
      "step": 2261
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1827.57373046875,
      "epoch": 0.6756776939735644,
      "grad_norm": 33.797950744628906,
      "kl": 2.318359375,
      "learning_rate": 1.4371564266791117e-07,
      "loss": 0.1421,
      "reward": 0.447544664144516,
      "reward_std": 0.1427716352045536,
      "rewards/accuracy_reward": 0.02455357275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 2262
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1750.4822387695312,
      "epoch": 0.6759764020610858,
      "grad_norm": 26.659408569335938,
      "kl": 2.78515625,
      "learning_rate": 1.4347968242773056e-07,
      "loss": 0.1764,
      "reward": 0.5206473469734192,
      "reward_std": 0.14315833523869514,
      "rewards/accuracy_reward": 0.1004464328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.420200914144516,
      "step": 2263
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1860.2991943359375,
      "epoch": 0.6762751101486073,
      "grad_norm": 11.19881820678711,
      "kl": 3.24609375,
      "learning_rate": 1.4324383807121544e-07,
      "loss": 0.1666,
      "reward": 0.494977705180645,
      "reward_std": 0.23616641014814377,
      "rewards/accuracy_reward": 0.08705357322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.407924123108387,
      "step": 2264
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1866.2835998535156,
      "epoch": 0.6765738182361287,
      "grad_norm": 7.676377773284912,
      "kl": 3.51953125,
      "learning_rate": 1.430081098549412e-07,
      "loss": 0.1931,
      "reward": 0.5217634215950966,
      "reward_std": 0.20982329919934273,
      "rewards/accuracy_reward": 0.10937500465661287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.412388414144516,
      "step": 2265
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1811.1697082519531,
      "epoch": 0.6768725263236502,
      "grad_norm": 14.971346855163574,
      "kl": 4.38671875,
      "learning_rate": 1.4277249803535707e-07,
      "loss": 0.2459,
      "reward": 0.4815848469734192,
      "reward_std": 0.1541061159223318,
      "rewards/accuracy_reward": 0.07366071757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4079241305589676,
      "step": 2266
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1812.6697387695312,
      "epoch": 0.6771712344111717,
      "grad_norm": 6.971721649169922,
      "kl": 3.5771484375,
      "learning_rate": 1.425370028687858e-07,
      "loss": 0.2008,
      "reward": 0.5747768208384514,
      "reward_std": 0.17602366767823696,
      "rewards/accuracy_reward": 0.1584821529686451,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.416294664144516,
      "step": 2267
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1819.4532165527344,
      "epoch": 0.6774699424986932,
      "grad_norm": 30.680177688598633,
      "kl": 2.6015625,
      "learning_rate": 1.4230162461142275e-07,
      "loss": 0.1582,
      "reward": 0.5106026977300644,
      "reward_std": 0.20192551985383034,
      "rewards/accuracy_reward": 0.10714285937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4034598395228386,
      "step": 2268
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1863.8259887695312,
      "epoch": 0.6777686505862146,
      "grad_norm": 46.24980163574219,
      "kl": 2.072265625,
      "learning_rate": 1.4206636351933656e-07,
      "loss": 0.1306,
      "reward": 0.532924123108387,
      "reward_std": 0.18215982615947723,
      "rewards/accuracy_reward": 0.113839291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4190848395228386,
      "step": 2269
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1848.7255249023438,
      "epoch": 0.6780673586737361,
      "grad_norm": 6.81874942779541,
      "kl": 3.19921875,
      "learning_rate": 1.4183121984846835e-07,
      "loss": 0.1832,
      "reward": 0.4458705559372902,
      "reward_std": 0.13625740073621273,
      "rewards/accuracy_reward": 0.053571431431919336,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3922991156578064,
      "step": 2270
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1848.6652221679688,
      "epoch": 0.6783660667612575,
      "grad_norm": 15.339544296264648,
      "kl": 3.609375,
      "learning_rate": 1.4159619385463116e-07,
      "loss": 0.1898,
      "reward": 0.4681919887661934,
      "reward_std": 0.15377994626760483,
      "rewards/accuracy_reward": 0.058035716181620955,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4101562649011612,
      "step": 2271
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1849.9442749023438,
      "epoch": 0.6786647748487791,
      "grad_norm": 11.581968307495117,
      "kl": 3.19140625,
      "learning_rate": 1.413612857935104e-07,
      "loss": 0.1798,
      "reward": 0.4726562649011612,
      "reward_std": 0.17398705147206783,
      "rewards/accuracy_reward": 0.06026786006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4123884066939354,
      "step": 2272
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1862.85498046875,
      "epoch": 0.6789634829363005,
      "grad_norm": 11.202001571655273,
      "kl": 3.9453125,
      "learning_rate": 1.411264959206629e-07,
      "loss": 0.2144,
      "reward": 0.5150669813156128,
      "reward_std": 0.15135764330625534,
      "rewards/accuracy_reward": 0.1026785746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4123884066939354,
      "step": 2273
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1838.4040832519531,
      "epoch": 0.679262191023822,
      "grad_norm": 27.210845947265625,
      "kl": 5.03125,
      "learning_rate": 1.4089182449151708e-07,
      "loss": 0.2816,
      "reward": 0.522321455180645,
      "reward_std": 0.19746014848351479,
      "rewards/accuracy_reward": 0.1339285783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3883928805589676,
      "step": 2274
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1814.555908203125,
      "epoch": 0.6795608991113434,
      "grad_norm": 37.84942626953125,
      "kl": 2.220703125,
      "learning_rate": 1.406572717613726e-07,
      "loss": 0.1441,
      "reward": 0.4291294738650322,
      "reward_std": 0.11951805092394352,
      "rewards/accuracy_reward": 0.006696428870782256,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330484867096,
      "step": 2275
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1832.3840026855469,
      "epoch": 0.679859607198865,
      "grad_norm": 38.643157958984375,
      "kl": 2.189453125,
      "learning_rate": 1.4042283798539957e-07,
      "loss": 0.1389,
      "reward": 0.4720982313156128,
      "reward_std": 0.1818946897983551,
      "rewards/accuracy_reward": 0.05357143096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268133878708,
      "step": 2276
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1752.3170166015625,
      "epoch": 0.6801583152863864,
      "grad_norm": 8.059804916381836,
      "kl": 3.087890625,
      "learning_rate": 1.4018852341863907e-07,
      "loss": 0.1831,
      "reward": 0.5708705559372902,
      "reward_std": 0.16612395457923412,
      "rewards/accuracy_reward": 0.15401786752045155,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.416852705180645,
      "step": 2277
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1842.8572387695312,
      "epoch": 0.6804570233739079,
      "grad_norm": 37.96380615234375,
      "kl": 5.40625,
      "learning_rate": 1.3995432831600254e-07,
      "loss": 0.2854,
      "reward": 0.4737723469734192,
      "reward_std": 0.1620611995458603,
      "rewards/accuracy_reward": 0.07366071827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.400111623108387,
      "step": 2278
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1797.13623046875,
      "epoch": 0.6807557314614293,
      "grad_norm": 26.19983673095703,
      "kl": 4.96875,
      "learning_rate": 1.3972025293227106e-07,
      "loss": 0.2602,
      "reward": 0.4335937649011612,
      "reward_std": 0.1328039299696684,
      "rewards/accuracy_reward": 0.022321430267766118,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.411272332072258,
      "step": 2279
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1767.1340026855469,
      "epoch": 0.6810544395489508,
      "grad_norm": 7.110368251800537,
      "kl": 3.9140625,
      "learning_rate": 1.394862975220958e-07,
      "loss": 0.2332,
      "reward": 0.602678582072258,
      "reward_std": 0.1831081472337246,
      "rewards/accuracy_reward": 0.180803582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 2280
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1810.5023193359375,
      "epoch": 0.6813531476364723,
      "grad_norm": 45.5844841003418,
      "kl": 2.296875,
      "learning_rate": 1.3925246233999743e-07,
      "loss": 0.1474,
      "reward": 0.5133928805589676,
      "reward_std": 0.13518951833248138,
      "rewards/accuracy_reward": 0.0870535746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263393059372902,
      "step": 2281
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1740.7634582519531,
      "epoch": 0.6816518557239938,
      "grad_norm": 27.854764938354492,
      "kl": 2.15625,
      "learning_rate": 1.3901874764036553e-07,
      "loss": 0.1383,
      "reward": 0.4743303805589676,
      "reward_std": 0.16959016397595406,
      "rewards/accuracy_reward": 0.053571431897580624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4207589402794838,
      "step": 2282
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1789.8147888183594,
      "epoch": 0.6819505638115152,
      "grad_norm": 43.78980255126953,
      "kl": 2.43359375,
      "learning_rate": 1.3878515367745886e-07,
      "loss": 0.1512,
      "reward": 0.5987723469734192,
      "reward_std": 0.161507036536932,
      "rewards/accuracy_reward": 0.1830357201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4157366305589676,
      "step": 2283
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1756.2455749511719,
      "epoch": 0.6822492718990366,
      "grad_norm": 20.92259407043457,
      "kl": 2.025390625,
      "learning_rate": 1.3855168070540484e-07,
      "loss": 0.1311,
      "reward": 0.5558035895228386,
      "reward_std": 0.1605326235294342,
      "rewards/accuracy_reward": 0.1116071492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4441964477300644,
      "step": 2284
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1913.6050109863281,
      "epoch": 0.6825479799865581,
      "grad_norm": 46.264522552490234,
      "kl": 5.3515625,
      "learning_rate": 1.3831832897819905e-07,
      "loss": 0.2391,
      "reward": 0.474888414144516,
      "reward_std": 0.18064400181174278,
      "rewards/accuracy_reward": 0.08482143469154835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3900669738650322,
      "step": 2285
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1831.5960693359375,
      "epoch": 0.6828466880740796,
      "grad_norm": 37.7057991027832,
      "kl": 5.0625,
      "learning_rate": 1.3808509874970541e-07,
      "loss": 0.2598,
      "reward": 0.5150669887661934,
      "reward_std": 0.17999398335814476,
      "rewards/accuracy_reward": 0.09821428917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4168526902794838,
      "step": 2286
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1844.3326721191406,
      "epoch": 0.6831453961616011,
      "grad_norm": 35.68173599243164,
      "kl": 5.46484375,
      "learning_rate": 1.378519902736554e-07,
      "loss": 0.2739,
      "reward": 0.4447544813156128,
      "reward_std": 0.17052270472049713,
      "rewards/accuracy_reward": 0.035714288242161274,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4090401977300644,
      "step": 2287
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1825.5938110351562,
      "epoch": 0.6834441042491225,
      "grad_norm": 18.42423439025879,
      "kl": 3.40625,
      "learning_rate": 1.3761900380364834e-07,
      "loss": 0.2059,
      "reward": 0.4910714402794838,
      "reward_std": 0.12804099172353745,
      "rewards/accuracy_reward": 0.0848214328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500149011612,
      "step": 2288
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1823.43310546875,
      "epoch": 0.683742812336644,
      "grad_norm": 27.275354385375977,
      "kl": 2.533203125,
      "learning_rate": 1.3738613959315066e-07,
      "loss": 0.169,
      "reward": 0.584821455180645,
      "reward_std": 0.2129845693707466,
      "rewards/accuracy_reward": 0.16964286053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4151785895228386,
      "step": 2289
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1778.1518859863281,
      "epoch": 0.6840415204241654,
      "grad_norm": 31.560060501098633,
      "kl": 3.09375,
      "learning_rate": 1.3715339789549564e-07,
      "loss": 0.2062,
      "reward": 0.5714285895228386,
      "reward_std": 0.12910557724535465,
      "rewards/accuracy_reward": 0.1584821492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4129464477300644,
      "step": 2290
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1761.7679443359375,
      "epoch": 0.684340228511687,
      "grad_norm": 11.656278610229492,
      "kl": 4.11328125,
      "learning_rate": 1.369207789638836e-07,
      "loss": 0.2398,
      "reward": 0.526227705180645,
      "reward_std": 0.16148805804550648,
      "rewards/accuracy_reward": 0.1116071492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205559372902,
      "step": 2291
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1857.7032165527344,
      "epoch": 0.6846389365992084,
      "grad_norm": 19.717029571533203,
      "kl": 5.1328125,
      "learning_rate": 1.3668828305138116e-07,
      "loss": 0.2705,
      "reward": 0.4469866380095482,
      "reward_std": 0.18407762795686722,
      "rewards/accuracy_reward": 0.051339289639145136,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3956473469734192,
      "step": 2292
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1793.5871276855469,
      "epoch": 0.6849376446867299,
      "grad_norm": 11.887115478515625,
      "kl": 3.88671875,
      "learning_rate": 1.3645591041092094e-07,
      "loss": 0.2364,
      "reward": 0.4815848544239998,
      "reward_std": 0.13165586814284325,
      "rewards/accuracy_reward": 0.08482143259607255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3967634066939354,
      "step": 2293
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1830.1228332519531,
      "epoch": 0.6852363527742513,
      "grad_norm": 33.36900329589844,
      "kl": 2.2734375,
      "learning_rate": 1.3622366129530166e-07,
      "loss": 0.1358,
      "reward": 0.4430803805589676,
      "reward_std": 0.13829679787158966,
      "rewards/accuracy_reward": 0.022321429569274187,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4207589402794838,
      "step": 2294
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1777.3237609863281,
      "epoch": 0.6855350608617728,
      "grad_norm": 19.76296043395996,
      "kl": 2.87109375,
      "learning_rate": 1.3599153595718773e-07,
      "loss": 0.1849,
      "reward": 0.513392873108387,
      "reward_std": 0.16689422726631165,
      "rewards/accuracy_reward": 0.1049107201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4084821566939354,
      "step": 2295
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1800.6250610351562,
      "epoch": 0.6858337689492943,
      "grad_norm": 29.906021118164062,
      "kl": 2.75,
      "learning_rate": 1.3575953464910855e-07,
      "loss": 0.1808,
      "reward": 0.5786830633878708,
      "reward_std": 0.14733159355819225,
      "rewards/accuracy_reward": 0.1562500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330559372902,
      "step": 2296
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1847.2679443359375,
      "epoch": 0.6861324770368158,
      "grad_norm": 13.44467830657959,
      "kl": 4.1875,
      "learning_rate": 1.3552765762345889e-07,
      "loss": 0.2318,
      "reward": 0.594308078289032,
      "reward_std": 0.16371478140354156,
      "rewards/accuracy_reward": 0.1785714328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.415736623108387,
      "step": 2297
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1806.0425109863281,
      "epoch": 0.6864311851243372,
      "grad_norm": 11.341994285583496,
      "kl": 4.6328125,
      "learning_rate": 1.3529590513249834e-07,
      "loss": 0.272,
      "reward": 0.4492187798023224,
      "reward_std": 0.14527273178100586,
      "rewards/accuracy_reward": 0.046875003492459655,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4023437649011612,
      "step": 2298
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1856.8505249023438,
      "epoch": 0.6867298932118587,
      "grad_norm": 14.80528736114502,
      "kl": 2.75,
      "learning_rate": 1.350642774283507e-07,
      "loss": 0.1592,
      "reward": 0.5139509215950966,
      "reward_std": 0.14822277054190636,
      "rewards/accuracy_reward": 0.09598214668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4179687723517418,
      "step": 2299
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1802.4420471191406,
      "epoch": 0.6870286012993801,
      "grad_norm": 14.312652587890625,
      "kl": 2.90625,
      "learning_rate": 1.3483277476300442e-07,
      "loss": 0.165,
      "reward": 0.4469866305589676,
      "reward_std": 0.15317968651652336,
      "rewards/accuracy_reward": 0.029017857974395156,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4179687649011612,
      "step": 2300
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1791.2188110351562,
      "epoch": 0.6873273093869017,
      "grad_norm": 17.865530014038086,
      "kl": 4.396484375,
      "learning_rate": 1.3460139738831144e-07,
      "loss": 0.2524,
      "reward": 0.4520089402794838,
      "reward_std": 0.1305291187018156,
      "rewards/accuracy_reward": 0.03571428847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4162946566939354,
      "step": 2301
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1826.2322387695312,
      "epoch": 0.6876260174744231,
      "grad_norm": 7.019198417663574,
      "kl": 3.2734375,
      "learning_rate": 1.3437014555598788e-07,
      "loss": 0.1863,
      "reward": 0.5039062723517418,
      "reward_std": 0.14002435468137264,
      "rewards/accuracy_reward": 0.08928571734577417,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205559372902,
      "step": 2302
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1783.5268859863281,
      "epoch": 0.6879247255619446,
      "grad_norm": 481.5315246582031,
      "kl": 7.37890625,
      "learning_rate": 1.341390195176131e-07,
      "loss": 0.3723,
      "reward": 0.4782366305589676,
      "reward_std": 0.14395453035831451,
      "rewards/accuracy_reward": 0.05357143026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4246651902794838,
      "step": 2303
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1833.6875915527344,
      "epoch": 0.688223433649466,
      "grad_norm": 10.538017272949219,
      "kl": 3.578125,
      "learning_rate": 1.339080195246294e-07,
      "loss": 0.211,
      "reward": 0.4614955559372902,
      "reward_std": 0.13816064968705177,
      "rewards/accuracy_reward": 0.05357143119908869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4079241156578064,
      "step": 2304
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1841.6540832519531,
      "epoch": 0.6885221417369876,
      "grad_norm": 4.950185775756836,
      "kl": 3.4921875,
      "learning_rate": 1.336771458283422e-07,
      "loss": 0.1959,
      "reward": 0.5585937723517418,
      "reward_std": 0.1743004210293293,
      "rewards/accuracy_reward": 0.1473214402794838,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4112723395228386,
      "step": 2305
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1802.4197387695312,
      "epoch": 0.688820849824509,
      "grad_norm": 13.231029510498047,
      "kl": 4.8359375,
      "learning_rate": 1.3344639867991957e-07,
      "loss": 0.2769,
      "reward": 0.5234375149011612,
      "reward_std": 0.13038799911737442,
      "rewards/accuracy_reward": 0.1160714328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4073660969734192,
      "step": 2306
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1861.2120971679688,
      "epoch": 0.6891195579120305,
      "grad_norm": 6.581903457641602,
      "kl": 3.7265625,
      "learning_rate": 1.3321577833039162e-07,
      "loss": 0.1871,
      "reward": 0.443080373108387,
      "reward_std": 0.14962735958397388,
      "rewards/accuracy_reward": 0.0357142873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4073660895228386,
      "step": 2307
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1757.2009887695312,
      "epoch": 0.6894182659995519,
      "grad_norm": 21.00966453552246,
      "kl": 2.244140625,
      "learning_rate": 1.3298528503065073e-07,
      "loss": 0.1488,
      "reward": 0.483816996216774,
      "reward_std": 0.17069623619318008,
      "rewards/accuracy_reward": 0.06696428777649999,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4168526902794838,
      "step": 2308
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1835.5558471679688,
      "epoch": 0.6897169740870734,
      "grad_norm": 18.678773880004883,
      "kl": 2.80078125,
      "learning_rate": 1.3275491903145118e-07,
      "loss": 0.1761,
      "reward": 0.4793526977300644,
      "reward_std": 0.17067987471818924,
      "rewards/accuracy_reward": 0.07142857694998384,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.407924123108387,
      "step": 2309
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1798.5782165527344,
      "epoch": 0.6900156821745949,
      "grad_norm": 28.260032653808594,
      "kl": 2.494140625,
      "learning_rate": 1.3252468058340835e-07,
      "loss": 0.1622,
      "reward": 0.4849330633878708,
      "reward_std": 0.14634467475116253,
      "rewards/accuracy_reward": 0.06250000232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330484867096,
      "step": 2310
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1818.6406860351562,
      "epoch": 0.6903143902621164,
      "grad_norm": 7.030909061431885,
      "kl": 3.251953125,
      "learning_rate": 1.3229456993699923e-07,
      "loss": 0.1873,
      "reward": 0.4799107313156128,
      "reward_std": 0.17785080149769783,
      "rewards/accuracy_reward": 0.0558035746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.424107164144516,
      "step": 2311
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1782.4531860351562,
      "epoch": 0.6906130983496378,
      "grad_norm": 28.04505157470703,
      "kl": 5.3203125,
      "learning_rate": 1.3206458734256177e-07,
      "loss": 0.2907,
      "reward": 0.5022321566939354,
      "reward_std": 0.1640472076833248,
      "rewards/accuracy_reward": 0.08035714644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 2312
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1790.5134582519531,
      "epoch": 0.6909118064371593,
      "grad_norm": 29.145143508911133,
      "kl": 4.6796875,
      "learning_rate": 1.3183473305029425e-07,
      "loss": 0.2585,
      "reward": 0.5658482313156128,
      "reward_std": 0.17875580117106438,
      "rewards/accuracy_reward": 0.13839286053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.427455373108387,
      "step": 2313
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1778.4487609863281,
      "epoch": 0.6912105145246807,
      "grad_norm": 16.276432037353516,
      "kl": 4.5390625,
      "learning_rate": 1.316050073102558e-07,
      "loss": 0.2463,
      "reward": 0.446986623108387,
      "reward_std": 0.14768668450415134,
      "rewards/accuracy_reward": 0.026785715017467737,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.420200914144516,
      "step": 2314
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1798.8438110351562,
      "epoch": 0.6915092226122023,
      "grad_norm": 21.490264892578125,
      "kl": 3.7265625,
      "learning_rate": 1.3137541037236537e-07,
      "loss": 0.2121,
      "reward": 0.5725446790456772,
      "reward_std": 0.18477138318121433,
      "rewards/accuracy_reward": 0.1540178619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185267984867096,
      "step": 2315
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1774.8683776855469,
      "epoch": 0.6918079306997237,
      "grad_norm": 26.730506896972656,
      "kl": 2.568359375,
      "learning_rate": 1.3114594248640203e-07,
      "loss": 0.1421,
      "reward": 0.4815848395228386,
      "reward_std": 0.13136619143188,
      "rewards/accuracy_reward": 0.06250000465661287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4190848469734192,
      "step": 2316
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1815.0693054199219,
      "epoch": 0.6921066387872452,
      "grad_norm": 12.669639587402344,
      "kl": 3.296875,
      "learning_rate": 1.3091660390200438e-07,
      "loss": 0.1937,
      "reward": 0.4715401977300644,
      "reward_std": 0.15540923923254013,
      "rewards/accuracy_reward": 0.06250000302679837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4090401977300644,
      "step": 2317
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1768.4933776855469,
      "epoch": 0.6924053468747666,
      "grad_norm": 18.05756950378418,
      "kl": 3.4921875,
      "learning_rate": 1.3068739486867015e-07,
      "loss": 0.2105,
      "reward": 0.5139509066939354,
      "reward_std": 0.17640399374067783,
      "rewards/accuracy_reward": 0.10044643376022577,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4135044887661934,
      "step": 2318
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1861.7857971191406,
      "epoch": 0.6927040549622882,
      "grad_norm": 10.468100547790527,
      "kl": 3.953125,
      "learning_rate": 1.3045831563575643e-07,
      "loss": 0.2219,
      "reward": 0.4481026902794838,
      "reward_std": 0.1887516789138317,
      "rewards/accuracy_reward": 0.046875001629814506,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4012276977300644,
      "step": 2319
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1775.1741943359375,
      "epoch": 0.6930027630498096,
      "grad_norm": 8.991233825683594,
      "kl": 3.546875,
      "learning_rate": 1.3022936645247904e-07,
      "loss": 0.1983,
      "reward": 0.4933035969734192,
      "reward_std": 0.15109683386981487,
      "rewards/accuracy_reward": 0.07142857392318547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 2320
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1898.3817749023438,
      "epoch": 0.6933014711373311,
      "grad_norm": 13.724143028259277,
      "kl": 4.4765625,
      "learning_rate": 1.3000054756791206e-07,
      "loss": 0.2284,
      "reward": 0.3917410895228386,
      "reward_std": 0.12733750976622105,
      "rewards/accuracy_reward": 0.013392857508733869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3783482313156128,
      "step": 2321
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1755.1853637695312,
      "epoch": 0.6936001792248525,
      "grad_norm": 62.894630432128906,
      "kl": 1.0888671875,
      "learning_rate": 1.2977185923098813e-07,
      "loss": 0.1181,
      "reward": 0.513950914144516,
      "reward_std": 0.15728253684937954,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4425223395228386,
      "step": 2322
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1800.9666137695312,
      "epoch": 0.693898887312374,
      "grad_norm": 70.9227066040039,
      "kl": 2.111328125,
      "learning_rate": 1.295433016904978e-07,
      "loss": 0.1389,
      "reward": 0.5703125298023224,
      "reward_std": 0.19661297276616096,
      "rewards/accuracy_reward": 0.15401786752045155,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.416294664144516,
      "step": 2323
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1807.2568054199219,
      "epoch": 0.6941975953998955,
      "grad_norm": 102.58361053466797,
      "kl": 1.33984375,
      "learning_rate": 1.2931487519508914e-07,
      "loss": 0.1137,
      "reward": 0.5290178880095482,
      "reward_std": 0.15740134939551353,
      "rewards/accuracy_reward": 0.10267857951112092,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263393133878708,
      "step": 2324
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1830.3572387695312,
      "epoch": 0.694496303487417,
      "grad_norm": 82.88196563720703,
      "kl": 1.4921875,
      "learning_rate": 1.290865799932679e-07,
      "loss": 0.1174,
      "reward": 0.4375000223517418,
      "reward_std": 0.11289165914058685,
      "rewards/accuracy_reward": 0.015625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 2325
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1713.27685546875,
      "epoch": 0.6947950115749384,
      "grad_norm": 53.8298225402832,
      "kl": 1.361328125,
      "learning_rate": 1.2885841633339685e-07,
      "loss": 0.1173,
      "reward": 0.4743303656578064,
      "reward_std": 0.13796906918287277,
      "rewards/accuracy_reward": 0.03794643096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.436383955180645,
      "step": 2326
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1865.8036804199219,
      "epoch": 0.6950937196624598,
      "grad_norm": 23.767873764038086,
      "kl": 2.875,
      "learning_rate": 1.286303844636956e-07,
      "loss": 0.1706,
      "reward": 0.5474330708384514,
      "reward_std": 0.17863717675209045,
      "rewards/accuracy_reward": 0.12723214854486287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4202009215950966,
      "step": 2327
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1854.2634887695312,
      "epoch": 0.6953924277499813,
      "grad_norm": 10.475944519042969,
      "kl": 4.3671875,
      "learning_rate": 1.284024846322407e-07,
      "loss": 0.2326,
      "reward": 0.4866071715950966,
      "reward_std": 0.17675423994660378,
      "rewards/accuracy_reward": 0.08482143026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4017857313156128,
      "step": 2328
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1768.71435546875,
      "epoch": 0.6956911358375027,
      "grad_norm": 8.587560653686523,
      "kl": 3.3125,
      "learning_rate": 1.2817471708696453e-07,
      "loss": 0.2055,
      "reward": 0.5161830484867096,
      "reward_std": 0.1239225510507822,
      "rewards/accuracy_reward": 0.08035714528523386,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4358259066939354,
      "step": 2329
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1771.8616943359375,
      "epoch": 0.6959898439250243,
      "grad_norm": 22.29239845275879,
      "kl": 4.25390625,
      "learning_rate": 1.2794708207565604e-07,
      "loss": 0.2302,
      "reward": 0.525669664144516,
      "reward_std": 0.1373024843633175,
      "rewards/accuracy_reward": 0.09598214644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875223517418,
      "step": 2330
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1827.7657165527344,
      "epoch": 0.6962885520125457,
      "grad_norm": 42.8067626953125,
      "kl": 5.1796875,
      "learning_rate": 1.2771957984595988e-07,
      "loss": 0.2681,
      "reward": 0.4687500149011612,
      "reward_std": 0.1681443639099598,
      "rewards/accuracy_reward": 0.0535714328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.415178582072258,
      "step": 2331
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1825.7188415527344,
      "epoch": 0.6965872601000672,
      "grad_norm": 6.760704517364502,
      "kl": 3.619140625,
      "learning_rate": 1.2749221064537596e-07,
      "loss": 0.2056,
      "reward": 0.545758955180645,
      "reward_std": 0.16525132581591606,
      "rewards/accuracy_reward": 0.11830357694998384,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4274553805589676,
      "step": 2332
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1772.9866638183594,
      "epoch": 0.6968859681875886,
      "grad_norm": 9.520299911499023,
      "kl": 3.8125,
      "learning_rate": 1.272649747212598e-07,
      "loss": 0.2141,
      "reward": 0.4257812723517418,
      "reward_std": 0.12550578080117702,
      "rewards/accuracy_reward": 0.008928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4168526902794838,
      "step": 2333
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1789.69873046875,
      "epoch": 0.6971846762751102,
      "grad_norm": 8.589119911193848,
      "kl": 2.61328125,
      "learning_rate": 1.2703787232082195e-07,
      "loss": 0.145,
      "reward": 0.5005580559372902,
      "reward_std": 0.20239262282848358,
      "rewards/accuracy_reward": 0.07812500419095159,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330484867096,
      "step": 2334
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1840.8281860351562,
      "epoch": 0.6974833843626316,
      "grad_norm": 24.76255226135254,
      "kl": 2.8203125,
      "learning_rate": 1.268109036911273e-07,
      "loss": 0.1586,
      "reward": 0.4966518133878708,
      "reward_std": 0.1398679930716753,
      "rewards/accuracy_reward": 0.0691964328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4274553805589676,
      "step": 2335
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1840.8996276855469,
      "epoch": 0.6977820924501531,
      "grad_norm": 7.063710689544678,
      "kl": 3.484375,
      "learning_rate": 1.2658406907909562e-07,
      "loss": 0.1791,
      "reward": 0.5457589402794838,
      "reward_std": 0.20056127384305,
      "rewards/accuracy_reward": 0.1316964365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625149011612,
      "step": 2336
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1778.9375610351562,
      "epoch": 0.6980808005376745,
      "grad_norm": 14.450241088867188,
      "kl": 2.77734375,
      "learning_rate": 1.263573687315008e-07,
      "loss": 0.171,
      "reward": 0.4732143133878708,
      "reward_std": 0.171335494145751,
      "rewards/accuracy_reward": 0.05133928661234677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 2337
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1853.5469665527344,
      "epoch": 0.698379508625196,
      "grad_norm": 9.449134826660156,
      "kl": 4.2890625,
      "learning_rate": 1.2613080289497046e-07,
      "loss": 0.2429,
      "reward": 0.4760044813156128,
      "reward_std": 0.13882718980312347,
      "rewards/accuracy_reward": 0.06919643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4068080484867096,
      "step": 2338
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1795.0447387695312,
      "epoch": 0.6986782167127175,
      "grad_norm": 21.11614990234375,
      "kl": 3.19921875,
      "learning_rate": 1.2590437181598608e-07,
      "loss": 0.2091,
      "reward": 0.4737723469734192,
      "reward_std": 0.11964461021125317,
      "rewards/accuracy_reward": 0.04910714481957257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4246651977300644,
      "step": 2339
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1799.0269165039062,
      "epoch": 0.698976924800239,
      "grad_norm": 12.637781143188477,
      "kl": 4.44140625,
      "learning_rate": 1.2567807574088252e-07,
      "loss": 0.2518,
      "reward": 0.4849330633878708,
      "reward_std": 0.15998396836221218,
      "rewards/accuracy_reward": 0.07366071757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4112723395228386,
      "step": 2340
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1773.1808471679688,
      "epoch": 0.6992756328877604,
      "grad_norm": 9.389719009399414,
      "kl": 4.046875,
      "learning_rate": 1.2545191491584766e-07,
      "loss": 0.252,
      "reward": 0.533482164144516,
      "reward_std": 0.15666637755930424,
      "rewards/accuracy_reward": 0.10937500186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.424107164144516,
      "step": 2341
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1781.618408203125,
      "epoch": 0.6995743409752819,
      "grad_norm": 7.314539432525635,
      "kl": 4.3671875,
      "learning_rate": 1.2522588958692242e-07,
      "loss": 0.2524,
      "reward": 0.4402901977300644,
      "reward_std": 0.14623912423849106,
      "rewards/accuracy_reward": 0.022321429569274187,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4179687649011612,
      "step": 2342
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1811.1228637695312,
      "epoch": 0.6998730490628033,
      "grad_norm": 11.186851501464844,
      "kl": 3.24609375,
      "learning_rate": 1.2500000000000005e-07,
      "loss": 0.1967,
      "reward": 0.5636160895228386,
      "reward_std": 0.18568257987499237,
      "rewards/accuracy_reward": 0.14062501024454832,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 2343
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1783.1116638183594,
      "epoch": 0.7001717571503249,
      "grad_norm": 6.269689083099365,
      "kl": 3.109375,
      "learning_rate": 1.2477424640082634e-07,
      "loss": 0.1864,
      "reward": 0.5385044887661934,
      "reward_std": 0.13482287526130676,
      "rewards/accuracy_reward": 0.1026785746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4358259215950966,
      "step": 2344
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1822.4822387695312,
      "epoch": 0.7004704652378463,
      "grad_norm": 6.870690822601318,
      "kl": 3.74609375,
      "learning_rate": 1.245486290349992e-07,
      "loss": 0.2093,
      "reward": 0.525111623108387,
      "reward_std": 0.170251727104187,
      "rewards/accuracy_reward": 0.11383929150179029,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4112723395228386,
      "step": 2345
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1788.5357971191406,
      "epoch": 0.7007691733253678,
      "grad_norm": 38.80241012573242,
      "kl": 2.546875,
      "learning_rate": 1.2432314814796808e-07,
      "loss": 0.1907,
      "reward": 0.4804687723517418,
      "reward_std": 0.12757463939487934,
      "rewards/accuracy_reward": 0.05803571757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330633878708,
      "step": 2346
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1858.212158203125,
      "epoch": 0.7010678814128892,
      "grad_norm": 9.211620330810547,
      "kl": 3.40625,
      "learning_rate": 1.2409780398503405e-07,
      "loss": 0.2041,
      "reward": 0.4570312723517418,
      "reward_std": 0.1657471749931574,
      "rewards/accuracy_reward": 0.04017857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4168526902794838,
      "step": 2347
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1850.9911804199219,
      "epoch": 0.7013665895004108,
      "grad_norm": 13.792752265930176,
      "kl": 3.6015625,
      "learning_rate": 1.2387259679134964e-07,
      "loss": 0.2128,
      "reward": 0.4849330559372902,
      "reward_std": 0.1466073002666235,
      "rewards/accuracy_reward": 0.06919643143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.415736623108387,
      "step": 2348
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1745.2746276855469,
      "epoch": 0.7016652975879322,
      "grad_norm": 21.6173038482666,
      "kl": 4.50390625,
      "learning_rate": 1.2364752681191806e-07,
      "loss": 0.2609,
      "reward": 0.4977678880095482,
      "reward_std": 0.1821570172905922,
      "rewards/accuracy_reward": 0.07812500419095159,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4196428805589676,
      "step": 2349
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1843.9018859863281,
      "epoch": 0.7019640056754537,
      "grad_norm": 40.01692199707031,
      "kl": 5.53125,
      "learning_rate": 1.234225942915934e-07,
      "loss": 0.2985,
      "reward": 0.4899553656578064,
      "reward_std": 0.13741761445999146,
      "rewards/accuracy_reward": 0.08258928940631449,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4073660895228386,
      "step": 2350
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1818.7590026855469,
      "epoch": 0.7022627137629751,
      "grad_norm": 12.28449821472168,
      "kl": 3.76953125,
      "learning_rate": 1.2319779947508032e-07,
      "loss": 0.1921,
      "reward": 0.4375000223517418,
      "reward_std": 0.14590265788137913,
      "rewards/accuracy_reward": 0.020089286845177412,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4174107313156128,
      "step": 2351
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1716.74560546875,
      "epoch": 0.7025614218504966,
      "grad_norm": 14.940701484680176,
      "kl": 3.046875,
      "learning_rate": 1.229731426069334e-07,
      "loss": 0.1876,
      "reward": 0.635044664144516,
      "reward_std": 0.20123385079205036,
      "rewards/accuracy_reward": 0.2075893022119999,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4274553805589676,
      "step": 2352
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1765.2031860351562,
      "epoch": 0.702860129938018,
      "grad_norm": 31.647991180419922,
      "kl": 2.45703125,
      "learning_rate": 1.2274862393155743e-07,
      "loss": 0.1607,
      "reward": 0.5463169813156128,
      "reward_std": 0.19926150515675545,
      "rewards/accuracy_reward": 0.12276786286383867,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.423549123108387,
      "step": 2353
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1788.8617248535156,
      "epoch": 0.7031588380255396,
      "grad_norm": 16.964784622192383,
      "kl": 3.078125,
      "learning_rate": 1.2252424369320655e-07,
      "loss": 0.1724,
      "reward": 0.439732164144516,
      "reward_std": 0.14505527541041374,
      "rewards/accuracy_reward": 0.024553572991862893,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4151785969734192,
      "step": 2354
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1790.1898193359375,
      "epoch": 0.703457546113061,
      "grad_norm": 28.85265350341797,
      "kl": 2.701171875,
      "learning_rate": 1.2230000213598456e-07,
      "loss": 0.1771,
      "reward": 0.489955373108387,
      "reward_std": 0.1441718377172947,
      "rewards/accuracy_reward": 0.06919643096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4207589477300644,
      "step": 2355
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1792.4398193359375,
      "epoch": 0.7037562542005825,
      "grad_norm": 15.449318885803223,
      "kl": 4.00390625,
      "learning_rate": 1.2207589950384438e-07,
      "loss": 0.2407,
      "reward": 0.4347098469734192,
      "reward_std": 0.14693696051836014,
      "rewards/accuracy_reward": 0.040178574388846755,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3945312723517418,
      "step": 2356
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1771.0603332519531,
      "epoch": 0.7040549622881039,
      "grad_norm": 9.857447624206543,
      "kl": 3.6171875,
      "learning_rate": 1.2185193604058757e-07,
      "loss": 0.2159,
      "reward": 0.5725446715950966,
      "reward_std": 0.18209994956851006,
      "rewards/accuracy_reward": 0.1540178656578064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268059372902,
      "step": 2357
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1773.6072082519531,
      "epoch": 0.7043536703756255,
      "grad_norm": 27.088909149169922,
      "kl": 5.984375,
      "learning_rate": 1.2162811198986447e-07,
      "loss": 0.3171,
      "reward": 0.5424107387661934,
      "reward_std": 0.13507209345698357,
      "rewards/accuracy_reward": 0.13392858067527413,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4084821566939354,
      "step": 2358
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1838.29248046875,
      "epoch": 0.7046523784631469,
      "grad_norm": 20.271066665649414,
      "kl": 4.06640625,
      "learning_rate": 1.2140442759517382e-07,
      "loss": 0.2283,
      "reward": 0.4665178805589676,
      "reward_std": 0.18487808108329773,
      "rewards/accuracy_reward": 0.055803573690354824,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4107143059372902,
      "step": 2359
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1765.7166137695312,
      "epoch": 0.7049510865506684,
      "grad_norm": 11.207869529724121,
      "kl": 3.201171875,
      "learning_rate": 1.2118088309986217e-07,
      "loss": 0.1879,
      "reward": 0.5602678805589676,
      "reward_std": 0.19498751126229763,
      "rewards/accuracy_reward": 0.13839286286383867,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750149011612,
      "step": 2360
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1848.2523498535156,
      "epoch": 0.7052497946381898,
      "grad_norm": 14.72805404663086,
      "kl": 3.98828125,
      "learning_rate": 1.2095747874712404e-07,
      "loss": 0.2264,
      "reward": 0.427455373108387,
      "reward_std": 0.14801684580743313,
      "rewards/accuracy_reward": 0.022321429569274187,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4051339477300644,
      "step": 2361
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1822.5335693359375,
      "epoch": 0.7055485027257113,
      "grad_norm": 15.283440589904785,
      "kl": 4.26953125,
      "learning_rate": 1.207342147800016e-07,
      "loss": 0.2296,
      "reward": 0.4921875298023224,
      "reward_std": 0.16096188127994537,
      "rewards/accuracy_reward": 0.08928571874275804,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4029018059372902,
      "step": 2362
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1808.3549499511719,
      "epoch": 0.7058472108132328,
      "grad_norm": 20.200292587280273,
      "kl": 3.5390625,
      "learning_rate": 1.20511091441384e-07,
      "loss": 0.2144,
      "reward": 0.478794664144516,
      "reward_std": 0.1604021191596985,
      "rewards/accuracy_reward": 0.06473214784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625149011612,
      "step": 2363
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1813.8393859863281,
      "epoch": 0.7061459189007543,
      "grad_norm": 5.426363945007324,
      "kl": 3.39453125,
      "learning_rate": 1.202881089740076e-07,
      "loss": 0.2029,
      "reward": 0.4626116380095482,
      "reward_std": 0.13159577175974846,
      "rewards/accuracy_reward": 0.051339288242161274,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4112723395228386,
      "step": 2364
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1821.5960693359375,
      "epoch": 0.7064446269882757,
      "grad_norm": 5.461429119110107,
      "kl": 4.16015625,
      "learning_rate": 1.2006526762045563e-07,
      "loss": 0.2407,
      "reward": 0.510044664144516,
      "reward_std": 0.18972396105527878,
      "rewards/accuracy_reward": 0.10044643329456449,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4095982387661934,
      "step": 2365
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1805.7567749023438,
      "epoch": 0.7067433350757972,
      "grad_norm": 9.928297996520996,
      "kl": 3.50390625,
      "learning_rate": 1.198425676231574e-07,
      "loss": 0.2105,
      "reward": 0.5792410969734192,
      "reward_std": 0.12962576746940613,
      "rewards/accuracy_reward": 0.15625000861473382,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 2366
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1824.7746276855469,
      "epoch": 0.7070420431633186,
      "grad_norm": 8.336638450622559,
      "kl": 3.8828125,
      "learning_rate": 1.196200092243889e-07,
      "loss": 0.2125,
      "reward": 0.5887276902794838,
      "reward_std": 0.12562943622469902,
      "rewards/accuracy_reward": 0.1562500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776977300644,
      "step": 2367
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1798.9107971191406,
      "epoch": 0.7073407512508402,
      "grad_norm": 6.459567070007324,
      "kl": 3.52734375,
      "learning_rate": 1.1939759266627164e-07,
      "loss": 0.2055,
      "reward": 0.5212053805589676,
      "reward_std": 0.11128736473619938,
      "rewards/accuracy_reward": 0.08705357578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341518059372902,
      "step": 2368
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1745.8371276855469,
      "epoch": 0.7076394593383616,
      "grad_norm": 15.765811920166016,
      "kl": 3.9765625,
      "learning_rate": 1.1917531819077318e-07,
      "loss": 0.2424,
      "reward": 0.5195312798023224,
      "reward_std": 0.1305846907198429,
      "rewards/accuracy_reward": 0.09375000488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812649011612,
      "step": 2369
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1858.0157165527344,
      "epoch": 0.707938167425883,
      "grad_norm": 6.259654998779297,
      "kl": 3.83984375,
      "learning_rate": 1.1895318603970641e-07,
      "loss": 0.2117,
      "reward": 0.4737723395228386,
      "reward_std": 0.16261932626366615,
      "rewards/accuracy_reward": 0.05580357322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4179687649011612,
      "step": 2370
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1739.5090026855469,
      "epoch": 0.7082368755134045,
      "grad_norm": 9.80095100402832,
      "kl": 4.296875,
      "learning_rate": 1.1873119645472915e-07,
      "loss": 0.2508,
      "reward": 0.5418527126312256,
      "reward_std": 0.18165267072618008,
      "rewards/accuracy_reward": 0.1316964365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4101562723517418,
      "step": 2371
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1716.1741943359375,
      "epoch": 0.7085355836009259,
      "grad_norm": 18.86343002319336,
      "kl": 1.43701171875,
      "learning_rate": 1.1850934967734444e-07,
      "loss": 0.096,
      "reward": 0.584263414144516,
      "reward_std": 0.1726251933723688,
      "rewards/accuracy_reward": 0.14062500558793545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443638414144516,
      "step": 2372
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1842.18310546875,
      "epoch": 0.7088342916884475,
      "grad_norm": 34.17057418823242,
      "kl": 2.24609375,
      "learning_rate": 1.1828764594889992e-07,
      "loss": 0.1379,
      "reward": 0.4715401977300644,
      "reward_std": 0.11822165921330452,
      "rewards/accuracy_reward": 0.042410716181620955,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294887661934,
      "step": 2373
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1841.8795471191406,
      "epoch": 0.7091329997759689,
      "grad_norm": 19.992822647094727,
      "kl": 2.49609375,
      "learning_rate": 1.180660855105873e-07,
      "loss": 0.1518,
      "reward": 0.525669664144516,
      "reward_std": 0.16457823105156422,
      "rewards/accuracy_reward": 0.10491072200238705,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.420758955180645,
      "step": 2374
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1770.5134887695312,
      "epoch": 0.7094317078634904,
      "grad_norm": 13.566108703613281,
      "kl": 3.408203125,
      "learning_rate": 1.1784466860344272e-07,
      "loss": 0.2166,
      "reward": 0.5351562723517418,
      "reward_std": 0.16907967627048492,
      "rewards/accuracy_reward": 0.10937500488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812649011612,
      "step": 2375
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1739.58935546875,
      "epoch": 0.7097304159510118,
      "grad_norm": 14.752470016479492,
      "kl": 4.41015625,
      "learning_rate": 1.1762339546834618e-07,
      "loss": 0.2627,
      "reward": 0.510044664144516,
      "reward_std": 0.1607834417372942,
      "rewards/accuracy_reward": 0.08035714668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875223517418,
      "step": 2376
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1826.0826721191406,
      "epoch": 0.7100291240385334,
      "grad_norm": 16.78652000427246,
      "kl": 4.71875,
      "learning_rate": 1.1740226634602099e-07,
      "loss": 0.2527,
      "reward": 0.5569196715950966,
      "reward_std": 0.18307537958025932,
      "rewards/accuracy_reward": 0.1428571529686451,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625149011612,
      "step": 2377
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1876.04248046875,
      "epoch": 0.7103278321260548,
      "grad_norm": 22.39483070373535,
      "kl": 4.63671875,
      "learning_rate": 1.1718128147703405e-07,
      "loss": 0.249,
      "reward": 0.4726562798023224,
      "reward_std": 0.16766344383358955,
      "rewards/accuracy_reward": 0.06473214528523386,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.407924123108387,
      "step": 2378
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1876.2054443359375,
      "epoch": 0.7106265402135763,
      "grad_norm": 7.543182849884033,
      "kl": 4.08984375,
      "learning_rate": 1.1696044110179532e-07,
      "loss": 0.1985,
      "reward": 0.5496651902794838,
      "reward_std": 0.1329356748610735,
      "rewards/accuracy_reward": 0.13839286239817739,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4112723395228386,
      "step": 2379
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1768.4107971191406,
      "epoch": 0.7109252483010977,
      "grad_norm": 30.028804779052734,
      "kl": 2.25390625,
      "learning_rate": 1.1673974546055726e-07,
      "loss": 0.1646,
      "reward": 0.5518973469734192,
      "reward_std": 0.16922985017299652,
      "rewards/accuracy_reward": 0.11383928963914514,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4380580559372902,
      "step": 2380
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1764.2411499023438,
      "epoch": 0.7112239563886192,
      "grad_norm": 49.80156707763672,
      "kl": 2.158203125,
      "learning_rate": 1.1651919479341538e-07,
      "loss": 0.1407,
      "reward": 0.4570312723517418,
      "reward_std": 0.1568269580602646,
      "rewards/accuracy_reward": 0.03125000116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812723517418,
      "step": 2381
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1767.1697082519531,
      "epoch": 0.7115226644761407,
      "grad_norm": 53.72437286376953,
      "kl": 1.927734375,
      "learning_rate": 1.162987893403069e-07,
      "loss": 0.1483,
      "reward": 0.5061384215950966,
      "reward_std": 0.12915064580738544,
      "rewards/accuracy_reward": 0.08035714644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812723517418,
      "step": 2382
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1859.665283203125,
      "epoch": 0.7118213725636622,
      "grad_norm": 32.8027229309082,
      "kl": 1.97265625,
      "learning_rate": 1.1607852934101151e-07,
      "loss": 0.1272,
      "reward": 0.4581473469734192,
      "reward_std": 0.17100156471133232,
      "rewards/accuracy_reward": 0.044642859837040305,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4135044813156128,
      "step": 2383
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1818.8148193359375,
      "epoch": 0.7121200806511836,
      "grad_norm": 10.564373970031738,
      "kl": 2.111328125,
      "learning_rate": 1.1585841503515056e-07,
      "loss": 0.1206,
      "reward": 0.4486607387661934,
      "reward_std": 0.15155724622309208,
      "rewards/accuracy_reward": 0.02455357275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.424107164144516,
      "step": 2384
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1833.9755554199219,
      "epoch": 0.7124187887387051,
      "grad_norm": 17.677459716796875,
      "kl": 3.2421875,
      "learning_rate": 1.1563844666218669e-07,
      "loss": 0.2065,
      "reward": 0.4771205633878708,
      "reward_std": 0.15114394389092922,
      "rewards/accuracy_reward": 0.06250000232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205559372902,
      "step": 2385
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1840.0915832519531,
      "epoch": 0.7127174968262265,
      "grad_norm": 9.652724266052246,
      "kl": 4.9140625,
      "learning_rate": 1.15418624461424e-07,
      "loss": 0.267,
      "reward": 0.4492187649011612,
      "reward_std": 0.12138241529464722,
      "rewards/accuracy_reward": 0.04017857206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4090401977300644,
      "step": 2386
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1754.5960693359375,
      "epoch": 0.7130162049137481,
      "grad_norm": 4.1428117752075195,
      "kl": 3.05859375,
      "learning_rate": 1.1519894867200761e-07,
      "loss": 0.1848,
      "reward": 0.495535746216774,
      "reward_std": 0.1754977498203516,
      "rewards/accuracy_reward": 0.0691964328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263393133878708,
      "step": 2387
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1798.7233276367188,
      "epoch": 0.7133149130012695,
      "grad_norm": 7.098261833190918,
      "kl": 4.046875,
      "learning_rate": 1.1497941953292303e-07,
      "loss": 0.2403,
      "reward": 0.577008955180645,
      "reward_std": 0.1458814423531294,
      "rewards/accuracy_reward": 0.16517857648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.411830373108387,
      "step": 2388
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1720.1273193359375,
      "epoch": 0.713613621088791,
      "grad_norm": 20.7607421875,
      "kl": 2.662109375,
      "learning_rate": 1.1476003728299657e-07,
      "loss": 0.1844,
      "reward": 0.5641741380095482,
      "reward_std": 0.11848766170442104,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4391741305589676,
      "step": 2389
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1749.0804138183594,
      "epoch": 0.7139123291763124,
      "grad_norm": 12.546521186828613,
      "kl": 3.52734375,
      "learning_rate": 1.1454080216089468e-07,
      "loss": 0.1856,
      "reward": 0.4933035969734192,
      "reward_std": 0.15864093601703644,
      "rewards/accuracy_reward": 0.06250000488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035969734192,
      "step": 2390
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1778.618408203125,
      "epoch": 0.714211037263834,
      "grad_norm": 8.720478057861328,
      "kl": 2.60546875,
      "learning_rate": 1.1432171440512353e-07,
      "loss": 0.1462,
      "reward": 0.5697544887661934,
      "reward_std": 0.19435515999794006,
      "rewards/accuracy_reward": 0.13839286379516125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4313616305589676,
      "step": 2391
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1806.2679138183594,
      "epoch": 0.7145097453513554,
      "grad_norm": 15.638729095458984,
      "kl": 2.8671875,
      "learning_rate": 1.141027742540292e-07,
      "loss": 0.1617,
      "reward": 0.5368303805589676,
      "reward_std": 0.1441308967769146,
      "rewards/accuracy_reward": 0.1049107201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4319196566939354,
      "step": 2392
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1792.6898193359375,
      "epoch": 0.7148084534388769,
      "grad_norm": 5.050838470458984,
      "kl": 3.33203125,
      "learning_rate": 1.1388398194579724e-07,
      "loss": 0.1964,
      "reward": 0.5239955559372902,
      "reward_std": 0.15415244549512863,
      "rewards/accuracy_reward": 0.0937500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455559372902,
      "step": 2393
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1803.5090026855469,
      "epoch": 0.7151071615263983,
      "grad_norm": 43.18879318237305,
      "kl": 5.16015625,
      "learning_rate": 1.1366533771845199e-07,
      "loss": 0.2595,
      "reward": 0.4732143133878708,
      "reward_std": 0.16946572810411453,
      "rewards/accuracy_reward": 0.05357143119908869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.419642873108387,
      "step": 2394
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1807.2165832519531,
      "epoch": 0.7154058696139198,
      "grad_norm": 24.81668472290039,
      "kl": 4.69921875,
      "learning_rate": 1.134468418098572e-07,
      "loss": 0.2759,
      "reward": 0.471540205180645,
      "reward_std": 0.1316543035209179,
      "rewards/accuracy_reward": 0.04910714505240321,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330559372902,
      "step": 2395
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1740.2344055175781,
      "epoch": 0.7157045777014412,
      "grad_norm": 7.656452178955078,
      "kl": 3.0390625,
      "learning_rate": 1.1322849445771473e-07,
      "loss": 0.1786,
      "reward": 0.547991082072258,
      "reward_std": 0.1985822692513466,
      "rewards/accuracy_reward": 0.10491072107106447,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4430803805589676,
      "step": 2396
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1681.0915832519531,
      "epoch": 0.7160032857889628,
      "grad_norm": 6.560885429382324,
      "kl": 2.810546875,
      "learning_rate": 1.1301029589956528e-07,
      "loss": 0.2041,
      "reward": 0.5128348395228386,
      "reward_std": 0.15756717883050442,
      "rewards/accuracy_reward": 0.08482143119908869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428013414144516,
      "step": 2397
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1753.3215026855469,
      "epoch": 0.7163019938764842,
      "grad_norm": 4.95013427734375,
      "kl": 3.54296875,
      "learning_rate": 1.1279224637278755e-07,
      "loss": 0.2162,
      "reward": 0.5479910895228386,
      "reward_std": 0.19631147757172585,
      "rewards/accuracy_reward": 0.13169643376022577,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4162946566939354,
      "step": 2398
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1786.7433776855469,
      "epoch": 0.7166007019640057,
      "grad_norm": 31.503252029418945,
      "kl": 2.458984375,
      "learning_rate": 1.1257434611459785e-07,
      "loss": 0.1537,
      "reward": 0.4810268059372902,
      "reward_std": 0.13181341998279095,
      "rewards/accuracy_reward": 0.06696428963914514,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625223517418,
      "step": 2399
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1801.4331359863281,
      "epoch": 0.7168994100515271,
      "grad_norm": 18.471357345581055,
      "kl": 2.89453125,
      "learning_rate": 1.1235659536205045e-07,
      "loss": 0.1763,
      "reward": 0.5864955633878708,
      "reward_std": 0.13726701773703098,
      "rewards/accuracy_reward": 0.16071428824216127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812723517418,
      "step": 2400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1806.0290832519531,
      "epoch": 0.7171981181390487,
      "grad_norm": 32.24931335449219,
      "kl": 2.55859375,
      "learning_rate": 1.1213899435203686e-07,
      "loss": 0.1674,
      "reward": 0.5574776977300644,
      "reward_std": 0.1549007184803486,
      "rewards/accuracy_reward": 0.129464291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428013414144516,
      "step": 2401
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1795.8706359863281,
      "epoch": 0.7174968262265701,
      "grad_norm": 22.35542869567871,
      "kl": 4.96484375,
      "learning_rate": 1.1192154332128547e-07,
      "loss": 0.295,
      "reward": 0.4486607313156128,
      "reward_std": 0.1741066500544548,
      "rewards/accuracy_reward": 0.03794643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4107142984867096,
      "step": 2402
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1759.7858276367188,
      "epoch": 0.7177955343140916,
      "grad_norm": 25.364810943603516,
      "kl": 5.0625,
      "learning_rate": 1.1170424250636185e-07,
      "loss": 0.3034,
      "reward": 0.5228794738650322,
      "reward_std": 0.16371148265898228,
      "rewards/accuracy_reward": 0.10937500558793545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4135044887661934,
      "step": 2403
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1781.01123046875,
      "epoch": 0.718094242401613,
      "grad_norm": 16.583526611328125,
      "kl": 3.765625,
      "learning_rate": 1.11487092143668e-07,
      "loss": 0.2104,
      "reward": 0.4737723469734192,
      "reward_std": 0.1625739485025406,
      "rewards/accuracy_reward": 0.05133928777649999,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330559372902,
      "step": 2404
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1821.7322082519531,
      "epoch": 0.7183929504891345,
      "grad_norm": 11.287320137023926,
      "kl": 3.5234375,
      "learning_rate": 1.1127009246944208e-07,
      "loss": 0.1933,
      "reward": 0.4899553880095482,
      "reward_std": 0.15909849293529987,
      "rewards/accuracy_reward": 0.06919643096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.420758955180645,
      "step": 2405
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1808.1295776367188,
      "epoch": 0.718691658576656,
      "grad_norm": 24.36695098876953,
      "kl": 3.41015625,
      "learning_rate": 1.1105324371975857e-07,
      "loss": 0.2203,
      "reward": 0.5535714477300644,
      "reward_std": 0.16505314968526363,
      "rewards/accuracy_reward": 0.145089291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4084821566939354,
      "step": 2406
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1797.4018859863281,
      "epoch": 0.7189903666641775,
      "grad_norm": 28.975238800048828,
      "kl": 2.451171875,
      "learning_rate": 1.1083654613052764e-07,
      "loss": 0.1542,
      "reward": 0.4547991380095482,
      "reward_std": 0.16203102096915245,
      "rewards/accuracy_reward": 0.029017858672887087,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812574505806,
      "step": 2407
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1739.74560546875,
      "epoch": 0.7192890747516989,
      "grad_norm": 560.644287109375,
      "kl": 8.359375,
      "learning_rate": 1.1061999993749508e-07,
      "loss": 0.4097,
      "reward": 0.4843750074505806,
      "reward_std": 0.16469849087297916,
      "rewards/accuracy_reward": 0.06026786123402417,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.424107164144516,
      "step": 2408
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1860.1764221191406,
      "epoch": 0.7195877828392204,
      "grad_norm": 20.137100219726562,
      "kl": 3.4921875,
      "learning_rate": 1.1040360537624191e-07,
      "loss": 0.1835,
      "reward": 0.4352678805589676,
      "reward_std": 0.12332583963871002,
      "rewards/accuracy_reward": 0.029017857974395156,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500223517418,
      "step": 2409
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1806.5313415527344,
      "epoch": 0.7198864909267418,
      "grad_norm": 40.23284912109375,
      "kl": 3.1640625,
      "learning_rate": 1.1018736268218412e-07,
      "loss": 0.2126,
      "reward": 0.463727705180645,
      "reward_std": 0.1408079881221056,
      "rewards/accuracy_reward": 0.049107146449387074,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205559372902,
      "step": 2410
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1762.6496276855469,
      "epoch": 0.7201851990142634,
      "grad_norm": 14.169441223144531,
      "kl": 2.71875,
      "learning_rate": 1.099712720905726e-07,
      "loss": 0.1609,
      "reward": 0.4726562723517418,
      "reward_std": 0.1601056009531021,
      "rewards/accuracy_reward": 0.04017857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776977300644,
      "step": 2411
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1785.0000915527344,
      "epoch": 0.7204839071017848,
      "grad_norm": 9.486129760742188,
      "kl": 3.71875,
      "learning_rate": 1.0975533383649285e-07,
      "loss": 0.2316,
      "reward": 0.5396205633878708,
      "reward_std": 0.14515913277864456,
      "rewards/accuracy_reward": 0.12946429150179029,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4101562649011612,
      "step": 2412
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1804.4509582519531,
      "epoch": 0.7207826151893062,
      "grad_norm": 5.653343677520752,
      "kl": 3.306640625,
      "learning_rate": 1.0953954815486436e-07,
      "loss": 0.1931,
      "reward": 0.474330373108387,
      "reward_std": 0.1691528856754303,
      "rewards/accuracy_reward": 0.04687500186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.427455373108387,
      "step": 2413
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1719.2255249023438,
      "epoch": 0.7210813232768277,
      "grad_norm": 17.845239639282227,
      "kl": 2.4765625,
      "learning_rate": 1.0932391528044094e-07,
      "loss": 0.1753,
      "reward": 0.5262277126312256,
      "reward_std": 0.1883224993944168,
      "rewards/accuracy_reward": 0.08482143143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4414062723517418,
      "step": 2414
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1775.5982971191406,
      "epoch": 0.7213800313643491,
      "grad_norm": 19.440322875976562,
      "kl": 5.359375,
      "learning_rate": 1.0910843544780987e-07,
      "loss": 0.2964,
      "reward": 0.5993303805589676,
      "reward_std": 0.20246483385562897,
      "rewards/accuracy_reward": 0.1808035783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268059372902,
      "step": 2415
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1784.7366943359375,
      "epoch": 0.7216787394518707,
      "grad_norm": 6.092052936553955,
      "kl": 3.6484375,
      "learning_rate": 1.088931088913922e-07,
      "loss": 0.2092,
      "reward": 0.559709832072258,
      "reward_std": 0.13023032806813717,
      "rewards/accuracy_reward": 0.1205357201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.439174123108387,
      "step": 2416
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1755.2813110351562,
      "epoch": 0.7219774475393921,
      "grad_norm": 13.594573974609375,
      "kl": 3.89453125,
      "learning_rate": 1.0867793584544216e-07,
      "loss": 0.2299,
      "reward": 0.5066964402794838,
      "reward_std": 0.12845467031002045,
      "rewards/accuracy_reward": 0.07589286006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035969734192,
      "step": 2417
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.7054443359375,
      "epoch": 0.7222761556269136,
      "grad_norm": 12.550678253173828,
      "kl": 3.90234375,
      "learning_rate": 1.084629165440468e-07,
      "loss": 0.2321,
      "reward": 0.493861623108387,
      "reward_std": 0.12634802795946598,
      "rewards/accuracy_reward": 0.08482143399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4090401902794838,
      "step": 2418
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1724.57373046875,
      "epoch": 0.722574863714435,
      "grad_norm": 9.8344144821167,
      "kl": 4.296875,
      "learning_rate": 1.082480512211261e-07,
      "loss": 0.2862,
      "reward": 0.6138393208384514,
      "reward_std": 0.1814941093325615,
      "rewards/accuracy_reward": 0.1875000074505806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263393059372902,
      "step": 2419
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1730.6407165527344,
      "epoch": 0.7228735718019565,
      "grad_norm": 15.728743553161621,
      "kl": 2.37109375,
      "learning_rate": 1.0803334011043258e-07,
      "loss": 0.1526,
      "reward": 0.5552455633878708,
      "reward_std": 0.18135413527488708,
      "rewards/accuracy_reward": 0.10937500325962901,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4458705559372902,
      "step": 2420
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1786.2478637695312,
      "epoch": 0.723172279889478,
      "grad_norm": 7.246221542358398,
      "kl": 3.61328125,
      "learning_rate": 1.0781878344555067e-07,
      "loss": 0.2163,
      "reward": 0.5915178805589676,
      "reward_std": 0.19372060522437096,
      "rewards/accuracy_reward": 0.15848215110599995,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357313156128,
      "step": 2421
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1857.40185546875,
      "epoch": 0.7234709879769995,
      "grad_norm": 41.586605072021484,
      "kl": 5.296875,
      "learning_rate": 1.0760438145989717e-07,
      "loss": 0.2696,
      "reward": 0.466517873108387,
      "reward_std": 0.1800931841135025,
      "rewards/accuracy_reward": 0.053571431431919336,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4129464477300644,
      "step": 2422
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1764.63623046875,
      "epoch": 0.7237696960645209,
      "grad_norm": 13.38475227355957,
      "kl": 3.103515625,
      "learning_rate": 1.0739013438672043e-07,
      "loss": 0.1955,
      "reward": 0.5072544813156128,
      "reward_std": 0.15691293589770794,
      "rewards/accuracy_reward": 0.07366071734577417,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937723517418,
      "step": 2423
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1836.5514221191406,
      "epoch": 0.7240684041520424,
      "grad_norm": 39.78252029418945,
      "kl": 4.6328125,
      "learning_rate": 1.0717604245910012e-07,
      "loss": 0.2433,
      "reward": 0.4709821715950966,
      "reward_std": 0.15256065130233765,
      "rewards/accuracy_reward": 0.05803571757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4129464402794838,
      "step": 2424
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1765.1920471191406,
      "epoch": 0.7243671122395638,
      "grad_norm": 7.882612228393555,
      "kl": 3.5,
      "learning_rate": 1.0696210590994736e-07,
      "loss": 0.2074,
      "reward": 0.5625000149011612,
      "reward_std": 0.17608734965324402,
      "rewards/accuracy_reward": 0.12723214668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435267873108387,
      "step": 2425
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1799.38623046875,
      "epoch": 0.7246658203270854,
      "grad_norm": 5.4424333572387695,
      "kl": 3.36328125,
      "learning_rate": 1.0674832497200425e-07,
      "loss": 0.1955,
      "reward": 0.512276791036129,
      "reward_std": 0.14732623100280762,
      "rewards/accuracy_reward": 0.09821429080329835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625149011612,
      "step": 2426
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1715.1741638183594,
      "epoch": 0.7249645284146068,
      "grad_norm": 17.144968032836914,
      "kl": 2.59765625,
      "learning_rate": 1.0653469987784331e-07,
      "loss": 0.1616,
      "reward": 0.5440848544239998,
      "reward_std": 0.1448826715350151,
      "rewards/accuracy_reward": 0.09598214644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4481026977300644,
      "step": 2427
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1738.4152526855469,
      "epoch": 0.7252632365021283,
      "grad_norm": 44.91627502441406,
      "kl": 2.47265625,
      "learning_rate": 1.0632123085986788e-07,
      "loss": 0.1716,
      "reward": 0.5262276977300644,
      "reward_std": 0.14621329680085182,
      "rewards/accuracy_reward": 0.08928571920841932,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419887661934,
      "step": 2428
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1770.0804138183594,
      "epoch": 0.7255619445896497,
      "grad_norm": 7.293946266174316,
      "kl": 4.0703125,
      "learning_rate": 1.0610791815031114e-07,
      "loss": 0.2371,
      "reward": 0.4603794813156128,
      "reward_std": 0.12955988198518753,
      "rewards/accuracy_reward": 0.049107146449387074,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4112723395228386,
      "step": 2429
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1827.9554443359375,
      "epoch": 0.7258606526771713,
      "grad_norm": 15.784100532531738,
      "kl": 4.1484375,
      "learning_rate": 1.058947619812365e-07,
      "loss": 0.2094,
      "reward": 0.5267857387661934,
      "reward_std": 0.17344837822020054,
      "rewards/accuracy_reward": 0.11383929336443543,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4129464402794838,
      "step": 2430
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1792.8438415527344,
      "epoch": 0.7261593607646927,
      "grad_norm": 16.269140243530273,
      "kl": 3.3203125,
      "learning_rate": 1.0568176258453712e-07,
      "loss": 0.1897,
      "reward": 0.4369419813156128,
      "reward_std": 0.1579836793243885,
      "rewards/accuracy_reward": 0.026785715017467737,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4101562723517418,
      "step": 2431
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1794.4375915527344,
      "epoch": 0.7264580688522142,
      "grad_norm": 12.674710273742676,
      "kl": 2.4970703125,
      "learning_rate": 1.0546892019193523e-07,
      "loss": 0.1452,
      "reward": 0.5552455633878708,
      "reward_std": 0.13266366347670555,
      "rewards/accuracy_reward": 0.1272321455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4280134066939354,
      "step": 2432
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1727.7077026367188,
      "epoch": 0.7267567769397356,
      "grad_norm": 27.235422134399414,
      "kl": 2.625,
      "learning_rate": 1.052562350349826e-07,
      "loss": 0.1916,
      "reward": 0.5379464626312256,
      "reward_std": 0.15297571197152138,
      "rewards/accuracy_reward": 0.10044643329456449,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000149011612,
      "step": 2433
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1768.0915832519531,
      "epoch": 0.7270554850272571,
      "grad_norm": 5.215934753417969,
      "kl": 3.517578125,
      "learning_rate": 1.0504370734505993e-07,
      "loss": 0.2037,
      "reward": 0.5390625149011612,
      "reward_std": 0.189414344727993,
      "rewards/accuracy_reward": 0.1093750074505806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875223517418,
      "step": 2434
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1707.0357971191406,
      "epoch": 0.7273541931147786,
      "grad_norm": 10.802674293518066,
      "kl": 2.728515625,
      "learning_rate": 1.0483133735337635e-07,
      "loss": 0.1941,
      "reward": 0.5083705708384514,
      "reward_std": 0.15484930016100407,
      "rewards/accuracy_reward": 0.064732147147879,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4436384066939354,
      "step": 2435
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1847.6741943359375,
      "epoch": 0.7276529012023001,
      "grad_norm": 20.80423355102539,
      "kl": 3.94140625,
      "learning_rate": 1.0461912529096967e-07,
      "loss": 0.2002,
      "reward": 0.5251116305589676,
      "reward_std": 0.16722449474036694,
      "rewards/accuracy_reward": 0.1026785746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330559372902,
      "step": 2436
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1860.5915832519531,
      "epoch": 0.7279516092898215,
      "grad_norm": 7.680060863494873,
      "kl": 3.71484375,
      "learning_rate": 1.0440707138870589e-07,
      "loss": 0.1908,
      "reward": 0.4944196715950966,
      "reward_std": 0.1624172255396843,
      "rewards/accuracy_reward": 0.07366071501746774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4207589477300644,
      "step": 2437
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1815.8148193359375,
      "epoch": 0.728250317377343,
      "grad_norm": 12.394641876220703,
      "kl": 2.900390625,
      "learning_rate": 1.0419517587727872e-07,
      "loss": 0.1781,
      "reward": 0.4893973469734192,
      "reward_std": 0.14169423654675484,
      "rewards/accuracy_reward": 0.05803571850992739,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.431361623108387,
      "step": 2438
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1770.5134887695312,
      "epoch": 0.7285490254648644,
      "grad_norm": 5.094322681427002,
      "kl": 3.796875,
      "learning_rate": 1.0398343898720987e-07,
      "loss": 0.2265,
      "reward": 0.5479910895228386,
      "reward_std": 0.16166459023952484,
      "rewards/accuracy_reward": 0.11830357555299997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875149011612,
      "step": 2439
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1791.227783203125,
      "epoch": 0.728847733552386,
      "grad_norm": 8.830268859863281,
      "kl": 3.91796875,
      "learning_rate": 1.0377186094884815e-07,
      "loss": 0.2223,
      "reward": 0.5094866380095482,
      "reward_std": 0.17784508690238,
      "rewards/accuracy_reward": 0.0892857201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.420200914144516,
      "step": 2440
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1761.7969665527344,
      "epoch": 0.7291464416399074,
      "grad_norm": 7.605828285217285,
      "kl": 2.3046875,
      "learning_rate": 1.0356044199236982e-07,
      "loss": 0.1354,
      "reward": 0.5552455633878708,
      "reward_std": 0.12088223733007908,
      "rewards/accuracy_reward": 0.113839291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4414062649011612,
      "step": 2441
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1795.3237609863281,
      "epoch": 0.7294451497274289,
      "grad_norm": 17.28439712524414,
      "kl": 2.6640625,
      "learning_rate": 1.0334918234777809e-07,
      "loss": 0.1694,
      "reward": 0.5396205708384514,
      "reward_std": 0.15077169053256512,
      "rewards/accuracy_reward": 0.10044643096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.439174123108387,
      "step": 2442
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1820.2232666015625,
      "epoch": 0.7297438578149503,
      "grad_norm": 15.56547737121582,
      "kl": 3.916015625,
      "learning_rate": 1.0313808224490253e-07,
      "loss": 0.1993,
      "reward": 0.5072544813156128,
      "reward_std": 0.1442029308527708,
      "rewards/accuracy_reward": 0.09375000605359674,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4135044813156128,
      "step": 2443
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1797.32373046875,
      "epoch": 0.7300425659024719,
      "grad_norm": 6.343161106109619,
      "kl": 2.859375,
      "learning_rate": 1.0292714191339946e-07,
      "loss": 0.1739,
      "reward": 0.4910714477300644,
      "reward_std": 0.16152603551745415,
      "rewards/accuracy_reward": 0.06026786006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035895228386,
      "step": 2444
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1752.4665832519531,
      "epoch": 0.7303412739899933,
      "grad_norm": 7.181160926818848,
      "kl": 3.1328125,
      "learning_rate": 1.0271636158275143e-07,
      "loss": 0.188,
      "reward": 0.5217634066939354,
      "reward_std": 0.13296915777027607,
      "rewards/accuracy_reward": 0.08482143143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419887661934,
      "step": 2445
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1811.415283203125,
      "epoch": 0.7306399820775148,
      "grad_norm": 5.441793918609619,
      "kl": 3.796875,
      "learning_rate": 1.0250574148226654e-07,
      "loss": 0.2165,
      "reward": 0.5033482387661934,
      "reward_std": 0.14979437738656998,
      "rewards/accuracy_reward": 0.09598215017467737,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.407366082072258,
      "step": 2446
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1810.32373046875,
      "epoch": 0.7309386901650362,
      "grad_norm": 9.2046480178833,
      "kl": 3.09765625,
      "learning_rate": 1.0229528184107892e-07,
      "loss": 0.1598,
      "reward": 0.4843750149011612,
      "reward_std": 0.12292457930743694,
      "rewards/accuracy_reward": 0.05133928800933063,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357387661934,
      "step": 2447
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1744.9040832519531,
      "epoch": 0.7312373982525577,
      "grad_norm": 16.724777221679688,
      "kl": 2.740234375,
      "learning_rate": 1.0208498288814813e-07,
      "loss": 0.1714,
      "reward": 0.4776785969734192,
      "reward_std": 0.1339544542133808,
      "rewards/accuracy_reward": 0.026785716181620955,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.450892873108387,
      "step": 2448
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1726.0670471191406,
      "epoch": 0.7315361063400792,
      "grad_norm": 9.860130310058594,
      "kl": 3.68359375,
      "learning_rate": 1.0187484485225864e-07,
      "loss": 0.2312,
      "reward": 0.547433041036129,
      "reward_std": 0.14477303624153137,
      "rewards/accuracy_reward": 0.11383928847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937649011612,
      "step": 2449
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1790.884033203125,
      "epoch": 0.7318348144276007,
      "grad_norm": 19.418649673461914,
      "kl": 4.29296875,
      "learning_rate": 1.0166486796202012e-07,
      "loss": 0.2452,
      "reward": 0.5083705559372902,
      "reward_std": 0.1649991013109684,
      "rewards/accuracy_reward": 0.0848214328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4235491305589676,
      "step": 2450
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1776.1496276855469,
      "epoch": 0.7321335225151221,
      "grad_norm": 18.984111785888672,
      "kl": 4.9921875,
      "learning_rate": 1.0145505244586697e-07,
      "loss": 0.2753,
      "reward": 0.5189732313156128,
      "reward_std": 0.1406539287418127,
      "rewards/accuracy_reward": 0.10267857694998384,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4162946566939354,
      "step": 2451
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1774.2255249023438,
      "epoch": 0.7324322306026436,
      "grad_norm": 4.973389625549316,
      "kl": 3.83203125,
      "learning_rate": 1.0124539853205766e-07,
      "loss": 0.2241,
      "reward": 0.5290178805589676,
      "reward_std": 0.14920012652873993,
      "rewards/accuracy_reward": 0.10044643143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 2452
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1844.9554443359375,
      "epoch": 0.732730938690165,
      "grad_norm": 4.226149559020996,
      "kl": 3.421875,
      "learning_rate": 1.0103590644867535e-07,
      "loss": 0.1858,
      "reward": 0.4860491305589676,
      "reward_std": 0.17224475741386414,
      "rewards/accuracy_reward": 0.07366071734577417,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4123884215950966,
      "step": 2453
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1851.4308776855469,
      "epoch": 0.7330296467776866,
      "grad_norm": 5.98801326751709,
      "kl": 3.576171875,
      "learning_rate": 1.0082657642362666e-07,
      "loss": 0.2057,
      "reward": 0.4910714477300644,
      "reward_std": 0.18428654968738556,
      "rewards/accuracy_reward": 0.08258928847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4084821566939354,
      "step": 2454
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1805.1719665527344,
      "epoch": 0.733328354865208,
      "grad_norm": 8.540138244628906,
      "kl": 3.421875,
      "learning_rate": 1.0061740868464227e-07,
      "loss": 0.1965,
      "reward": 0.4698660969734192,
      "reward_std": 0.17062033526599407,
      "rewards/accuracy_reward": 0.046875001629814506,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 2455
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1812.634033203125,
      "epoch": 0.7336270629527294,
      "grad_norm": 35.15149688720703,
      "kl": 1.94140625,
      "learning_rate": 1.0040840345927623e-07,
      "loss": 0.1212,
      "reward": 0.5279018059372902,
      "reward_std": 0.19380417466163635,
      "rewards/accuracy_reward": 0.1049107201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 2456
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1686.7098999023438,
      "epoch": 0.7339257710402509,
      "grad_norm": 30.371028900146484,
      "kl": 1.5791015625,
      "learning_rate": 1.0019956097490562e-07,
      "loss": 0.1114,
      "reward": 0.5892857387661934,
      "reward_std": 0.16236157529056072,
      "rewards/accuracy_reward": 0.13616071944124997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4531250223517418,
      "step": 2457
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1738.6585693359375,
      "epoch": 0.7342244791277723,
      "grad_norm": 22.71346664428711,
      "kl": 2.146484375,
      "learning_rate": 9.999088145873064e-08,
      "loss": 0.1417,
      "reward": 0.5619419887661934,
      "reward_std": 0.1414557546377182,
      "rewards/accuracy_reward": 0.11830357555299997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4436384215950966,
      "step": 2458
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1768.9732971191406,
      "epoch": 0.7345231872152939,
      "grad_norm": 12.978499412536621,
      "kl": 3.859375,
      "learning_rate": 9.978236513777421e-08,
      "loss": 0.2077,
      "reward": 0.5117187574505806,
      "reward_std": 0.17660164088010788,
      "rewards/accuracy_reward": 0.08928571944124997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330559372902,
      "step": 2459
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1711.8282165527344,
      "epoch": 0.7348218953028153,
      "grad_norm": 8.386109352111816,
      "kl": 2.7734375,
      "learning_rate": 9.957401223888156e-08,
      "loss": 0.1661,
      "reward": 0.5401785969734192,
      "reward_std": 0.12458506878465414,
      "rewards/accuracy_reward": 0.0937500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4464285895228386,
      "step": 2460
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1736.68310546875,
      "epoch": 0.7351206033903368,
      "grad_norm": 7.0329813957214355,
      "kl": 4.103515625,
      "learning_rate": 9.936582298872023e-08,
      "loss": 0.243,
      "reward": 0.6149553805589676,
      "reward_std": 0.13996917940676212,
      "rewards/accuracy_reward": 0.1852678619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875149011612,
      "step": 2461
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1843.3639221191406,
      "epoch": 0.7354193114778582,
      "grad_norm": 7.624131679534912,
      "kl": 4.20703125,
      "learning_rate": 9.915779761377987e-08,
      "loss": 0.2334,
      "reward": 0.4838169738650322,
      "reward_std": 0.13326245918869972,
      "rewards/accuracy_reward": 0.05580357275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4280134066939354,
      "step": 2462
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1703.0246276855469,
      "epoch": 0.7357180195653797,
      "grad_norm": 9.913413047790527,
      "kl": 3.69921875,
      "learning_rate": 9.894993634037147e-08,
      "loss": 0.2067,
      "reward": 0.6735491454601288,
      "reward_std": 0.18679275922477245,
      "rewards/accuracy_reward": 0.2433035783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455559372902,
      "step": 2463
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1805.6942749023438,
      "epoch": 0.7360167276529012,
      "grad_norm": 22.81328773498535,
      "kl": 4.88671875,
      "learning_rate": 9.874223939462784e-08,
      "loss": 0.2542,
      "reward": 0.4642857387661934,
      "reward_std": 0.1239042580127716,
      "rewards/accuracy_reward": 0.0558035746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.408482164144516,
      "step": 2464
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1799.2880249023438,
      "epoch": 0.7363154357404227,
      "grad_norm": 21.951324462890625,
      "kl": 2.96875,
      "learning_rate": 9.8534707002503e-08,
      "loss": 0.1756,
      "reward": 0.4492187723517418,
      "reward_std": 0.13758663088083267,
      "rewards/accuracy_reward": 0.02008928661234677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294887661934,
      "step": 2465
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1827.83935546875,
      "epoch": 0.7366141438279441,
      "grad_norm": 5.277919769287109,
      "kl": 3.609375,
      "learning_rate": 9.832733938977164e-08,
      "loss": 0.2072,
      "reward": 0.451450914144516,
      "reward_std": 0.1796775683760643,
      "rewards/accuracy_reward": 0.04017857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.411272332072258,
      "step": 2466
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1834.5626220703125,
      "epoch": 0.7369128519154656,
      "grad_norm": 20.796598434448242,
      "kl": 2.9375,
      "learning_rate": 9.812013678202957e-08,
      "loss": 0.1798,
      "reward": 0.6054687798023224,
      "reward_std": 0.11634495109319687,
      "rewards/accuracy_reward": 0.1830357238650322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330633878708,
      "step": 2467
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1768.2813415527344,
      "epoch": 0.737211560002987,
      "grad_norm": 15.44597053527832,
      "kl": 3.49609375,
      "learning_rate": 9.791309940469281e-08,
      "loss": 0.2138,
      "reward": 0.4213169813156128,
      "reward_std": 0.12068344466388226,
      "rewards/accuracy_reward": 0.004464285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4168526977300644,
      "step": 2468
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1773.6764221191406,
      "epoch": 0.7375102680905086,
      "grad_norm": 6.350347518920898,
      "kl": 3.2734375,
      "learning_rate": 9.770622748299775e-08,
      "loss": 0.1928,
      "reward": 0.5440848395228386,
      "reward_std": 0.16485394537448883,
      "rewards/accuracy_reward": 0.10937500558793545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098395228386,
      "step": 2469
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1715.4666137695312,
      "epoch": 0.73780897617803,
      "grad_norm": 6.997522354125977,
      "kl": 3.44140625,
      "learning_rate": 9.749952124200089e-08,
      "loss": 0.2275,
      "reward": 0.5697544738650322,
      "reward_std": 0.14852512441575527,
      "rewards/accuracy_reward": 0.13169643515720963,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4380580559372902,
      "step": 2470
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1712.7389221191406,
      "epoch": 0.7381076842655515,
      "grad_norm": 7.35548734664917,
      "kl": 3.73046875,
      "learning_rate": 9.72929809065782e-08,
      "loss": 0.2412,
      "reward": 0.5446428954601288,
      "reward_std": 0.16495750844478607,
      "rewards/accuracy_reward": 0.1026785746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4419643059372902,
      "step": 2471
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1725.716552734375,
      "epoch": 0.7384063923530729,
      "grad_norm": 43.503604888916016,
      "kl": 5.53515625,
      "learning_rate": 9.708660670142538e-08,
      "loss": 0.2927,
      "reward": 0.6222098395228386,
      "reward_std": 0.15264825522899628,
      "rewards/accuracy_reward": 0.1897321529686451,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776977300644,
      "step": 2472
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1775.7545471191406,
      "epoch": 0.7387051004405945,
      "grad_norm": 23.964923858642578,
      "kl": 3.30078125,
      "learning_rate": 9.688039885105742e-08,
      "loss": 0.1757,
      "reward": 0.5736607313156128,
      "reward_std": 0.16235435381531715,
      "rewards/accuracy_reward": 0.12276786053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4508928805589676,
      "step": 2473
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1817.0045776367188,
      "epoch": 0.7390038085281159,
      "grad_norm": 42.71257400512695,
      "kl": 6.9140625,
      "learning_rate": 9.667435757980808e-08,
      "loss": 0.3571,
      "reward": 0.5312500149011612,
      "reward_std": 0.13582192361354828,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500223517418,
      "step": 2474
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1745.5670471191406,
      "epoch": 0.7393025166156374,
      "grad_norm": 22.660520553588867,
      "kl": 4.81640625,
      "learning_rate": 9.646848311183018e-08,
      "loss": 0.2645,
      "reward": 0.4810268133878708,
      "reward_std": 0.16368885710835457,
      "rewards/accuracy_reward": 0.06473214621655643,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.416294664144516,
      "step": 2475
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1787.0781860351562,
      "epoch": 0.7396012247031588,
      "grad_norm": 11.5211820602417,
      "kl": 3.203125,
      "learning_rate": 9.626277567109503e-08,
      "loss": 0.1864,
      "reward": 0.4520089477300644,
      "reward_std": 0.15423032455146313,
      "rewards/accuracy_reward": 0.033482145285233855,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268059372902,
      "step": 2476
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1703.2880249023438,
      "epoch": 0.7398999327906803,
      "grad_norm": 47.9050407409668,
      "kl": 1.58203125,
      "learning_rate": 9.6057235481392e-08,
      "loss": 0.1335,
      "reward": 0.5468750223517418,
      "reward_std": 0.20343011990189552,
      "rewards/accuracy_reward": 0.0959821455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4508928805589676,
      "step": 2477
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1760.9063415527344,
      "epoch": 0.7401986408782018,
      "grad_norm": 43.88084030151367,
      "kl": 2.48046875,
      "learning_rate": 9.585186276632875e-08,
      "loss": 0.1498,
      "reward": 0.486049123108387,
      "reward_std": 0.1505572684109211,
      "rewards/accuracy_reward": 0.0535714328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.432477705180645,
      "step": 2478
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1787.3750915527344,
      "epoch": 0.7404973489657233,
      "grad_norm": 12116.1376953125,
      "kl": 23.037109375,
      "learning_rate": 9.564665774933075e-08,
      "loss": 0.9966,
      "reward": 0.5362723395228386,
      "reward_std": 0.21773753687739372,
      "rewards/accuracy_reward": 0.11830357648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4179687723517418,
      "step": 2479
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1781.1697387695312,
      "epoch": 0.7407960570532447,
      "grad_norm": 10.233570098876953,
      "kl": 3.302734375,
      "learning_rate": 9.54416206536408e-08,
      "loss": 0.1815,
      "reward": 0.509486623108387,
      "reward_std": 0.138621149584651,
      "rewards/accuracy_reward": 0.08035714412108064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294813156128,
      "step": 2480
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1835.0357971191406,
      "epoch": 0.7410947651407662,
      "grad_norm": 15.677043914794922,
      "kl": 2.337890625,
      "learning_rate": 9.523675170231929e-08,
      "loss": 0.1415,
      "reward": 0.6434151977300644,
      "reward_std": 0.22734371572732925,
      "rewards/accuracy_reward": 0.20089286752045155,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4425223395228386,
      "step": 2481
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1766.77685546875,
      "epoch": 0.7413934732282876,
      "grad_norm": 16.84552574157715,
      "kl": 3.46875,
      "learning_rate": 9.503205111824344e-08,
      "loss": 0.2271,
      "reward": 0.5837053805589676,
      "reward_std": 0.16218005679547787,
      "rewards/accuracy_reward": 0.14732143469154835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4363839477300644,
      "step": 2482
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1785.1295776367188,
      "epoch": 0.7416921813158092,
      "grad_norm": 9.836202621459961,
      "kl": 4.35546875,
      "learning_rate": 9.482751912410748e-08,
      "loss": 0.2488,
      "reward": 0.501116082072258,
      "reward_std": 0.09269302058964968,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875149011612,
      "step": 2483
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1730.0782165527344,
      "epoch": 0.7419908894033306,
      "grad_norm": 6.52803373336792,
      "kl": 3.138671875,
      "learning_rate": 9.462315594242223e-08,
      "loss": 0.1921,
      "reward": 0.5424107313156128,
      "reward_std": 0.15229442343115807,
      "rewards/accuracy_reward": 0.1004464328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4419643059372902,
      "step": 2484
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1847.9755554199219,
      "epoch": 0.7422895974908521,
      "grad_norm": 24.102272033691406,
      "kl": 4.83984375,
      "learning_rate": 9.441896179551467e-08,
      "loss": 0.2496,
      "reward": 0.4492187723517418,
      "reward_std": 0.15179810300469398,
      "rewards/accuracy_reward": 0.02901785750873387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.420200914144516,
      "step": 2485
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1797.5536499023438,
      "epoch": 0.7425883055783735,
      "grad_norm": 35.3285026550293,
      "kl": 4.9609375,
      "learning_rate": 9.421493690552812e-08,
      "loss": 0.2487,
      "reward": 0.4832589477300644,
      "reward_std": 0.1297183446586132,
      "rewards/accuracy_reward": 0.06919643143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625223517418,
      "step": 2486
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1855.9331665039062,
      "epoch": 0.742887013665895,
      "grad_norm": 8.837518692016602,
      "kl": 4.29296875,
      "learning_rate": 9.401108149442172e-08,
      "loss": 0.2242,
      "reward": 0.474330373108387,
      "reward_std": 0.1323937438428402,
      "rewards/accuracy_reward": 0.0558035746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185267984867096,
      "step": 2487
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1813.4331359863281,
      "epoch": 0.7431857217534165,
      "grad_norm": 5.106071472167969,
      "kl": 3.87109375,
      "learning_rate": 9.380739578396999e-08,
      "loss": 0.2177,
      "reward": 0.495535746216774,
      "reward_std": 0.16601544246077538,
      "rewards/accuracy_reward": 0.07142857322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4241071566939354,
      "step": 2488
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1793.4800109863281,
      "epoch": 0.743484429840938,
      "grad_norm": 33.44401550292969,
      "kl": 2.517578125,
      "learning_rate": 9.360387999576314e-08,
      "loss": 0.1951,
      "reward": 0.5479910895228386,
      "reward_std": 0.14646285399794579,
      "rewards/accuracy_reward": 0.12500000605359674,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.422991082072258,
      "step": 2489
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1834.5179443359375,
      "epoch": 0.7437831379284594,
      "grad_norm": 9.561552047729492,
      "kl": 3.609375,
      "learning_rate": 9.340053435120648e-08,
      "loss": 0.202,
      "reward": 0.4609375074505806,
      "reward_std": 0.1355385221540928,
      "rewards/accuracy_reward": 0.05357143026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.407366082072258,
      "step": 2490
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1838.0759887695312,
      "epoch": 0.7440818460159809,
      "grad_norm": 27.870573043823242,
      "kl": 2.29296875,
      "learning_rate": 9.319735907151996e-08,
      "loss": 0.1288,
      "reward": 0.514508955180645,
      "reward_std": 0.1199816819280386,
      "rewards/accuracy_reward": 0.082589291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4319196715950966,
      "step": 2491
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1829.4018249511719,
      "epoch": 0.7443805541035023,
      "grad_norm": 8.88051700592041,
      "kl": 3.072265625,
      "learning_rate": 9.299435437773845e-08,
      "loss": 0.1911,
      "reward": 0.5167410969734192,
      "reward_std": 0.14815832115709782,
      "rewards/accuracy_reward": 0.10267857578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625149011612,
      "step": 2492
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1785.5514526367188,
      "epoch": 0.7446792621910239,
      "grad_norm": 22.785863876342773,
      "kl": 3.16015625,
      "learning_rate": 9.279152049071126e-08,
      "loss": 0.2108,
      "reward": 0.4899553656578064,
      "reward_std": 0.1146912183612585,
      "rewards/accuracy_reward": 0.0736607164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4162946566939354,
      "step": 2493
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1722.7210693359375,
      "epoch": 0.7449779702785453,
      "grad_norm": 9.721294403076172,
      "kl": 4.40625,
      "learning_rate": 9.258885763110161e-08,
      "loss": 0.2759,
      "reward": 0.5385044738650322,
      "reward_std": 0.17283786460757256,
      "rewards/accuracy_reward": 0.11607143143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330559372902,
      "step": 2494
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1831.82373046875,
      "epoch": 0.7452766783660668,
      "grad_norm": 22.2852783203125,
      "kl": 4.71875,
      "learning_rate": 9.238636601938699e-08,
      "loss": 0.25,
      "reward": 0.5011161044239998,
      "reward_std": 0.14202029816806316,
      "rewards/accuracy_reward": 0.0870535746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625149011612,
      "step": 2495
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1735.7188110351562,
      "epoch": 0.7455753864535882,
      "grad_norm": 9.159541130065918,
      "kl": 4.0546875,
      "learning_rate": 9.218404587585824e-08,
      "loss": 0.2393,
      "reward": 0.549107164144516,
      "reward_std": 0.15040829591453075,
      "rewards/accuracy_reward": 0.1272321492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 2496
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.4487609863281,
      "epoch": 0.7458740945411098,
      "grad_norm": 4.916046142578125,
      "kl": 3.048828125,
      "learning_rate": 9.198189742061996e-08,
      "loss": 0.172,
      "reward": 0.502232164144516,
      "reward_std": 0.14042324759066105,
      "rewards/accuracy_reward": 0.06696428847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435267873108387,
      "step": 2497
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1760.1139526367188,
      "epoch": 0.7461728026286312,
      "grad_norm": 9.095718383789062,
      "kl": 4.125,
      "learning_rate": 9.177992087358996e-08,
      "loss": 0.2452,
      "reward": 0.5172991305589676,
      "reward_std": 0.1437222994863987,
      "rewards/accuracy_reward": 0.1049107201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.412388414144516,
      "step": 2498
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1700.16748046875,
      "epoch": 0.7464715107161526,
      "grad_norm": 22.993492126464844,
      "kl": 2.498046875,
      "learning_rate": 9.15781164544987e-08,
      "loss": 0.1606,
      "reward": 0.593191996216774,
      "reward_std": 0.17209437116980553,
      "rewards/accuracy_reward": 0.1517857238650322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4414062649011612,
      "step": 2499
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1814.8505554199219,
      "epoch": 0.7467702188036741,
      "grad_norm": 14.00835132598877,
      "kl": 3.890625,
      "learning_rate": 9.137648438288972e-08,
      "loss": 0.1909,
      "reward": 0.5117187723517418,
      "reward_std": 0.23183442652225494,
      "rewards/accuracy_reward": 0.08928571757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330559372902,
      "step": 2500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1826.1384887695312,
      "epoch": 0.7470689268911955,
      "grad_norm": 15.795931816101074,
      "kl": 2.62890625,
      "learning_rate": 9.117502487811907e-08,
      "loss": 0.1518,
      "reward": 0.4882812723517418,
      "reward_std": 0.1846756376326084,
      "rewards/accuracy_reward": 0.06250000325962901,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812723517418,
      "step": 2501
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1696.4486999511719,
      "epoch": 0.747367634978717,
      "grad_norm": 16.167802810668945,
      "kl": 2.875,
      "learning_rate": 9.097373815935472e-08,
      "loss": 0.1732,
      "reward": 0.4966518059372902,
      "reward_std": 0.1530086062848568,
      "rewards/accuracy_reward": 0.07142857648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232313156128,
      "step": 2502
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1757.5804138183594,
      "epoch": 0.7476663430662385,
      "grad_norm": 21.651351928710938,
      "kl": 2.45703125,
      "learning_rate": 9.077262444557704e-08,
      "loss": 0.1477,
      "reward": 0.537946455180645,
      "reward_std": 0.16259736940264702,
      "rewards/accuracy_reward": 0.1071428582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035969734192,
      "step": 2503
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1814.60498046875,
      "epoch": 0.74796505115376,
      "grad_norm": 7.348222732543945,
      "kl": 3.5,
      "learning_rate": 9.057168395557805e-08,
      "loss": 0.1798,
      "reward": 0.474330373108387,
      "reward_std": 0.1610089372843504,
      "rewards/accuracy_reward": 0.049107145983725786,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232313156128,
      "step": 2504
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1809.9375610351562,
      "epoch": 0.7482637592412814,
      "grad_norm": 14.76254940032959,
      "kl": 3.16015625,
      "learning_rate": 9.03709169079612e-08,
      "loss": 0.1818,
      "reward": 0.4648437723517418,
      "reward_std": 0.15371510200202465,
      "rewards/accuracy_reward": 0.03794643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4268973469734192,
      "step": 2505
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1785.1005249023438,
      "epoch": 0.7485624673288029,
      "grad_norm": 5.664505958557129,
      "kl": 4.015625,
      "learning_rate": 9.01703235211414e-08,
      "loss": 0.2246,
      "reward": 0.462611623108387,
      "reward_std": 0.1539636105298996,
      "rewards/accuracy_reward": 0.04464285867288709,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4179687649011612,
      "step": 2506
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1801.2143859863281,
      "epoch": 0.7488611754163244,
      "grad_norm": 5.6104607582092285,
      "kl": 3.61328125,
      "learning_rate": 8.996990401334473e-08,
      "loss": 0.1938,
      "reward": 0.4380580559372902,
      "reward_std": 0.13196823187172413,
      "rewards/accuracy_reward": 0.020089286379516125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4179687649011612,
      "step": 2507
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1664.4822387695312,
      "epoch": 0.7491598835038459,
      "grad_norm": 10.017193794250488,
      "kl": 2.720703125,
      "learning_rate": 8.976965860260777e-08,
      "loss": 0.1816,
      "reward": 0.5775669813156128,
      "reward_std": 0.18173165619373322,
      "rewards/accuracy_reward": 0.13169643841683865,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4458705559372902,
      "step": 2508
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1721.05810546875,
      "epoch": 0.7494585915913673,
      "grad_norm": 16.97480010986328,
      "kl": 2.900390625,
      "learning_rate": 8.956958750677804e-08,
      "loss": 0.1879,
      "reward": 0.4765625149011612,
      "reward_std": 0.1632473971694708,
      "rewards/accuracy_reward": 0.03571428661234677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4408482387661934,
      "step": 2509
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1826.3572082519531,
      "epoch": 0.7497572996788888,
      "grad_norm": 5.590494155883789,
      "kl": 4.48046875,
      "learning_rate": 8.936969094351318e-08,
      "loss": 0.2445,
      "reward": 0.4531250223517418,
      "reward_std": 0.1315528191626072,
      "rewards/accuracy_reward": 0.046875000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500149011612,
      "step": 2510
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1706.0134887695312,
      "epoch": 0.7500560077664102,
      "grad_norm": 9.51931381225586,
      "kl": 3.36328125,
      "learning_rate": 8.916996913028111e-08,
      "loss": 0.2182,
      "reward": 0.5412946790456772,
      "reward_std": 0.16308420337736607,
      "rewards/accuracy_reward": 0.09598214668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125149011612,
      "step": 2511
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1731.274658203125,
      "epoch": 0.7503547158539318,
      "grad_norm": 27.894014358520508,
      "kl": 5.06640625,
      "learning_rate": 8.897042228435966e-08,
      "loss": 0.2963,
      "reward": 0.5708705633878708,
      "reward_std": 0.14316944405436516,
      "rewards/accuracy_reward": 0.15848214738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.412388414144516,
      "step": 2512
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1665.7701416015625,
      "epoch": 0.7506534239414532,
      "grad_norm": 6.558772087097168,
      "kl": 4.0703125,
      "learning_rate": 8.877105062283608e-08,
      "loss": 0.2457,
      "reward": 0.5546875298023224,
      "reward_std": 0.14785056747496128,
      "rewards/accuracy_reward": 0.11160715017467737,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4430803805589676,
      "step": 2513
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.9375610351562,
      "epoch": 0.7509521320289747,
      "grad_norm": 56.47796630859375,
      "kl": 6.29296875,
      "learning_rate": 8.857185436260731e-08,
      "loss": 0.3165,
      "reward": 0.4726562723517418,
      "reward_std": 0.12784571200609207,
      "rewards/accuracy_reward": 0.05803571757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205559372902,
      "step": 2514
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1806.4621276855469,
      "epoch": 0.7512508401164961,
      "grad_norm": 16.063987731933594,
      "kl": 4.48828125,
      "learning_rate": 8.83728337203794e-08,
      "loss": 0.2585,
      "reward": 0.4977678805589676,
      "reward_std": 0.18256431072950363,
      "rewards/accuracy_reward": 0.08258929196745157,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4151785969734192,
      "step": 2515
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1686.83935546875,
      "epoch": 0.7515495482040176,
      "grad_norm": 6.521564960479736,
      "kl": 3.87109375,
      "learning_rate": 8.817398891266716e-08,
      "loss": 0.2172,
      "reward": 0.4508928805589676,
      "reward_std": 0.15058153495192528,
      "rewards/accuracy_reward": 0.02455357275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263392984867096,
      "step": 2516
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1746.0782165527344,
      "epoch": 0.7518482562915391,
      "grad_norm": 21.691272735595703,
      "kl": 1.21484375,
      "learning_rate": 8.797532015579435e-08,
      "loss": 0.1006,
      "reward": 0.5368303805589676,
      "reward_std": 0.14051378518342972,
      "rewards/accuracy_reward": 0.08482143213041127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.452008955180645,
      "step": 2517
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1773.32373046875,
      "epoch": 0.7521469643790606,
      "grad_norm": 22.037166595458984,
      "kl": 2.728515625,
      "learning_rate": 8.777682766589314e-08,
      "loss": 0.1747,
      "reward": 0.5597098544239998,
      "reward_std": 0.18050232902169228,
      "rewards/accuracy_reward": 0.1316964291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4280134215950966,
      "step": 2518
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1881.8237609863281,
      "epoch": 0.752445672466582,
      "grad_norm": 21.99489974975586,
      "kl": 2.462890625,
      "learning_rate": 8.757851165890376e-08,
      "loss": 0.1442,
      "reward": 0.473772332072258,
      "reward_std": 0.12414992041885853,
      "rewards/accuracy_reward": 0.04687500232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.426897332072258,
      "step": 2519
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1782.3773193359375,
      "epoch": 0.7527443805541035,
      "grad_norm": 9.935748100280762,
      "kl": 3.1796875,
      "learning_rate": 8.738037235057477e-08,
      "loss": 0.2038,
      "reward": 0.503348246216774,
      "reward_std": 0.1278661135584116,
      "rewards/accuracy_reward": 0.08482143236324191,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268059372902,
      "step": 2520
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1759.0536499023438,
      "epoch": 0.753043088641625,
      "grad_norm": 13.794991493225098,
      "kl": 3.9453125,
      "learning_rate": 8.718240995646217e-08,
      "loss": 0.256,
      "reward": 0.4687500223517418,
      "reward_std": 0.15572605654597282,
      "rewards/accuracy_reward": 0.05357143236324191,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4151785895228386,
      "step": 2521
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1799.4911804199219,
      "epoch": 0.7533417967291465,
      "grad_norm": 4.69957971572876,
      "kl": 3.5390625,
      "learning_rate": 8.698462469192965e-08,
      "loss": 0.1799,
      "reward": 0.4983259215950966,
      "reward_std": 0.19922034069895744,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435825914144516,
      "step": 2522
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.7768859863281,
      "epoch": 0.7536405048166679,
      "grad_norm": 10.544771194458008,
      "kl": 3.453125,
      "learning_rate": 8.678701677214837e-08,
      "loss": 0.1988,
      "reward": 0.5368303805589676,
      "reward_std": 0.203833669424057,
      "rewards/accuracy_reward": 0.10267857648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341518059372902,
      "step": 2523
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1867.1786499023438,
      "epoch": 0.7539392129041894,
      "grad_norm": 24.625850677490234,
      "kl": 5.01171875,
      "learning_rate": 8.658958641209616e-08,
      "loss": 0.2479,
      "reward": 0.4614955633878708,
      "reward_std": 0.12141005881130695,
      "rewards/accuracy_reward": 0.0513392873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4101562723517418,
      "step": 2524
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1772.3081359863281,
      "epoch": 0.7542379209917108,
      "grad_norm": 6.263752460479736,
      "kl": 3.361328125,
      "learning_rate": 8.639233382655798e-08,
      "loss": 0.1929,
      "reward": 0.5000000223517418,
      "reward_std": 0.1610074806958437,
      "rewards/accuracy_reward": 0.07812500279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 2525
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1824.2255249023438,
      "epoch": 0.7545366290792324,
      "grad_norm": 22.25933074951172,
      "kl": 2.689453125,
      "learning_rate": 8.619525923012542e-08,
      "loss": 0.1672,
      "reward": 0.468191996216774,
      "reward_std": 0.14204774051904678,
      "rewards/accuracy_reward": 0.051339288242161274,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4168526977300644,
      "step": 2526
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1712.0469665527344,
      "epoch": 0.7548353371667538,
      "grad_norm": 29.006227493286133,
      "kl": 3.01171875,
      "learning_rate": 8.59983628371961e-08,
      "loss": 0.1992,
      "reward": 0.6941964626312256,
      "reward_std": 0.1673908270895481,
      "rewards/accuracy_reward": 0.2633928656578064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035969734192,
      "step": 2527
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1713.0625915527344,
      "epoch": 0.7551340452542753,
      "grad_norm": 15.818685531616211,
      "kl": 3.248046875,
      "learning_rate": 8.580164486197417e-08,
      "loss": 0.2141,
      "reward": 0.553013414144516,
      "reward_std": 0.14394951052963734,
      "rewards/accuracy_reward": 0.12500000558793545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428013414144516,
      "step": 2528
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1795.2523193359375,
      "epoch": 0.7554327533417967,
      "grad_norm": 10.3928861618042,
      "kl": 3.48046875,
      "learning_rate": 8.560510551846953e-08,
      "loss": 0.2159,
      "reward": 0.4827009215950966,
      "reward_std": 0.16248556785285473,
      "rewards/accuracy_reward": 0.06919643096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4135044887661934,
      "step": 2529
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1841.305908203125,
      "epoch": 0.7557314614293182,
      "grad_norm": 10.75475788116455,
      "kl": 4.267578125,
      "learning_rate": 8.540874502049758e-08,
      "loss": 0.2309,
      "reward": 0.4882812798023224,
      "reward_std": 0.15325036644935608,
      "rewards/accuracy_reward": 0.07142857369035482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4168526977300644,
      "step": 2530
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1786.5603332519531,
      "epoch": 0.7560301695168397,
      "grad_norm": 5.4561991691589355,
      "kl": 3.59375,
      "learning_rate": 8.521256358167936e-08,
      "loss": 0.2042,
      "reward": 0.4799107313156128,
      "reward_std": 0.1281038261950016,
      "rewards/accuracy_reward": 0.06250000232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4174107313156128,
      "step": 2531
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1762.7054443359375,
      "epoch": 0.7563288776043612,
      "grad_norm": 7.350508689880371,
      "kl": 2.31640625,
      "learning_rate": 8.501656141544113e-08,
      "loss": 0.1519,
      "reward": 0.490513414144516,
      "reward_std": 0.15046704933047295,
      "rewards/accuracy_reward": 0.044642857974395156,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4458705484867096,
      "step": 2532
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.0759887695312,
      "epoch": 0.7566275856918826,
      "grad_norm": 13.729381561279297,
      "kl": 3.3515625,
      "learning_rate": 8.482073873501392e-08,
      "loss": 0.1805,
      "reward": 0.5959821715950966,
      "reward_std": 0.15152987651526928,
      "rewards/accuracy_reward": 0.16294643771834671,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357313156128,
      "step": 2533
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1786.852783203125,
      "epoch": 0.7569262937794041,
      "grad_norm": 6.062173843383789,
      "kl": 2.94140625,
      "learning_rate": 8.462509575343371e-08,
      "loss": 0.1535,
      "reward": 0.4927455708384514,
      "reward_std": 0.18924615904688835,
      "rewards/accuracy_reward": 0.06026785937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.432477705180645,
      "step": 2534
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1739.4197082519531,
      "epoch": 0.7572250018669255,
      "grad_norm": 5.073615550994873,
      "kl": 3.423828125,
      "learning_rate": 8.442963268354075e-08,
      "loss": 0.2045,
      "reward": 0.5122768059372902,
      "reward_std": 0.12731828913092613,
      "rewards/accuracy_reward": 0.08258928917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875223517418,
      "step": 2535
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1804.4107971191406,
      "epoch": 0.7575237099544471,
      "grad_norm": 17.259685516357422,
      "kl": 4.8203125,
      "learning_rate": 8.423434973797983e-08,
      "loss": 0.2555,
      "reward": 0.4972098469734192,
      "reward_std": 0.19531960040330887,
      "rewards/accuracy_reward": 0.08928571874275804,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.407924123108387,
      "step": 2536
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1785.587158203125,
      "epoch": 0.7578224180419685,
      "grad_norm": 17.887813568115234,
      "kl": 2.78125,
      "learning_rate": 8.403924712919965e-08,
      "loss": 0.1755,
      "reward": 0.4559151977300644,
      "reward_std": 0.1422435138374567,
      "rewards/accuracy_reward": 0.03348214388824999,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330559372902,
      "step": 2537
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1731.8304138183594,
      "epoch": 0.75812112612949,
      "grad_norm": 12.730782508850098,
      "kl": 2.296875,
      "learning_rate": 8.384432506945263e-08,
      "loss": 0.1421,
      "reward": 0.5680803805589676,
      "reward_std": 0.19015506654977798,
      "rewards/accuracy_reward": 0.1272321529686451,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4408482387661934,
      "step": 2538
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1852.6273193359375,
      "epoch": 0.7584198342170114,
      "grad_norm": 14.77025318145752,
      "kl": 3.02734375,
      "learning_rate": 8.364958377079495e-08,
      "loss": 0.1823,
      "reward": 0.5133928805589676,
      "reward_std": 0.19640015810728073,
      "rewards/accuracy_reward": 0.0937500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4196428805589676,
      "step": 2539
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1777.5715026855469,
      "epoch": 0.758718542304533,
      "grad_norm": 18.738475799560547,
      "kl": 2.6484375,
      "learning_rate": 8.34550234450861e-08,
      "loss": 0.1773,
      "reward": 0.549107164144516,
      "reward_std": 0.19636374339461327,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4241071566939354,
      "step": 2540
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1849.0045471191406,
      "epoch": 0.7590172503920544,
      "grad_norm": 5.070845127105713,
      "kl": 3.35546875,
      "learning_rate": 8.326064430398849e-08,
      "loss": 0.2025,
      "reward": 0.4804687723517418,
      "reward_std": 0.15960625559091568,
      "rewards/accuracy_reward": 0.07366071827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4068080559372902,
      "step": 2541
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1717.2478332519531,
      "epoch": 0.7593159584795758,
      "grad_norm": 4.99845027923584,
      "kl": 2.58203125,
      "learning_rate": 8.306644655896766e-08,
      "loss": 0.164,
      "reward": 0.6328125298023224,
      "reward_std": 0.2145119234919548,
      "rewards/accuracy_reward": 0.19419643841683865,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4386160969734192,
      "step": 2542
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1816.9487609863281,
      "epoch": 0.7596146665670973,
      "grad_norm": 11.495452880859375,
      "kl": 3.23828125,
      "learning_rate": 8.287243042129182e-08,
      "loss": 0.1936,
      "reward": 0.5050223395228386,
      "reward_std": 0.13667232356965542,
      "rewards/accuracy_reward": 0.06473214668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.440290205180645,
      "step": 2543
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1780.2009582519531,
      "epoch": 0.7599133746546187,
      "grad_norm": 199.31536865234375,
      "kl": 6.16015625,
      "learning_rate": 8.267859610203131e-08,
      "loss": 0.3194,
      "reward": 0.4681919887661934,
      "reward_std": 0.1352878287434578,
      "rewards/accuracy_reward": 0.04017857206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428013414144516,
      "step": 2544
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1744.8504943847656,
      "epoch": 0.7602120827421402,
      "grad_norm": 40.7397346496582,
      "kl": 5.7421875,
      "learning_rate": 8.248494381205903e-08,
      "loss": 0.2943,
      "reward": 0.5318080484867096,
      "reward_std": 0.17522741854190826,
      "rewards/accuracy_reward": 0.11383929383009672,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4179687649011612,
      "step": 2545
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1744.3817443847656,
      "epoch": 0.7605107908296617,
      "grad_norm": 28.998428344726562,
      "kl": 4.8046875,
      "learning_rate": 8.22914737620497e-08,
      "loss": 0.2782,
      "reward": 0.534598246216774,
      "reward_std": 0.13515443168580532,
      "rewards/accuracy_reward": 0.10044643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341518133878708,
      "step": 2546
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1806.4665832519531,
      "epoch": 0.7608094989171832,
      "grad_norm": 42.94977569580078,
      "kl": 5.84375,
      "learning_rate": 8.209818616247963e-08,
      "loss": 0.3211,
      "reward": 0.5864955559372902,
      "reward_std": 0.15437506884336472,
      "rewards/accuracy_reward": 0.16741071874275804,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4190848395228386,
      "step": 2547
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1772.33935546875,
      "epoch": 0.7611082070047046,
      "grad_norm": 10.259172439575195,
      "kl": 3.66796875,
      "learning_rate": 8.190508122362694e-08,
      "loss": 0.2251,
      "reward": 0.5507812649011612,
      "reward_std": 0.12391670420765877,
      "rewards/accuracy_reward": 0.1183035746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776977300644,
      "step": 2548
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1687.5447082519531,
      "epoch": 0.7614069150922261,
      "grad_norm": 8.613177299499512,
      "kl": 3.41015625,
      "learning_rate": 8.171215915557075e-08,
      "loss": 0.2253,
      "reward": 0.627232164144516,
      "reward_std": 0.22335196658968925,
      "rewards/accuracy_reward": 0.1808035783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4464285969734192,
      "step": 2549
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1787.3505554199219,
      "epoch": 0.7617056231797475,
      "grad_norm": 7.240253448486328,
      "kl": 3.7578125,
      "learning_rate": 8.151942016819141e-08,
      "loss": 0.2202,
      "reward": 0.4464285969734192,
      "reward_std": 0.11437452584505081,
      "rewards/accuracy_reward": 0.008928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000149011612,
      "step": 2550
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1809.8817749023438,
      "epoch": 0.7620043312672691,
      "grad_norm": 13.29697036743164,
      "kl": 3.26171875,
      "learning_rate": 8.132686447117013e-08,
      "loss": 0.187,
      "reward": 0.5184151902794838,
      "reward_std": 0.16947563365101814,
      "rewards/accuracy_reward": 0.09151786169968545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4268973395228386,
      "step": 2551
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1774.5112609863281,
      "epoch": 0.7623030393547905,
      "grad_norm": 9.201582908630371,
      "kl": 4.15625,
      "learning_rate": 8.113449227398847e-08,
      "loss": 0.2491,
      "reward": 0.4637277126312256,
      "reward_std": 0.15675169229507446,
      "rewards/accuracy_reward": 0.053571431431919336,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4101562723517418,
      "step": 2552
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1826.8661804199219,
      "epoch": 0.762601747442312,
      "grad_norm": 36.36302947998047,
      "kl": 5.66796875,
      "learning_rate": 8.094230378592865e-08,
      "loss": 0.3073,
      "reward": 0.4525669887661934,
      "reward_std": 0.15038013085722923,
      "rewards/accuracy_reward": 0.05803571850992739,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3945312649011612,
      "step": 2553
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1825.9465026855469,
      "epoch": 0.7629004555298334,
      "grad_norm": 26.15264129638672,
      "kl": 2.6015625,
      "learning_rate": 8.075029921607293e-08,
      "loss": 0.1607,
      "reward": 0.4492187723517418,
      "reward_std": 0.14164195954799652,
      "rewards/accuracy_reward": 0.02455357206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.424665205180645,
      "step": 2554
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1802.8415832519531,
      "epoch": 0.763199163617355,
      "grad_norm": 16.039710998535156,
      "kl": 3.78125,
      "learning_rate": 8.055847877330332e-08,
      "loss": 0.2149,
      "reward": 0.5284598469734192,
      "reward_std": 0.12426152639091015,
      "rewards/accuracy_reward": 0.11830357694998384,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4101562649011612,
      "step": 2555
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1782.1317749023438,
      "epoch": 0.7634978717048764,
      "grad_norm": 32.790618896484375,
      "kl": 2.396484375,
      "learning_rate": 8.036684266630176e-08,
      "loss": 0.1668,
      "reward": 0.5083705708384514,
      "reward_std": 0.149326890707016,
      "rewards/accuracy_reward": 0.0803571455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428013414144516,
      "step": 2556
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1777.6697082519531,
      "epoch": 0.7637965797923979,
      "grad_norm": 15.758880615234375,
      "kl": 3.53515625,
      "learning_rate": 8.017539110354965e-08,
      "loss": 0.2194,
      "reward": 0.483258955180645,
      "reward_std": 0.1491873934864998,
      "rewards/accuracy_reward": 0.0625000016298145,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4207589477300644,
      "step": 2557
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1768.8058776855469,
      "epoch": 0.7640952878799193,
      "grad_norm": 30.996421813964844,
      "kl": 3.43359375,
      "learning_rate": 7.99841242933273e-08,
      "loss": 0.1947,
      "reward": 0.5887276977300644,
      "reward_std": 0.1620193738490343,
      "rewards/accuracy_reward": 0.15625000861473382,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776977300644,
      "step": 2558
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1755.1429138183594,
      "epoch": 0.7643939959674408,
      "grad_norm": 16.56763458251953,
      "kl": 2.830078125,
      "learning_rate": 7.979304244371441e-08,
      "loss": 0.1736,
      "reward": 0.474888414144516,
      "reward_std": 0.14932403899729252,
      "rewards/accuracy_reward": 0.042410715483129025,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776902794838,
      "step": 2559
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1778.8215026855469,
      "epoch": 0.7646927040549623,
      "grad_norm": 5.034185409545898,
      "kl": 3.40234375,
      "learning_rate": 7.96021457625893e-08,
      "loss": 0.2016,
      "reward": 0.5948660969734192,
      "reward_std": 0.17062820494174957,
      "rewards/accuracy_reward": 0.1651785783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875149011612,
      "step": 2560
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1801.3973999023438,
      "epoch": 0.7649914121424838,
      "grad_norm": 27.855850219726562,
      "kl": 5.24609375,
      "learning_rate": 7.941143445762876e-08,
      "loss": 0.274,
      "reward": 0.5117187723517418,
      "reward_std": 0.12441137991845608,
      "rewards/accuracy_reward": 0.09151786053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.420200914144516,
      "step": 2561
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1796.2210693359375,
      "epoch": 0.7652901202300052,
      "grad_norm": 53.55901336669922,
      "kl": 4.9609375,
      "learning_rate": 7.922090873630815e-08,
      "loss": 0.262,
      "reward": 0.4966517984867096,
      "reward_std": 0.14928488247096539,
      "rewards/accuracy_reward": 0.0736607164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 2562
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1798.8036499023438,
      "epoch": 0.7655888283175267,
      "grad_norm": 47.46797561645508,
      "kl": 5.90625,
      "learning_rate": 7.903056880590059e-08,
      "loss": 0.3221,
      "reward": 0.4737723395228386,
      "reward_std": 0.1267513409256935,
      "rewards/accuracy_reward": 0.051339288242161274,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330559372902,
      "step": 2563
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1804.5268859863281,
      "epoch": 0.7658875364050481,
      "grad_norm": 17.308568954467773,
      "kl": 3.91015625,
      "learning_rate": 7.884041487347737e-08,
      "loss": 0.2156,
      "reward": 0.4899553805589676,
      "reward_std": 0.1892781015485525,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.427455373108387,
      "step": 2564
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1805.5223999023438,
      "epoch": 0.7661862444925697,
      "grad_norm": 23.61330223083496,
      "kl": 4.26953125,
      "learning_rate": 7.865044714590738e-08,
      "loss": 0.2447,
      "reward": 0.4698660895228386,
      "reward_std": 0.186822809278965,
      "rewards/accuracy_reward": 0.058035718742758036,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4118303805589676,
      "step": 2565
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1799.5156860351562,
      "epoch": 0.7664849525800911,
      "grad_norm": 7.196094512939453,
      "kl": 3.63671875,
      "learning_rate": 7.846066582985678e-08,
      "loss": 0.1999,
      "reward": 0.4905134215950966,
      "reward_std": 0.12475656718015671,
      "rewards/accuracy_reward": 0.08258928963914514,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.407924123108387,
      "step": 2566
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1778.9375610351562,
      "epoch": 0.7667836606676126,
      "grad_norm": 20.412097930908203,
      "kl": 2.669921875,
      "learning_rate": 7.827107113178911e-08,
      "loss": 0.1643,
      "reward": 0.4983259215950966,
      "reward_std": 0.16344180889427662,
      "rewards/accuracy_reward": 0.07589286216534674,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330559372902,
      "step": 2567
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1793.8460693359375,
      "epoch": 0.767082368755134,
      "grad_norm": 40.36150360107422,
      "kl": 1.609375,
      "learning_rate": 7.808166325796483e-08,
      "loss": 0.0974,
      "reward": 0.5106027126312256,
      "reward_std": 0.09678547270596027,
      "rewards/accuracy_reward": 0.07589286053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098469734192,
      "step": 2568
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1747.3661499023438,
      "epoch": 0.7673810768426556,
      "grad_norm": 44.59108352661133,
      "kl": 1.6181640625,
      "learning_rate": 7.78924424144411e-08,
      "loss": 0.1101,
      "reward": 0.5781250223517418,
      "reward_std": 0.19728682562708855,
      "rewards/accuracy_reward": 0.1473214328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035895228386,
      "step": 2569
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1808.2098999023438,
      "epoch": 0.767679784930177,
      "grad_norm": 40.747589111328125,
      "kl": 2.22265625,
      "learning_rate": 7.770340880707163e-08,
      "loss": 0.144,
      "reward": 0.5223214402794838,
      "reward_std": 0.14578425139188766,
      "rewards/accuracy_reward": 0.10044643469154835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750149011612,
      "step": 2570
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1742.4978637695312,
      "epoch": 0.7679784930176985,
      "grad_norm": 45.561527252197266,
      "kl": 1.576171875,
      "learning_rate": 7.751456264150657e-08,
      "loss": 0.1084,
      "reward": 0.494419664144516,
      "reward_std": 0.1180021520704031,
      "rewards/accuracy_reward": 0.05357143026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4408482387661934,
      "step": 2571
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1809.8572082519531,
      "epoch": 0.7682772011052199,
      "grad_norm": 29.935483932495117,
      "kl": 2.6171875,
      "learning_rate": 7.732590412319193e-08,
      "loss": 0.1552,
      "reward": 0.4771205559372902,
      "reward_std": 0.13989804685115814,
      "rewards/accuracy_reward": 0.05580357322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4213169887661934,
      "step": 2572
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1817.1719360351562,
      "epoch": 0.7685759091927414,
      "grad_norm": 13.055599212646484,
      "kl": 1.8095703125,
      "learning_rate": 7.71374334573697e-08,
      "loss": 0.0987,
      "reward": 0.4760044813156128,
      "reward_std": 0.1261488664895296,
      "rewards/accuracy_reward": 0.037946430034935474,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4380580559372902,
      "step": 2573
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1833.3103637695312,
      "epoch": 0.7688746172802629,
      "grad_norm": 8.414482116699219,
      "kl": 4.111328125,
      "learning_rate": 7.694915084907758e-08,
      "loss": 0.2182,
      "reward": 0.4581473395228386,
      "reward_std": 0.13189874030649662,
      "rewards/accuracy_reward": 0.04687500116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4112723395228386,
      "step": 2574
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1815.3973999023438,
      "epoch": 0.7691733253677844,
      "grad_norm": 7.036265850067139,
      "kl": 4.5859375,
      "learning_rate": 7.676105650314846e-08,
      "loss": 0.2386,
      "reward": 0.5312500298023224,
      "reward_std": 0.10332498699426651,
      "rewards/accuracy_reward": 0.1093750037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 2575
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1784.4777526855469,
      "epoch": 0.7694720334553058,
      "grad_norm": 9.23177433013916,
      "kl": 4.32421875,
      "learning_rate": 7.657315062421071e-08,
      "loss": 0.2426,
      "reward": 0.455915205180645,
      "reward_std": 0.1437736675143242,
      "rewards/accuracy_reward": 0.03794643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4179687723517418,
      "step": 2576
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1788.805908203125,
      "epoch": 0.7697707415428273,
      "grad_norm": 6.710785388946533,
      "kl": 2.85546875,
      "learning_rate": 7.638543341668735e-08,
      "loss": 0.1562,
      "reward": 0.5083705633878708,
      "reward_std": 0.17915258556604385,
      "rewards/accuracy_reward": 0.07812500302679837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455559372902,
      "step": 2577
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1755.0246276855469,
      "epoch": 0.7700694496303487,
      "grad_norm": 10.012096405029297,
      "kl": 3.578125,
      "learning_rate": 7.619790508479637e-08,
      "loss": 0.2144,
      "reward": 0.5223214626312256,
      "reward_std": 0.1272389031946659,
      "rewards/accuracy_reward": 0.10267857206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4196428880095482,
      "step": 2578
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1823.0648193359375,
      "epoch": 0.7703681577178703,
      "grad_norm": 13.80045223236084,
      "kl": 3.34765625,
      "learning_rate": 7.601056583255027e-08,
      "loss": 0.2025,
      "reward": 0.5055803954601288,
      "reward_std": 0.16695699840784073,
      "rewards/accuracy_reward": 0.08035714481957257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232387661934,
      "step": 2579
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1825.1652526855469,
      "epoch": 0.7706668658053917,
      "grad_norm": 10.490732192993164,
      "kl": 2.08984375,
      "learning_rate": 7.582341586375571e-08,
      "loss": 0.1223,
      "reward": 0.5206473469734192,
      "reward_std": 0.14675583317875862,
      "rewards/accuracy_reward": 0.08928571757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4313616305589676,
      "step": 2580
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1766.7701416015625,
      "epoch": 0.7709655738929132,
      "grad_norm": 10.774175643920898,
      "kl": 3.14453125,
      "learning_rate": 7.563645538201355e-08,
      "loss": 0.1871,
      "reward": 0.5864955559372902,
      "reward_std": 0.15112685039639473,
      "rewards/accuracy_reward": 0.15178572200238705,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098469734192,
      "step": 2581
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1810.4978942871094,
      "epoch": 0.7712642819804346,
      "grad_norm": 8.275562286376953,
      "kl": 2.80078125,
      "learning_rate": 7.544968459071851e-08,
      "loss": 0.157,
      "reward": 0.5223214477300644,
      "reward_std": 0.13444994948804379,
      "rewards/accuracy_reward": 0.08928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357313156128,
      "step": 2582
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1811.9710998535156,
      "epoch": 0.7715629900679561,
      "grad_norm": 19.367002487182617,
      "kl": 4.66015625,
      "learning_rate": 7.526310369305883e-08,
      "loss": 0.27,
      "reward": 0.4508928805589676,
      "reward_std": 0.15218351781368256,
      "rewards/accuracy_reward": 0.03125000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.419642873108387,
      "step": 2583
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1807.696533203125,
      "epoch": 0.7718616981554776,
      "grad_norm": 7.166604518890381,
      "kl": 3.140625,
      "learning_rate": 7.507671289201625e-08,
      "loss": 0.1699,
      "reward": 0.4988839477300644,
      "reward_std": 0.161212844774127,
      "rewards/accuracy_reward": 0.07142857578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.427455373108387,
      "step": 2584
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1752.4107971191406,
      "epoch": 0.772160406242999,
      "grad_norm": 6.1773529052734375,
      "kl": 3.90234375,
      "learning_rate": 7.489051239036581e-08,
      "loss": 0.2378,
      "reward": 0.520647332072258,
      "reward_std": 0.1087775956839323,
      "rewards/accuracy_reward": 0.08035714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4402901902794838,
      "step": 2585
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1827.0647888183594,
      "epoch": 0.7724591143305205,
      "grad_norm": 8.261789321899414,
      "kl": 3.92578125,
      "learning_rate": 7.470450239067525e-08,
      "loss": 0.2194,
      "reward": 0.4888393059372902,
      "reward_std": 0.18182992190122604,
      "rewards/accuracy_reward": 0.07366071571595967,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4151785895228386,
      "step": 2586
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1826.0380249023438,
      "epoch": 0.7727578224180419,
      "grad_norm": 28.2338924407959,
      "kl": 4.63671875,
      "learning_rate": 7.451868309530523e-08,
      "loss": 0.2468,
      "reward": 0.486607164144516,
      "reward_std": 0.15239087119698524,
      "rewards/accuracy_reward": 0.0625000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4241071566939354,
      "step": 2587
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1766.7723999023438,
      "epoch": 0.7730565305055634,
      "grad_norm": 12.279585838317871,
      "kl": 3.640625,
      "learning_rate": 7.43330547064091e-08,
      "loss": 0.2028,
      "reward": 0.5530134215950966,
      "reward_std": 0.17561093904078007,
      "rewards/accuracy_reward": 0.12053571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776977300644,
      "step": 2588
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1713.9041137695312,
      "epoch": 0.7733552385930849,
      "grad_norm": 31.374746322631836,
      "kl": 2.736328125,
      "learning_rate": 7.414761742593217e-08,
      "loss": 0.2052,
      "reward": 0.5401785895228386,
      "reward_std": 0.14525115489959717,
      "rewards/accuracy_reward": 0.10044643399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.439732164144516,
      "step": 2589
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1813.8639221191406,
      "epoch": 0.7736539466806064,
      "grad_norm": 7.422126770019531,
      "kl": 2.365234375,
      "learning_rate": 7.396237145561216e-08,
      "loss": 0.1284,
      "reward": 0.4654018059372902,
      "reward_std": 0.14411384798586369,
      "rewards/accuracy_reward": 0.03125000232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341517984867096,
      "step": 2590
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1783.6340026855469,
      "epoch": 0.7739526547681278,
      "grad_norm": 40.946285247802734,
      "kl": 2.759765625,
      "learning_rate": 7.377731699697839e-08,
      "loss": 0.1992,
      "reward": 0.557477705180645,
      "reward_std": 0.16929181851446629,
      "rewards/accuracy_reward": 0.13616071874275804,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4213169813156128,
      "step": 2591
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1776.5335693359375,
      "epoch": 0.7742513628556493,
      "grad_norm": 6.593284606933594,
      "kl": 4.1015625,
      "learning_rate": 7.359245425135204e-08,
      "loss": 0.2612,
      "reward": 0.4793527126312256,
      "reward_std": 0.20341967418789864,
      "rewards/accuracy_reward": 0.08482143143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3945312649011612,
      "step": 2592
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1720.1764221191406,
      "epoch": 0.7745500709431707,
      "grad_norm": 24.739835739135742,
      "kl": 2.53515625,
      "learning_rate": 7.340778341984578e-08,
      "loss": 0.1774,
      "reward": 0.612723246216774,
      "reward_std": 0.1848236545920372,
      "rewards/accuracy_reward": 0.1830357238650322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875149011612,
      "step": 2593
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1763.1898193359375,
      "epoch": 0.7748487790306923,
      "grad_norm": 11.416841506958008,
      "kl": 2.826171875,
      "learning_rate": 7.322330470336313e-08,
      "loss": 0.1719,
      "reward": 0.5172991156578064,
      "reward_std": 0.2113415040075779,
      "rewards/accuracy_reward": 0.09151786309666932,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812723517418,
      "step": 2594
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1795.6429443359375,
      "epoch": 0.7751474871182137,
      "grad_norm": 13.142464637756348,
      "kl": 3.09765625,
      "learning_rate": 7.303901830259901e-08,
      "loss": 0.1773,
      "reward": 0.4665178805589676,
      "reward_std": 0.15197203680872917,
      "rewards/accuracy_reward": 0.0334821455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357313156128,
      "step": 2595
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1734.5000610351562,
      "epoch": 0.7754461952057352,
      "grad_norm": 4.362053871154785,
      "kl": 3.734375,
      "learning_rate": 7.2854924418039e-08,
      "loss": 0.2392,
      "reward": 0.5050223395228386,
      "reward_std": 0.15453839674592018,
      "rewards/accuracy_reward": 0.07142857392318547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937723517418,
      "step": 2596
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1693.88623046875,
      "epoch": 0.7757449032932566,
      "grad_norm": 3.745897054672241,
      "kl": 4.015625,
      "learning_rate": 7.267102324995909e-08,
      "loss": 0.2567,
      "reward": 0.4921875298023224,
      "reward_std": 0.15427321940660477,
      "rewards/accuracy_reward": 0.06250000465661287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875223517418,
      "step": 2597
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1804.8348693847656,
      "epoch": 0.7760436113807782,
      "grad_norm": 26.579376220703125,
      "kl": 5.6640625,
      "learning_rate": 7.248731499842575e-08,
      "loss": 0.3017,
      "reward": 0.5742187798023224,
      "reward_std": 0.1528733242303133,
      "rewards/accuracy_reward": 0.1517857238650322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330484867096,
      "step": 2598
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1838.2947387695312,
      "epoch": 0.7763423194682996,
      "grad_norm": 34.577247619628906,
      "kl": 5.78515625,
      "learning_rate": 7.230379986329568e-08,
      "loss": 0.2852,
      "reward": 0.463169664144516,
      "reward_std": 0.16504784300923347,
      "rewards/accuracy_reward": 0.04910714505240321,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625149011612,
      "step": 2599
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1812.9107971191406,
      "epoch": 0.7766410275558211,
      "grad_norm": 29.398542404174805,
      "kl": 4.7578125,
      "learning_rate": 7.212047804421517e-08,
      "loss": 0.2611,
      "reward": 0.4369419887661934,
      "reward_std": 0.14650611393153667,
      "rewards/accuracy_reward": 0.020089286379516125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.416852705180645,
      "step": 2600
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1719.5514221191406,
      "epoch": 0.7769397356433425,
      "grad_norm": 6.415352821350098,
      "kl": 3.796875,
      "learning_rate": 7.193734974062054e-08,
      "loss": 0.2274,
      "reward": 0.5664062723517418,
      "reward_std": 0.13747592456638813,
      "rewards/accuracy_reward": 0.1383928656578064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428013414144516,
      "step": 2601
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1774.3952026367188,
      "epoch": 0.777238443730864,
      "grad_norm": 5.695383548736572,
      "kl": 3.86328125,
      "learning_rate": 7.175441515173733e-08,
      "loss": 0.2232,
      "reward": 0.5412946790456772,
      "reward_std": 0.16458679735660553,
      "rewards/accuracy_reward": 0.11830358020961285,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 2602
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1755.0045471191406,
      "epoch": 0.7775371518183855,
      "grad_norm": 7.6023759841918945,
      "kl": 3.1484375,
      "learning_rate": 7.157167447658046e-08,
      "loss": 0.1778,
      "reward": 0.4737723469734192,
      "reward_std": 0.127156350761652,
      "rewards/accuracy_reward": 0.03348214412108064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4402901902794838,
      "step": 2603
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1856.9241943359375,
      "epoch": 0.777835859905907,
      "grad_norm": 6.773701190948486,
      "kl": 3.423828125,
      "learning_rate": 7.138912791395402e-08,
      "loss": 0.1879,
      "reward": 0.498325914144516,
      "reward_std": 0.13170508295297623,
      "rewards/accuracy_reward": 0.0870535783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4112723469734192,
      "step": 2604
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1758.7791137695312,
      "epoch": 0.7781345679934284,
      "grad_norm": 25.66123390197754,
      "kl": 3.3984375,
      "learning_rate": 7.120677566245056e-08,
      "loss": 0.2219,
      "reward": 0.5993303805589676,
      "reward_std": 0.16810395568609238,
      "rewards/accuracy_reward": 0.1785714328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4207589477300644,
      "step": 2605
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1692.1317443847656,
      "epoch": 0.7784332760809499,
      "grad_norm": 23.810937881469727,
      "kl": 3.1796875,
      "learning_rate": 7.102461792045156e-08,
      "loss": 0.2374,
      "reward": 0.5161830559372902,
      "reward_std": 0.1274232193827629,
      "rewards/accuracy_reward": 0.0959821492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.420200914144516,
      "step": 2606
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1846.4197692871094,
      "epoch": 0.7787319841684713,
      "grad_norm": 21.621829986572266,
      "kl": 2.158203125,
      "learning_rate": 7.084265488612686e-08,
      "loss": 0.1169,
      "reward": 0.5245535895228386,
      "reward_std": 0.12732814252376556,
      "rewards/accuracy_reward": 0.08928571688011289,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435267873108387,
      "step": 2607
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1774.1719970703125,
      "epoch": 0.7790306922559929,
      "grad_norm": 5.051156520843506,
      "kl": 3.1875,
      "learning_rate": 7.066088675743429e-08,
      "loss": 0.1873,
      "reward": 0.5747768133878708,
      "reward_std": 0.16054519265890121,
      "rewards/accuracy_reward": 0.1450892947614193,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875298023224,
      "step": 2608
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1798.8460693359375,
      "epoch": 0.7793294003435143,
      "grad_norm": 5.270697116851807,
      "kl": 4.0546875,
      "learning_rate": 7.047931373211979e-08,
      "loss": 0.2318,
      "reward": 0.502790205180645,
      "reward_std": 0.1417302880436182,
      "rewards/accuracy_reward": 0.08928571874275804,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4135044813156128,
      "step": 2609
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1682.8817749023438,
      "epoch": 0.7796281084310358,
      "grad_norm": 5.234367847442627,
      "kl": 2.5078125,
      "learning_rate": 7.029793600771711e-08,
      "loss": 0.1576,
      "reward": 0.549107164144516,
      "reward_std": 0.17250079475343227,
      "rewards/accuracy_reward": 0.09598214784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4531250223517418,
      "step": 2610
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1781.180908203125,
      "epoch": 0.7799268165185572,
      "grad_norm": 37.177146911621094,
      "kl": 5.71875,
      "learning_rate": 7.011675378154728e-08,
      "loss": 0.3142,
      "reward": 0.4720982387661934,
      "reward_std": 0.17425001971423626,
      "rewards/accuracy_reward": 0.058035718044266105,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625149011612,
      "step": 2611
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1775.44873046875,
      "epoch": 0.7802255246060787,
      "grad_norm": 47.473731994628906,
      "kl": 5.96875,
      "learning_rate": 6.993576725071887e-08,
      "loss": 0.3173,
      "reward": 0.4609375223517418,
      "reward_std": 0.17072391137480736,
      "rewards/accuracy_reward": 0.04687500232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625149011612,
      "step": 2612
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1838.3125915527344,
      "epoch": 0.7805242326936002,
      "grad_norm": 18.83847999572754,
      "kl": 4.16015625,
      "learning_rate": 6.975497661212759e-08,
      "loss": 0.2282,
      "reward": 0.4458705559372902,
      "reward_std": 0.13977430760860443,
      "rewards/accuracy_reward": 0.01562500069849193,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455559372902,
      "step": 2613
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1733.5960388183594,
      "epoch": 0.7808229407811217,
      "grad_norm": 23.1492977142334,
      "kl": 5.8671875,
      "learning_rate": 6.957438206245574e-08,
      "loss": 0.3431,
      "reward": 0.4843750149011612,
      "reward_std": 0.14722616225481033,
      "rewards/accuracy_reward": 0.06026785937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4241071566939354,
      "step": 2614
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1848.1407165527344,
      "epoch": 0.7811216488686431,
      "grad_norm": 16.575899124145508,
      "kl": 4.89453125,
      "learning_rate": 6.939398379817271e-08,
      "loss": 0.2734,
      "reward": 0.5368303805589676,
      "reward_std": 0.18778608180582523,
      "rewards/accuracy_reward": 0.12500000116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.411830373108387,
      "step": 2615
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1786.3795471191406,
      "epoch": 0.7814203569561646,
      "grad_norm": 6.960030555725098,
      "kl": 4.00390625,
      "learning_rate": 6.921378201553394e-08,
      "loss": 0.2487,
      "reward": 0.465959832072258,
      "reward_std": 0.11664685793220997,
      "rewards/accuracy_reward": 0.0446428582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4213169813156128,
      "step": 2616
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1836.6630554199219,
      "epoch": 0.781719065043686,
      "grad_norm": 5.789151668548584,
      "kl": 3.1328125,
      "learning_rate": 6.903377691058135e-08,
      "loss": 0.1767,
      "reward": 0.4335937723517418,
      "reward_std": 0.13940389081835747,
      "rewards/accuracy_reward": 0.0200892873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4135044887661934,
      "step": 2617
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1698.52685546875,
      "epoch": 0.7820177731312076,
      "grad_norm": 8.267499923706055,
      "kl": 2.880859375,
      "learning_rate": 6.8853968679143e-08,
      "loss": 0.1685,
      "reward": 0.6021205633878708,
      "reward_std": 0.17515825852751732,
      "rewards/accuracy_reward": 0.16741072479635477,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098469734192,
      "step": 2618
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1811.7076721191406,
      "epoch": 0.782316481218729,
      "grad_norm": 5.618217468261719,
      "kl": 3.3828125,
      "learning_rate": 6.86743575168324e-08,
      "loss": 0.1962,
      "reward": 0.4860491305589676,
      "reward_std": 0.1859223209321499,
      "rewards/accuracy_reward": 0.06026786100119352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812723517418,
      "step": 2619
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1718.4018859863281,
      "epoch": 0.7826151893062505,
      "grad_norm": 13.793340682983398,
      "kl": 3.310546875,
      "learning_rate": 6.849494361904901e-08,
      "loss": 0.2114,
      "reward": 0.4497768133878708,
      "reward_std": 0.1381816752254963,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268059372902,
      "step": 2620
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1751.2902526855469,
      "epoch": 0.7829138973937719,
      "grad_norm": 5.818290710449219,
      "kl": 3.578125,
      "learning_rate": 6.831572718097765e-08,
      "loss": 0.2305,
      "reward": 0.5440848469734192,
      "reward_std": 0.13982860930263996,
      "rewards/accuracy_reward": 0.12946428963914514,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205484867096,
      "step": 2621
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1836.6027526855469,
      "epoch": 0.7832126054812935,
      "grad_norm": 8.277383804321289,
      "kl": 4.181640625,
      "learning_rate": 6.813670839758804e-08,
      "loss": 0.213,
      "reward": 0.5820312723517418,
      "reward_std": 0.1880913209170103,
      "rewards/accuracy_reward": 0.1741071492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4079241305589676,
      "step": 2622
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1832.40185546875,
      "epoch": 0.7835113135688149,
      "grad_norm": 6.086855411529541,
      "kl": 3.52734375,
      "learning_rate": 6.795788746363518e-08,
      "loss": 0.1955,
      "reward": 0.4497767984867096,
      "reward_std": 0.15682524628937244,
      "rewards/accuracy_reward": 0.03125000069849193,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185267984867096,
      "step": 2623
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1746.8505249023438,
      "epoch": 0.7838100216563364,
      "grad_norm": 11.244091987609863,
      "kl": 2.6953125,
      "learning_rate": 6.777926457365879e-08,
      "loss": 0.1722,
      "reward": 0.4849330484867096,
      "reward_std": 0.15624531917273998,
      "rewards/accuracy_reward": 0.04687500116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4380580559372902,
      "step": 2624
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1684.1116638183594,
      "epoch": 0.7841087297438578,
      "grad_norm": 6.74306583404541,
      "kl": 3.6953125,
      "learning_rate": 6.760083992198293e-08,
      "loss": 0.2113,
      "reward": 0.5401785969734192,
      "reward_std": 0.12515992112457752,
      "rewards/accuracy_reward": 0.11160714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 2625
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1810.9866943359375,
      "epoch": 0.7844074378313793,
      "grad_norm": 12.638365745544434,
      "kl": 3.79296875,
      "learning_rate": 6.742261370271618e-08,
      "loss": 0.2366,
      "reward": 0.5273437798023224,
      "reward_std": 0.13212844729423523,
      "rewards/accuracy_reward": 0.1049107164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330484867096,
      "step": 2626
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1758.3995971679688,
      "epoch": 0.7847061459189008,
      "grad_norm": 3.19701886177063,
      "kl": 3.078125,
      "learning_rate": 6.724458610975131e-08,
      "loss": 0.1619,
      "reward": 0.431919664144516,
      "reward_std": 0.11859615333378315,
      "rewards/accuracy_reward": 0.008928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 2627
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1802.1027526855469,
      "epoch": 0.7850048540064222,
      "grad_norm": 15.597496032714844,
      "kl": 4.9296875,
      "learning_rate": 6.70667573367647e-08,
      "loss": 0.2659,
      "reward": 0.505580373108387,
      "reward_std": 0.15213859640061855,
      "rewards/accuracy_reward": 0.08705357578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268059372902,
      "step": 2628
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1860.4398498535156,
      "epoch": 0.7853035620939437,
      "grad_norm": 9.682565689086914,
      "kl": 4.6875,
      "learning_rate": 6.688912757721682e-08,
      "loss": 0.2466,
      "reward": 0.4380580559372902,
      "reward_std": 0.13988455198705196,
      "rewards/accuracy_reward": 0.022321430034935474,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4157366305589676,
      "step": 2629
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1810.0558776855469,
      "epoch": 0.7856022701814651,
      "grad_norm": 25.46393394470215,
      "kl": 4.2578125,
      "learning_rate": 6.671169702435125e-08,
      "loss": 0.2389,
      "reward": 0.5011160895228386,
      "reward_std": 0.16951162740588188,
      "rewards/accuracy_reward": 0.08482143213041127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.416294664144516,
      "step": 2630
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1796.5179443359375,
      "epoch": 0.7859009782689866,
      "grad_norm": 3.847527027130127,
      "kl": 2.693359375,
      "learning_rate": 6.653446587119513e-08,
      "loss": 0.1521,
      "reward": 0.5658482387661934,
      "reward_std": 0.15450603514909744,
      "rewards/accuracy_reward": 0.1250000074505806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4408482313156128,
      "step": 2631
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1801.5045471191406,
      "epoch": 0.786199686356508,
      "grad_norm": 12.678567886352539,
      "kl": 3.61328125,
      "learning_rate": 6.635743431055868e-08,
      "loss": 0.1939,
      "reward": 0.513950914144516,
      "reward_std": 0.15575001761317253,
      "rewards/accuracy_reward": 0.08928571618162096,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4246651977300644,
      "step": 2632
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1829.4018859863281,
      "epoch": 0.7864983944440296,
      "grad_norm": 13.165144920349121,
      "kl": 3.08203125,
      "learning_rate": 6.618060253503471e-08,
      "loss": 0.1759,
      "reward": 0.4793526977300644,
      "reward_std": 0.18314307555556297,
      "rewards/accuracy_reward": 0.051339288940653205,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4280134066939354,
      "step": 2633
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1746.4598693847656,
      "epoch": 0.786797102531551,
      "grad_norm": 21.526649475097656,
      "kl": 3.453125,
      "learning_rate": 6.600397073699893e-08,
      "loss": 0.2338,
      "reward": 0.5145089402794838,
      "reward_std": 0.16112812422215939,
      "rewards/accuracy_reward": 0.10044643469154835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625149011612,
      "step": 2634
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1708.540283203125,
      "epoch": 0.7870958106190725,
      "grad_norm": 15.20854377746582,
      "kl": 2.4765625,
      "learning_rate": 6.582753910860952e-08,
      "loss": 0.1663,
      "reward": 0.4464285895228386,
      "reward_std": 0.13517841510474682,
      "rewards/accuracy_reward": 0.0200892873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263393059372902,
      "step": 2635
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1735.0670471191406,
      "epoch": 0.7873945187065939,
      "grad_norm": 7.885658264160156,
      "kl": 2.5703125,
      "learning_rate": 6.565130784180667e-08,
      "loss": 0.1592,
      "reward": 0.4782366305589676,
      "reward_std": 0.14847423508763313,
      "rewards/accuracy_reward": 0.04241071757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4358259066939354,
      "step": 2636
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1768.9620971679688,
      "epoch": 0.7876932267941155,
      "grad_norm": 25.886028289794922,
      "kl": 2.490234375,
      "learning_rate": 6.547527712831274e-08,
      "loss": 0.1771,
      "reward": 0.528459832072258,
      "reward_std": 0.19036472588777542,
      "rewards/accuracy_reward": 0.1004464328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4280134066939354,
      "step": 2637
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1810.6407470703125,
      "epoch": 0.7879919348816369,
      "grad_norm": 9.92557430267334,
      "kl": 4.03125,
      "learning_rate": 6.529944715963204e-08,
      "loss": 0.226,
      "reward": 0.4960937723517418,
      "reward_std": 0.16224156692624092,
      "rewards/accuracy_reward": 0.0915178619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.404575914144516,
      "step": 2638
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1718.8036193847656,
      "epoch": 0.7882906429691584,
      "grad_norm": 27.776004791259766,
      "kl": 2.564453125,
      "learning_rate": 6.512381812705012e-08,
      "loss": 0.1647,
      "reward": 0.5959821790456772,
      "reward_std": 0.14928384125232697,
      "rewards/accuracy_reward": 0.1540178619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4419643059372902,
      "step": 2639
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1729.8706359863281,
      "epoch": 0.7885893510566798,
      "grad_norm": 12.225401878356934,
      "kl": 3.00390625,
      "learning_rate": 6.494839022163429e-08,
      "loss": 0.1905,
      "reward": 0.526227705180645,
      "reward_std": 0.17418872937560081,
      "rewards/accuracy_reward": 0.09598214505240321,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455484867096,
      "step": 2640
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1812.6072082519531,
      "epoch": 0.7888880591442013,
      "grad_norm": 10.067612648010254,
      "kl": 4.5,
      "learning_rate": 6.477316363423297e-08,
      "loss": 0.2432,
      "reward": 0.4581473395228386,
      "reward_std": 0.1527930125594139,
      "rewards/accuracy_reward": 0.03125000209547579,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4268973395228386,
      "step": 2641
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1692.8014221191406,
      "epoch": 0.7891867672317228,
      "grad_norm": 12.770482063293457,
      "kl": 2.802734375,
      "learning_rate": 6.459813855547533e-08,
      "loss": 0.1898,
      "reward": 0.5033482387661934,
      "reward_std": 0.1317837107926607,
      "rewards/accuracy_reward": 0.06026785867288709,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443080373108387,
      "step": 2642
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1779.6541137695312,
      "epoch": 0.7894854753192443,
      "grad_norm": 12.473730087280273,
      "kl": 3.9765625,
      "learning_rate": 6.442331517577168e-08,
      "loss": 0.2352,
      "reward": 0.5680803880095482,
      "reward_std": 0.1985703930258751,
      "rewards/accuracy_reward": 0.13169643469154835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4363839477300644,
      "step": 2643
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1763.8728332519531,
      "epoch": 0.7897841834067657,
      "grad_norm": 16.01439666748047,
      "kl": 3.8359375,
      "learning_rate": 6.42486936853126e-08,
      "loss": 0.2176,
      "reward": 0.4715401902794838,
      "reward_std": 0.14035964012145996,
      "rewards/accuracy_reward": 0.04017857206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.431361623108387,
      "step": 2644
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1730.2545776367188,
      "epoch": 0.7900828914942872,
      "grad_norm": 13.4894437789917,
      "kl": 4.796875,
      "learning_rate": 6.407427427406919e-08,
      "loss": 0.3041,
      "reward": 0.5111607387661934,
      "reward_std": 0.15011709742248058,
      "rewards/accuracy_reward": 0.0803571455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035969734192,
      "step": 2645
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1804.3438110351562,
      "epoch": 0.7903815995818086,
      "grad_norm": 32.61983108520508,
      "kl": 4.8203125,
      "learning_rate": 6.390005713179277e-08,
      "loss": 0.2421,
      "reward": 0.5602678656578064,
      "reward_std": 0.14925477467477322,
      "rewards/accuracy_reward": 0.13839286239817739,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 2646
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1820.66748046875,
      "epoch": 0.7906803076693302,
      "grad_norm": 10.622252464294434,
      "kl": 4.8671875,
      "learning_rate": 6.372604244801438e-08,
      "loss": 0.2661,
      "reward": 0.558593787252903,
      "reward_std": 0.16292515583336353,
      "rewards/accuracy_reward": 0.1540178619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4045759066939354,
      "step": 2647
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1771.5045471191406,
      "epoch": 0.7909790157568516,
      "grad_norm": 10.448858261108398,
      "kl": 3.24609375,
      "learning_rate": 6.355223041204496e-08,
      "loss": 0.2075,
      "reward": 0.5295759066939354,
      "reward_std": 0.16583773121237755,
      "rewards/accuracy_reward": 0.10044643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294813156128,
      "step": 2648
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1737.3951416015625,
      "epoch": 0.7912777238443731,
      "grad_norm": 15.623565673828125,
      "kl": 3.35546875,
      "learning_rate": 6.337862121297511e-08,
      "loss": 0.2108,
      "reward": 0.5385044813156128,
      "reward_std": 0.11819391697645187,
      "rewards/accuracy_reward": 0.11607143399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330559372902,
      "step": 2649
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1834.9911499023438,
      "epoch": 0.7915764319318945,
      "grad_norm": 14.918390274047852,
      "kl": 3.13671875,
      "learning_rate": 6.320521503967444e-08,
      "loss": 0.1845,
      "reward": 0.497209832072258,
      "reward_std": 0.1772791799157858,
      "rewards/accuracy_reward": 0.08035714831203222,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4168526977300644,
      "step": 2650
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1805.6384887695312,
      "epoch": 0.7918751400194161,
      "grad_norm": 6.5464396476745605,
      "kl": 3.9453125,
      "learning_rate": 6.303201208079196e-08,
      "loss": 0.2275,
      "reward": 0.5530134215950966,
      "reward_std": 0.14531385898590088,
      "rewards/accuracy_reward": 0.13616072502918541,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4168526977300644,
      "step": 2651
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1750.1920166015625,
      "epoch": 0.7921738481069375,
      "grad_norm": 10.428670883178711,
      "kl": 3.4296875,
      "learning_rate": 6.285901252475556e-08,
      "loss": 0.1997,
      "reward": 0.4704241305589676,
      "reward_std": 0.12357258051633835,
      "rewards/accuracy_reward": 0.044642860535532236,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812649011612,
      "step": 2652
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1788.2143859863281,
      "epoch": 0.792472556194459,
      "grad_norm": 16.69830894470215,
      "kl": 2.7109375,
      "learning_rate": 6.268621655977166e-08,
      "loss": 0.1681,
      "reward": 0.4458705484867096,
      "reward_std": 0.1195985097438097,
      "rewards/accuracy_reward": 0.011160715017467737,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098395228386,
      "step": 2653
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1737.3371276855469,
      "epoch": 0.7927712642819804,
      "grad_norm": 13.762235641479492,
      "kl": 3.296875,
      "learning_rate": 6.251362437382543e-08,
      "loss": 0.2129,
      "reward": 0.4665178880095482,
      "reward_std": 0.10243247449398041,
      "rewards/accuracy_reward": 0.0357142873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035969734192,
      "step": 2654
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1712.7277526855469,
      "epoch": 0.7930699723695019,
      "grad_norm": 3.9527676105499268,
      "kl": 3.796875,
      "learning_rate": 6.234123615468031e-08,
      "loss": 0.2408,
      "reward": 0.4905134215950966,
      "reward_std": 0.1691918671131134,
      "rewards/accuracy_reward": 0.06696428847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4235491305589676,
      "step": 2655
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1751.6317749023438,
      "epoch": 0.7933686804570234,
      "grad_norm": 25.405956268310547,
      "kl": 4.2109375,
      "learning_rate": 6.216905208987766e-08,
      "loss": 0.2351,
      "reward": 0.4720982387661934,
      "reward_std": 0.15214226208627224,
      "rewards/accuracy_reward": 0.0446428582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4274553805589676,
      "step": 2656
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1746.3326721191406,
      "epoch": 0.7936673885445449,
      "grad_norm": 36.62749481201172,
      "kl": 5.3828125,
      "learning_rate": 6.199707236673702e-08,
      "loss": 0.2846,
      "reward": 0.4704241305589676,
      "reward_std": 0.1415501032024622,
      "rewards/accuracy_reward": 0.055803573690354824,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205484867096,
      "step": 2657
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1757.1496276855469,
      "epoch": 0.7939660966320663,
      "grad_norm": 55.959503173828125,
      "kl": 6.9765625,
      "learning_rate": 6.182529717235526e-08,
      "loss": 0.3784,
      "reward": 0.4838169887661934,
      "reward_std": 0.1719274688512087,
      "rewards/accuracy_reward": 0.07366071827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4101562723517418,
      "step": 2658
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1714.01123046875,
      "epoch": 0.7942648047195878,
      "grad_norm": 6.323443412780762,
      "kl": 3.388671875,
      "learning_rate": 6.165372669360708e-08,
      "loss": 0.2038,
      "reward": 0.5284598469734192,
      "reward_std": 0.14461936242878437,
      "rewards/accuracy_reward": 0.08928571571595967,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4391741305589676,
      "step": 2659
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1756.5023193359375,
      "epoch": 0.7945635128071092,
      "grad_norm": 8.904153823852539,
      "kl": 3.830078125,
      "learning_rate": 6.148236111714444e-08,
      "loss": 0.2172,
      "reward": 0.5351562798023224,
      "reward_std": 0.1743953749537468,
      "rewards/accuracy_reward": 0.1116071492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.423549123108387,
      "step": 2660
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1720.1116943359375,
      "epoch": 0.7948622208946308,
      "grad_norm": 9.021265983581543,
      "kl": 3.875,
      "learning_rate": 6.13112006293961e-08,
      "loss": 0.2386,
      "reward": 0.5206473395228386,
      "reward_std": 0.14494216814637184,
      "rewards/accuracy_reward": 0.09151786053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294887661934,
      "step": 2661
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1818.4509887695312,
      "epoch": 0.7951609289821522,
      "grad_norm": 22.548654556274414,
      "kl": 5.16015625,
      "learning_rate": 6.114024541656798e-08,
      "loss": 0.2705,
      "reward": 0.4782366380095482,
      "reward_std": 0.14867137372493744,
      "rewards/accuracy_reward": 0.07142857578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4068080484867096,
      "step": 2662
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1759.4353637695312,
      "epoch": 0.7954596370696737,
      "grad_norm": 12.398488998413086,
      "kl": 3.4609375,
      "learning_rate": 6.096949566464268e-08,
      "loss": 0.2072,
      "reward": 0.5424107387661934,
      "reward_std": 0.17735926993191242,
      "rewards/accuracy_reward": 0.113839291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428571455180645,
      "step": 2663
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1742.5090026855469,
      "epoch": 0.7957583451571951,
      "grad_norm": 35.125389099121094,
      "kl": 3.6640625,
      "learning_rate": 6.079895155937903e-08,
      "loss": 0.2617,
      "reward": 0.5993303805589676,
      "reward_std": 0.18505988642573357,
      "rewards/accuracy_reward": 0.17633929289877415,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 2664
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1786.9175109863281,
      "epoch": 0.7960570532447167,
      "grad_norm": 14.177800178527832,
      "kl": 3.06640625,
      "learning_rate": 6.062861328631241e-08,
      "loss": 0.1845,
      "reward": 0.5954241380095482,
      "reward_std": 0.17716820910573006,
      "rewards/accuracy_reward": 0.17633929895237088,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4190848395228386,
      "step": 2665
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1786.2791137695312,
      "epoch": 0.7963557613322381,
      "grad_norm": 12.560492515563965,
      "kl": 3.0859375,
      "learning_rate": 6.045848103075421e-08,
      "loss": 0.1964,
      "reward": 0.4218750223517418,
      "reward_std": 0.1330568864941597,
      "rewards/accuracy_reward": 0.008928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4129464477300644,
      "step": 2666
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1818.74560546875,
      "epoch": 0.7966544694197596,
      "grad_norm": 14.246768951416016,
      "kl": 3.19921875,
      "learning_rate": 6.02885549777915e-08,
      "loss": 0.1822,
      "reward": 0.4720982387661934,
      "reward_std": 0.11956014856696129,
      "rewards/accuracy_reward": 0.04464285937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4274553805589676,
      "step": 2667
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1807.6005249023438,
      "epoch": 0.796953177507281,
      "grad_norm": 5.390991687774658,
      "kl": 3.1796875,
      "learning_rate": 6.011883531228726e-08,
      "loss": 0.1888,
      "reward": 0.4765625223517418,
      "reward_std": 0.1474661510437727,
      "rewards/accuracy_reward": 0.05580357299186289,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4207589477300644,
      "step": 2668
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1702.1161499023438,
      "epoch": 0.7972518855948025,
      "grad_norm": 11.103914260864258,
      "kl": 3.291015625,
      "learning_rate": 5.994932221887991e-08,
      "loss": 0.2005,
      "reward": 0.4760044813156128,
      "reward_std": 0.14368155226111412,
      "rewards/accuracy_reward": 0.03794643096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4380580559372902,
      "step": 2669
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1740.0290832519531,
      "epoch": 0.797550593682324,
      "grad_norm": 12.066716194152832,
      "kl": 3.64453125,
      "learning_rate": 5.978001588198295e-08,
      "loss": 0.2162,
      "reward": 0.4575893133878708,
      "reward_std": 0.15499232150614262,
      "rewards/accuracy_reward": 0.029017858440056443,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 2670
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1809.8750915527344,
      "epoch": 0.7978493017698454,
      "grad_norm": 9.404921531677246,
      "kl": 4.16015625,
      "learning_rate": 5.961091648578523e-08,
      "loss": 0.2347,
      "reward": 0.5558035895228386,
      "reward_std": 0.1346595361828804,
      "rewards/accuracy_reward": 0.1316964328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4241071566939354,
      "step": 2671
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1710.2255249023438,
      "epoch": 0.7981480098573669,
      "grad_norm": 9.469446182250977,
      "kl": 4.16015625,
      "learning_rate": 5.944202421425015e-08,
      "loss": 0.265,
      "reward": 0.530133955180645,
      "reward_std": 0.17563442513346672,
      "rewards/accuracy_reward": 0.08928571920841932,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4408482387661934,
      "step": 2672
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.1451721191406,
      "epoch": 0.7984467179448883,
      "grad_norm": 32.00592041015625,
      "kl": 5.65234375,
      "learning_rate": 5.927333925111605e-08,
      "loss": 0.3045,
      "reward": 0.5742187723517418,
      "reward_std": 0.1626296602189541,
      "rewards/accuracy_reward": 0.14732143469154835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.426897332072258,
      "step": 2673
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1742.9219665527344,
      "epoch": 0.7987454260324098,
      "grad_norm": 6.323920249938965,
      "kl": 3.2890625,
      "learning_rate": 5.910486177989571e-08,
      "loss": 0.1869,
      "reward": 0.5602678656578064,
      "reward_std": 0.19033929519355297,
      "rewards/accuracy_reward": 0.1294642947614193,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035895228386,
      "step": 2674
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1792.6362609863281,
      "epoch": 0.7990441341199312,
      "grad_norm": 6.123708248138428,
      "kl": 4.0390625,
      "learning_rate": 5.893659198387593e-08,
      "loss": 0.2396,
      "reward": 0.5362723544239998,
      "reward_std": 0.16536353528499603,
      "rewards/accuracy_reward": 0.10937500488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4268973469734192,
      "step": 2675
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1775.9598999023438,
      "epoch": 0.7993428422074528,
      "grad_norm": 16.610986709594727,
      "kl": 3.65234375,
      "learning_rate": 5.8768530046117844e-08,
      "loss": 0.22,
      "reward": 0.4564732387661934,
      "reward_std": 0.1524439938366413,
      "rewards/accuracy_reward": 0.029017857974395156,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4274553805589676,
      "step": 2676
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1742.8505249023438,
      "epoch": 0.7996415502949742,
      "grad_norm": 16.505613327026367,
      "kl": 3.365234375,
      "learning_rate": 5.8600676149456435e-08,
      "loss": 0.1915,
      "reward": 0.515066996216774,
      "reward_std": 0.1809722352772951,
      "rewards/accuracy_reward": 0.082589291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.432477705180645,
      "step": 2677
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1718.6295776367188,
      "epoch": 0.7999402583824957,
      "grad_norm": 15.703069686889648,
      "kl": 3.220703125,
      "learning_rate": 5.8433030476500165e-08,
      "loss": 0.1944,
      "reward": 0.5150669738650322,
      "reward_std": 0.10030623432248831,
      "rewards/accuracy_reward": 0.0781250037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419813156128,
      "step": 2678
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1786.0648193359375,
      "epoch": 0.8002389664700171,
      "grad_norm": 5.701539993286133,
      "kl": 3.51953125,
      "learning_rate": 5.8265593209631145e-08,
      "loss": 0.1945,
      "reward": 0.4916294887661934,
      "reward_std": 0.16214940883219242,
      "rewards/accuracy_reward": 0.07812500232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4135044813156128,
      "step": 2679
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1770.3639221191406,
      "epoch": 0.8005376745575387,
      "grad_norm": 17.66838264465332,
      "kl": 2.96875,
      "learning_rate": 5.809836453100478e-08,
      "loss": 0.188,
      "reward": 0.4882812798023224,
      "reward_std": 0.15546676144003868,
      "rewards/accuracy_reward": 0.06250000465661287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812723517418,
      "step": 2680
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1759.1228332519531,
      "epoch": 0.8008363826450601,
      "grad_norm": 11.211368560791016,
      "kl": 2.986328125,
      "learning_rate": 5.793134462254939e-08,
      "loss": 0.177,
      "reward": 0.4882812723517418,
      "reward_std": 0.12264377996325493,
      "rewards/accuracy_reward": 0.06026786006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4280134066939354,
      "step": 2681
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1714.0291137695312,
      "epoch": 0.8011350907325816,
      "grad_norm": 15.79074478149414,
      "kl": 3.05859375,
      "learning_rate": 5.776453366596634e-08,
      "loss": 0.2193,
      "reward": 0.5373884215950966,
      "reward_std": 0.12578619830310345,
      "rewards/accuracy_reward": 0.09821429033763707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4391741305589676,
      "step": 2682
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1723.0201416015625,
      "epoch": 0.801433798820103,
      "grad_norm": 13.78646469116211,
      "kl": 2.994140625,
      "learning_rate": 5.7597931842729515e-08,
      "loss": 0.1952,
      "reward": 0.475446455180645,
      "reward_std": 0.10618437454104424,
      "rewards/accuracy_reward": 0.04464285937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035969734192,
      "step": 2683
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1791.6764221191406,
      "epoch": 0.8017325069076245,
      "grad_norm": 3.4600656032562256,
      "kl": 3.087890625,
      "learning_rate": 5.74315393340854e-08,
      "loss": 0.1635,
      "reward": 0.4793526977300644,
      "reward_std": 0.13877943716943264,
      "rewards/accuracy_reward": 0.04910714365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455559372902,
      "step": 2684
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1719.4710693359375,
      "epoch": 0.802031214995146,
      "grad_norm": 4.950695037841797,
      "kl": 2.779296875,
      "learning_rate": 5.726535632105281e-08,
      "loss": 0.1491,
      "reward": 0.643415205180645,
      "reward_std": 0.14173772744834423,
      "rewards/accuracy_reward": 0.2008928619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4425223395228386,
      "step": 2685
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1784.18310546875,
      "epoch": 0.8023299230826675,
      "grad_norm": 7.332329273223877,
      "kl": 3.93359375,
      "learning_rate": 5.709938298442246e-08,
      "loss": 0.2306,
      "reward": 0.479352705180645,
      "reward_std": 0.1445106640458107,
      "rewards/accuracy_reward": 0.04687500209547579,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776977300644,
      "step": 2686
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1745.4710388183594,
      "epoch": 0.8026286311701889,
      "grad_norm": 40.964088439941406,
      "kl": 6.6796875,
      "learning_rate": 5.6933619504757144e-08,
      "loss": 0.3821,
      "reward": 0.5111607387661934,
      "reward_std": 0.18121357448399067,
      "rewards/accuracy_reward": 0.0937500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4174107313156128,
      "step": 2687
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.55810546875,
      "epoch": 0.8029273392577104,
      "grad_norm": 13.422542572021484,
      "kl": 4.00390625,
      "learning_rate": 5.676806606239132e-08,
      "loss": 0.2217,
      "reward": 0.5770089477300644,
      "reward_std": 0.21027339063584805,
      "rewards/accuracy_reward": 0.1473214328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875223517418,
      "step": 2688
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1752.7924499511719,
      "epoch": 0.8032260473452318,
      "grad_norm": 5.352603435516357,
      "kl": 4.3515625,
      "learning_rate": 5.660272283743078e-08,
      "loss": 0.2667,
      "reward": 0.4966518059372902,
      "reward_std": 0.1422208696603775,
      "rewards/accuracy_reward": 0.06696428963914514,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875149011612,
      "step": 2689
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1763.1563415527344,
      "epoch": 0.8035247554327534,
      "grad_norm": 16.05196189880371,
      "kl": 3.6953125,
      "learning_rate": 5.6437590009752844e-08,
      "loss": 0.1804,
      "reward": 0.5094866305589676,
      "reward_std": 0.19575892388820648,
      "rewards/accuracy_reward": 0.0691964328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4402901977300644,
      "step": 2690
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1813.7634887695312,
      "epoch": 0.8038234635202748,
      "grad_norm": 30.903993606567383,
      "kl": 5.71875,
      "learning_rate": 5.627266775900588e-08,
      "loss": 0.2992,
      "reward": 0.482142873108387,
      "reward_std": 0.15746891126036644,
      "rewards/accuracy_reward": 0.0758928619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500223517418,
      "step": 2691
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1823.0313720703125,
      "epoch": 0.8041221716077963,
      "grad_norm": 17.795106887817383,
      "kl": 4.125,
      "learning_rate": 5.610795626460901e-08,
      "loss": 0.2262,
      "reward": 0.5396205633878708,
      "reward_std": 0.15288034081459045,
      "rewards/accuracy_reward": 0.1272321492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4123884066939354,
      "step": 2692
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1690.6161499023438,
      "epoch": 0.8044208796953177,
      "grad_norm": 15.001956939697266,
      "kl": 4.234375,
      "learning_rate": 5.594345570575226e-08,
      "loss": 0.2642,
      "reward": 0.458705373108387,
      "reward_std": 0.16571844927966595,
      "rewards/accuracy_reward": 0.040178573690354824,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185267984867096,
      "step": 2693
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1803.6384887695312,
      "epoch": 0.8047195877828393,
      "grad_norm": 6.8132500648498535,
      "kl": 4.01171875,
      "learning_rate": 5.5779166261396173e-08,
      "loss": 0.215,
      "reward": 0.5385044887661934,
      "reward_std": 0.14865673705935478,
      "rewards/accuracy_reward": 0.1160714328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330559372902,
      "step": 2694
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1798.7991943359375,
      "epoch": 0.8050182958703607,
      "grad_norm": 8.135534286499023,
      "kl": 2.9140625,
      "learning_rate": 5.561508811027146e-08,
      "loss": 0.1735,
      "reward": 0.5407366380095482,
      "reward_std": 0.16322751715779305,
      "rewards/accuracy_reward": 0.11383929080329835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4268973469734192,
      "step": 2695
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1775.2567749023438,
      "epoch": 0.8053170039578822,
      "grad_norm": 11.127165794372559,
      "kl": 4.046875,
      "learning_rate": 5.545122143087916e-08,
      "loss": 0.2369,
      "reward": 0.416852705180645,
      "reward_std": 0.11688901856541634,
      "rewards/accuracy_reward": 0.006696428870782256,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4101562723517418,
      "step": 2696
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1775.5938415527344,
      "epoch": 0.8056157120454036,
      "grad_norm": 10.633901596069336,
      "kl": 3.65234375,
      "learning_rate": 5.528756640149004e-08,
      "loss": 0.2172,
      "reward": 0.5479910969734192,
      "reward_std": 0.16496592015028,
      "rewards/accuracy_reward": 0.12946428917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268133878708,
      "step": 2697
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1841.1317749023438,
      "epoch": 0.8059144201329251,
      "grad_norm": 8.39854621887207,
      "kl": 3.82421875,
      "learning_rate": 5.512412320014484e-08,
      "loss": 0.2115,
      "reward": 0.4854910895228386,
      "reward_std": 0.1501553263515234,
      "rewards/accuracy_reward": 0.06473214528523386,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4207589477300644,
      "step": 2698
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1688.93310546875,
      "epoch": 0.8062131282204466,
      "grad_norm": 15.52246379852295,
      "kl": 2.91015625,
      "learning_rate": 5.496089200465376e-08,
      "loss": 0.2087,
      "reward": 0.541852705180645,
      "reward_std": 0.15498312935233116,
      "rewards/accuracy_reward": 0.10267857392318547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.439174123108387,
      "step": 2699
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1743.8371276855469,
      "epoch": 0.8065118363079681,
      "grad_norm": 9.185998916625977,
      "kl": 3.23046875,
      "learning_rate": 5.4797872992596225e-08,
      "loss": 0.1906,
      "reward": 0.571986623108387,
      "reward_std": 0.14406094700098038,
      "rewards/accuracy_reward": 0.13392857694998384,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4380580559372902,
      "step": 2700
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1751.2277526855469,
      "epoch": 0.8068105443954895,
      "grad_norm": 9.003678321838379,
      "kl": 3.8203125,
      "learning_rate": 5.4635066341321006e-08,
      "loss": 0.2414,
      "reward": 0.5390625298023224,
      "reward_std": 0.16397096775472164,
      "rewards/accuracy_reward": 0.113839291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232313156128,
      "step": 2701
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1788.4308776855469,
      "epoch": 0.807109252483011,
      "grad_norm": 22.01239013671875,
      "kl": 4.171875,
      "learning_rate": 5.447247222794585e-08,
      "loss": 0.237,
      "reward": 0.5619420036673546,
      "reward_std": 0.14808448404073715,
      "rewards/accuracy_reward": 0.13839286309666932,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4235491305589676,
      "step": 2702
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1831.1920471191406,
      "epoch": 0.8074079605705324,
      "grad_norm": 15.586058616638184,
      "kl": 4.89453125,
      "learning_rate": 5.431009082935709e-08,
      "loss": 0.2662,
      "reward": 0.462053582072258,
      "reward_std": 0.13528158329427242,
      "rewards/accuracy_reward": 0.04910714505240321,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4129464477300644,
      "step": 2703
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1773.5023193359375,
      "epoch": 0.807706668658054,
      "grad_norm": 24.383127212524414,
      "kl": 4.65625,
      "learning_rate": 5.414792232220983e-08,
      "loss": 0.2733,
      "reward": 0.5172991305589676,
      "reward_std": 0.17912793532013893,
      "rewards/accuracy_reward": 0.1116071492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4056919813156128,
      "step": 2704
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1716.1161499023438,
      "epoch": 0.8080053767455754,
      "grad_norm": 16.167301177978516,
      "kl": 3.61328125,
      "learning_rate": 5.398596688292758e-08,
      "loss": 0.2262,
      "reward": 0.459263414144516,
      "reward_std": 0.13561873324215412,
      "rewards/accuracy_reward": 0.0290178582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455559372902,
      "step": 2705
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1695.3594665527344,
      "epoch": 0.8083040848330969,
      "grad_norm": 17.307071685791016,
      "kl": 3.07421875,
      "learning_rate": 5.382422468770187e-08,
      "loss": 0.2097,
      "reward": 0.466517873108387,
      "reward_std": 0.14342264831066132,
      "rewards/accuracy_reward": 0.026785715715959668,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.439732164144516,
      "step": 2706
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1773.6094360351562,
      "epoch": 0.8086027929206183,
      "grad_norm": 7.1943817138671875,
      "kl": 2.28515625,
      "learning_rate": 5.366269591249242e-08,
      "loss": 0.1376,
      "reward": 0.5820312723517418,
      "reward_std": 0.20988063886761665,
      "rewards/accuracy_reward": 0.14508929220028222,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419887661934,
      "step": 2707
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1661.8951721191406,
      "epoch": 0.8089015010081398,
      "grad_norm": 4.238203525543213,
      "kl": 3.45703125,
      "learning_rate": 5.350138073302673e-08,
      "loss": 0.199,
      "reward": 0.5881696715950966,
      "reward_std": 0.16262873448431492,
      "rewards/accuracy_reward": 0.15401786379516125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341518059372902,
      "step": 2708
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1764.0826721191406,
      "epoch": 0.8092002090956613,
      "grad_norm": 7.046487808227539,
      "kl": 3.283203125,
      "learning_rate": 5.334027932479981e-08,
      "loss": 0.1859,
      "reward": 0.4765625298023224,
      "reward_std": 0.1326500754803419,
      "rewards/accuracy_reward": 0.04017857415601611,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4363839402794838,
      "step": 2709
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1746.6719360351562,
      "epoch": 0.8094989171831828,
      "grad_norm": 11.575340270996094,
      "kl": 2.869140625,
      "learning_rate": 5.317939186307435e-08,
      "loss": 0.1937,
      "reward": 0.5156250223517418,
      "reward_std": 0.13782999105751514,
      "rewards/accuracy_reward": 0.07589285937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.439732164144516,
      "step": 2710
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1828.8906860351562,
      "epoch": 0.8097976252707042,
      "grad_norm": 16.950571060180664,
      "kl": 4.99609375,
      "learning_rate": 5.301871852287995e-08,
      "loss": 0.2663,
      "reward": 0.431361623108387,
      "reward_std": 0.14767584018409252,
      "rewards/accuracy_reward": 0.020089286379516125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4112723395228386,
      "step": 2711
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1763.6295471191406,
      "epoch": 0.8100963333582257,
      "grad_norm": 3.0434694290161133,
      "kl": 3.646484375,
      "learning_rate": 5.2858259479013565e-08,
      "loss": 0.2056,
      "reward": 0.472098246216774,
      "reward_std": 0.1568599808961153,
      "rewards/accuracy_reward": 0.03794643119908869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341517984867096,
      "step": 2712
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1807.1116943359375,
      "epoch": 0.8103950414457471,
      "grad_norm": 34.010902404785156,
      "kl": 5.30078125,
      "learning_rate": 5.269801490603898e-08,
      "loss": 0.2875,
      "reward": 0.4570312798023224,
      "reward_std": 0.1838645115494728,
      "rewards/accuracy_reward": 0.0491071455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4079241305589676,
      "step": 2713
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1780.1228637695312,
      "epoch": 0.8106937495332686,
      "grad_norm": 4.481138706207275,
      "kl": 3.33984375,
      "learning_rate": 5.253798497828643e-08,
      "loss": 0.1995,
      "reward": 0.4838169813156128,
      "reward_std": 0.14566761441528797,
      "rewards/accuracy_reward": 0.05580357415601611,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428013414144516,
      "step": 2714
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1801.4644165039062,
      "epoch": 0.8109924576207901,
      "grad_norm": 6.651227951049805,
      "kl": 3.6484375,
      "learning_rate": 5.2378169869852854e-08,
      "loss": 0.2174,
      "reward": 0.5251116305589676,
      "reward_std": 0.18003006651997566,
      "rewards/accuracy_reward": 0.10714286053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4179687723517418,
      "step": 2715
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1728.3237609863281,
      "epoch": 0.8112911657083115,
      "grad_norm": 5.555882453918457,
      "kl": 4.171875,
      "learning_rate": 5.221856975460154e-08,
      "loss": 0.263,
      "reward": 0.5926339626312256,
      "reward_std": 0.13673717714846134,
      "rewards/accuracy_reward": 0.16071429592557251,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.431919664144516,
      "step": 2716
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1825.7857971191406,
      "epoch": 0.811589873795833,
      "grad_norm": 3.1283352375030518,
      "kl": 3.3828125,
      "learning_rate": 5.205918480616156e-08,
      "loss": 0.1969,
      "reward": 0.4676339477300644,
      "reward_std": 0.15566161088645458,
      "rewards/accuracy_reward": 0.046875002793967724,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4207589477300644,
      "step": 2717
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1734.7634582519531,
      "epoch": 0.8118885818833544,
      "grad_norm": 4.940022945404053,
      "kl": 3.25390625,
      "learning_rate": 5.190001519792825e-08,
      "loss": 0.189,
      "reward": 0.5055803880095482,
      "reward_std": 0.1674402318894863,
      "rewards/accuracy_reward": 0.07812500395812094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.427455373108387,
      "step": 2718
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1723.1630554199219,
      "epoch": 0.812187289970876,
      "grad_norm": 12.245909690856934,
      "kl": 2.671875,
      "learning_rate": 5.174106110306259e-08,
      "loss": 0.1627,
      "reward": 0.5357143059372902,
      "reward_std": 0.17117277346551418,
      "rewards/accuracy_reward": 0.10044643399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4352678805589676,
      "step": 2719
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1788.6139221191406,
      "epoch": 0.8124859980583974,
      "grad_norm": 3.9004311561584473,
      "kl": 3.390625,
      "learning_rate": 5.158232269449089e-08,
      "loss": 0.2017,
      "reward": 0.5764509215950966,
      "reward_std": 0.20167012326419353,
      "rewards/accuracy_reward": 0.1495535746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4268973395228386,
      "step": 2720
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1730.7969360351562,
      "epoch": 0.8127847061459189,
      "grad_norm": 8.614546775817871,
      "kl": 3.1328125,
      "learning_rate": 5.14238001449051e-08,
      "loss": 0.1972,
      "reward": 0.5000000223517418,
      "reward_std": 0.1376197785139084,
      "rewards/accuracy_reward": 0.05803571757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4419643059372902,
      "step": 2721
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1792.0581359863281,
      "epoch": 0.8130834142334403,
      "grad_norm": 12.848841667175293,
      "kl": 3.375,
      "learning_rate": 5.126549362676227e-08,
      "loss": 0.2104,
      "reward": 0.536830373108387,
      "reward_std": 0.1582099962979555,
      "rewards/accuracy_reward": 0.10491071827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.431919664144516,
      "step": 2722
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1760.4777526855469,
      "epoch": 0.8133821223209619,
      "grad_norm": 7.018321990966797,
      "kl": 3.08984375,
      "learning_rate": 5.110740331228422e-08,
      "loss": 0.1674,
      "reward": 0.5664062798023224,
      "reward_std": 0.1776423417031765,
      "rewards/accuracy_reward": 0.1339285783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.432477705180645,
      "step": 2723
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1660.5513916015625,
      "epoch": 0.8136808304084833,
      "grad_norm": 24.32977867126465,
      "kl": 4.1171875,
      "learning_rate": 5.094952937345787e-08,
      "loss": 0.2407,
      "reward": 0.5200893059372902,
      "reward_std": 0.15926779992878437,
      "rewards/accuracy_reward": 0.08258929010480642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000223517418,
      "step": 2724
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1794.9152526855469,
      "epoch": 0.8139795384960048,
      "grad_norm": 6.6254987716674805,
      "kl": 3.3984375,
      "learning_rate": 5.079187198203444e-08,
      "loss": 0.1889,
      "reward": 0.4430803805589676,
      "reward_std": 0.13320189528167248,
      "rewards/accuracy_reward": 0.01785714365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232387661934,
      "step": 2725
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1808.6875915527344,
      "epoch": 0.8142782465835262,
      "grad_norm": 7.362576484680176,
      "kl": 4.1953125,
      "learning_rate": 5.06344313095298e-08,
      "loss": 0.2413,
      "reward": 0.4966518059372902,
      "reward_std": 0.14513612538576126,
      "rewards/accuracy_reward": 0.08482143259607255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4118303805589676,
      "step": 2726
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1810.8639221191406,
      "epoch": 0.8145769546710477,
      "grad_norm": 9.73324966430664,
      "kl": 4.55859375,
      "learning_rate": 5.047720752722404e-08,
      "loss": 0.2525,
      "reward": 0.5145089477300644,
      "reward_std": 0.11431975848972797,
      "rewards/accuracy_reward": 0.11160714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4029017984867096,
      "step": 2727
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1762.24560546875,
      "epoch": 0.8148756627585692,
      "grad_norm": 12.345558166503906,
      "kl": 2.9296875,
      "learning_rate": 5.032020080616109e-08,
      "loss": 0.1764,
      "reward": 0.5184151977300644,
      "reward_std": 0.1498600821942091,
      "rewards/accuracy_reward": 0.08482143399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937723517418,
      "step": 2728
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1732.26123046875,
      "epoch": 0.8151743708460907,
      "grad_norm": 5.124916076660156,
      "kl": 4.0234375,
      "learning_rate": 5.0163411317148934e-08,
      "loss": 0.243,
      "reward": 0.4520089477300644,
      "reward_std": 0.11838055960834026,
      "rewards/accuracy_reward": 0.026785715017467737,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232313156128,
      "step": 2729
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1721.9844665527344,
      "epoch": 0.8154730789336121,
      "grad_norm": 15.696487426757812,
      "kl": 2.51171875,
      "learning_rate": 5.000683923075924e-08,
      "loss": 0.1662,
      "reward": 0.5329241380095482,
      "reward_std": 0.13920924719423056,
      "rewards/accuracy_reward": 0.10044643306173384,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776977300644,
      "step": 2730
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1908.6161499023438,
      "epoch": 0.8157717870211336,
      "grad_norm": 6.437333583831787,
      "kl": 3.548828125,
      "learning_rate": 4.9850484717326946e-08,
      "loss": 0.1893,
      "reward": 0.4213169813156128,
      "reward_std": 0.1636672541499138,
      "rewards/accuracy_reward": 0.024553572526201606,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3967634066939354,
      "step": 2731
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1783.9688110351562,
      "epoch": 0.816070495108655,
      "grad_norm": 19.7557430267334,
      "kl": 2.359375,
      "learning_rate": 4.9694347946950545e-08,
      "loss": 0.1494,
      "reward": 0.5368303954601288,
      "reward_std": 0.19813701510429382,
      "rewards/accuracy_reward": 0.098214291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4386160969734192,
      "step": 2732
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1766.6406860351562,
      "epoch": 0.8163692031961766,
      "grad_norm": 13.689188003540039,
      "kl": 3.04296875,
      "learning_rate": 4.9538429089491576e-08,
      "loss": 0.1847,
      "reward": 0.4854910895228386,
      "reward_std": 0.13879569806158543,
      "rewards/accuracy_reward": 0.06696428917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268059372902,
      "step": 2733
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1695.8014221191406,
      "epoch": 0.816667911283698,
      "grad_norm": 24.04045867919922,
      "kl": 2.02734375,
      "learning_rate": 4.9382728314574324e-08,
      "loss": 0.1453,
      "reward": 0.5463169813156128,
      "reward_std": 0.151080759242177,
      "rewards/accuracy_reward": 0.09598214738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4503348469734192,
      "step": 2734
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1822.2679443359375,
      "epoch": 0.8169666193712195,
      "grad_norm": 21.430410385131836,
      "kl": 2.912109375,
      "learning_rate": 4.9227245791586075e-08,
      "loss": 0.169,
      "reward": 0.5044643133878708,
      "reward_std": 0.16666945070028305,
      "rewards/accuracy_reward": 0.07142857508733869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357313156128,
      "step": 2735
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1820.8907165527344,
      "epoch": 0.8172653274587409,
      "grad_norm": 14.485987663269043,
      "kl": 2.98046875,
      "learning_rate": 4.9071981689676634e-08,
      "loss": 0.164,
      "reward": 0.5290178880095482,
      "reward_std": 0.16725178435444832,
      "rewards/accuracy_reward": 0.08705357578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4419643059372902,
      "step": 2736
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1694.6675109863281,
      "epoch": 0.8175640355462624,
      "grad_norm": 6.408156394958496,
      "kl": 3.6796875,
      "learning_rate": 4.8916936177758e-08,
      "loss": 0.2245,
      "reward": 0.4419643059372902,
      "reward_std": 0.11656048893928528,
      "rewards/accuracy_reward": 0.0133928582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 2737
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1760.5848999023438,
      "epoch": 0.8178627436337839,
      "grad_norm": 5.579799175262451,
      "kl": 3.4609375,
      "learning_rate": 4.8762109424504566e-08,
      "loss": 0.188,
      "reward": 0.5066964477300644,
      "reward_std": 0.1982294637709856,
      "rewards/accuracy_reward": 0.0669642873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.439732164144516,
      "step": 2738
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1786.9978332519531,
      "epoch": 0.8181614517213054,
      "grad_norm": 5.808907508850098,
      "kl": 4.00390625,
      "learning_rate": 4.8607501598352614e-08,
      "loss": 0.2357,
      "reward": 0.5239955484867096,
      "reward_std": 0.14542509242892265,
      "rewards/accuracy_reward": 0.08928571990691125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098395228386,
      "step": 2739
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1867.72998046875,
      "epoch": 0.8184601598088268,
      "grad_norm": 18.80755615234375,
      "kl": 4.4140625,
      "learning_rate": 4.845311286750034e-08,
      "loss": 0.232,
      "reward": 0.499441996216774,
      "reward_std": 0.16835850104689598,
      "rewards/accuracy_reward": 0.0848214328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205484867096,
      "step": 2740
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1693.1473999023438,
      "epoch": 0.8187588678963483,
      "grad_norm": 6.932196617126465,
      "kl": 4.265625,
      "learning_rate": 4.829894339990759e-08,
      "loss": 0.2736,
      "reward": 0.4977678805589676,
      "reward_std": 0.11465507559478283,
      "rewards/accuracy_reward": 0.06696428847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035895228386,
      "step": 2741
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1730.8282165527344,
      "epoch": 0.8190575759838697,
      "grad_norm": 14.599860191345215,
      "kl": 3.203125,
      "learning_rate": 4.8144993363295554e-08,
      "loss": 0.1721,
      "reward": 0.4654018059372902,
      "reward_std": 0.1146368533372879,
      "rewards/accuracy_reward": 0.020089287078008056,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125223517418,
      "step": 2742
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1799.7210693359375,
      "epoch": 0.8193562840713913,
      "grad_norm": 12.48774528503418,
      "kl": 4.96875,
      "learning_rate": 4.7991262925146796e-08,
      "loss": 0.2771,
      "reward": 0.4877232387661934,
      "reward_std": 0.14502577856183052,
      "rewards/accuracy_reward": 0.06250000116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232387661934,
      "step": 2743
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1864.7991943359375,
      "epoch": 0.8196549921589127,
      "grad_norm": 15.269882202148438,
      "kl": 4.9453125,
      "learning_rate": 4.783775225270506e-08,
      "loss": 0.262,
      "reward": 0.4620535895228386,
      "reward_std": 0.12518161162734032,
      "rewards/accuracy_reward": 0.05580357392318547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500149011612,
      "step": 2744
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1771.19873046875,
      "epoch": 0.8199537002464342,
      "grad_norm": 7.249768257141113,
      "kl": 3.34765625,
      "learning_rate": 4.768446151297478e-08,
      "loss": 0.2055,
      "reward": 0.493861623108387,
      "reward_std": 0.17075259797275066,
      "rewards/accuracy_reward": 0.06919643236324191,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4246651977300644,
      "step": 2745
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1720.6853637695312,
      "epoch": 0.8202524083339556,
      "grad_norm": 10.755812644958496,
      "kl": 2.3515625,
      "learning_rate": 4.753139087272134e-08,
      "loss": 0.1564,
      "reward": 0.5753348395228386,
      "reward_std": 0.16246528550982475,
      "rewards/accuracy_reward": 0.13392858020961285,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4414062723517418,
      "step": 2746
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1733.08935546875,
      "epoch": 0.8205511164214772,
      "grad_norm": 26.904523849487305,
      "kl": 2.625,
      "learning_rate": 4.737854049847062e-08,
      "loss": 0.1824,
      "reward": 0.502232164144516,
      "reward_std": 0.13225445337593555,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4397321566939354,
      "step": 2747
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1750.7969360351562,
      "epoch": 0.8208498245089986,
      "grad_norm": 8.477344512939453,
      "kl": 3.41796875,
      "learning_rate": 4.722591055650873e-08,
      "loss": 0.2192,
      "reward": 0.5106026977300644,
      "reward_std": 0.19650974869728088,
      "rewards/accuracy_reward": 0.08705357392318547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.423549123108387,
      "step": 2748
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1760.3840026855469,
      "epoch": 0.8211485325965201,
      "grad_norm": 3.753859519958496,
      "kl": 3.125,
      "learning_rate": 4.7073501212882175e-08,
      "loss": 0.1788,
      "reward": 0.4642857387661934,
      "reward_std": 0.16779973357915878,
      "rewards/accuracy_reward": 0.04241071594879031,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 2749
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1737.915283203125,
      "epoch": 0.8214472406840415,
      "grad_norm": 19.342784881591797,
      "kl": 3.30078125,
      "learning_rate": 4.6921312633397435e-08,
      "loss": 0.2043,
      "reward": 0.4330357387661934,
      "reward_std": 0.10739328898489475,
      "rewards/accuracy_reward": 0.004464285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 2750
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1728.9822387695312,
      "epoch": 0.821745948771563,
      "grad_norm": 21.5251522064209,
      "kl": 2.783203125,
      "learning_rate": 4.676934498362062e-08,
      "loss": 0.1855,
      "reward": 0.5691964626312256,
      "reward_std": 0.16651872545480728,
      "rewards/accuracy_reward": 0.1205357201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4486607313156128,
      "step": 2751
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1762.44873046875,
      "epoch": 0.8220446568590845,
      "grad_norm": 10.28956413269043,
      "kl": 3.09375,
      "learning_rate": 4.66175984288778e-08,
      "loss": 0.1942,
      "reward": 0.4531250223517418,
      "reward_std": 0.1373790167272091,
      "rewards/accuracy_reward": 0.024553571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 2752
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1811.2322387695312,
      "epoch": 0.822343364946606,
      "grad_norm": 4.028258323669434,
      "kl": 4.015625,
      "learning_rate": 4.646607313425419e-08,
      "loss": 0.2332,
      "reward": 0.482700914144516,
      "reward_std": 0.14489728212356567,
      "rewards/accuracy_reward": 0.05803571571595967,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4246651977300644,
      "step": 2753
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1698.6808776855469,
      "epoch": 0.8226420730341274,
      "grad_norm": 9.155792236328125,
      "kl": 4.0703125,
      "learning_rate": 4.631476926459454e-08,
      "loss": 0.2402,
      "reward": 0.4670759066939354,
      "reward_std": 0.14625351503491402,
      "rewards/accuracy_reward": 0.037946430733427405,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294887661934,
      "step": 2754
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1766.26123046875,
      "epoch": 0.8229407811216489,
      "grad_norm": 18.793636322021484,
      "kl": 4.75390625,
      "learning_rate": 4.616368698450271e-08,
      "loss": 0.246,
      "reward": 0.5463169887661934,
      "reward_std": 0.17295417562127113,
      "rewards/accuracy_reward": 0.1361607201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4101562723517418,
      "step": 2755
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1757.1116943359375,
      "epoch": 0.8232394892091703,
      "grad_norm": 12.060697555541992,
      "kl": 4.03125,
      "learning_rate": 4.601282645834126e-08,
      "loss": 0.2253,
      "reward": 0.486049123108387,
      "reward_std": 0.18290539272129536,
      "rewards/accuracy_reward": 0.058035715483129025,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428013414144516,
      "step": 2756
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1816.8215026855469,
      "epoch": 0.8235381972966918,
      "grad_norm": 10.645362854003906,
      "kl": 3.12890625,
      "learning_rate": 4.5862187850231715e-08,
      "loss": 0.1804,
      "reward": 0.5189732387661934,
      "reward_std": 0.17457383126020432,
      "rewards/accuracy_reward": 0.09821428917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4207589477300644,
      "step": 2757
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1682.3281860351562,
      "epoch": 0.8238369053842133,
      "grad_norm": 8.564470291137695,
      "kl": 3.83203125,
      "learning_rate": 4.571177132405421e-08,
      "loss": 0.2386,
      "reward": 0.4888393059372902,
      "reward_std": 0.17207377217710018,
      "rewards/accuracy_reward": 0.0513392873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000223517418,
      "step": 2758
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1824.087158203125,
      "epoch": 0.8241356134717347,
      "grad_norm": 9.462615966796875,
      "kl": 3.92578125,
      "learning_rate": 4.556157704344707e-08,
      "loss": 0.2122,
      "reward": 0.489397332072258,
      "reward_std": 0.15512429550290108,
      "rewards/accuracy_reward": 0.06919643143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.420200914144516,
      "step": 2759
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1709.977783203125,
      "epoch": 0.8244343215592562,
      "grad_norm": 13.1007661819458,
      "kl": 3.0234375,
      "learning_rate": 4.541160517180695e-08,
      "loss": 0.1915,
      "reward": 0.5915178954601288,
      "reward_std": 0.17268083058297634,
      "rewards/accuracy_reward": 0.1540178619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000223517418,
      "step": 2760
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1849.9264221191406,
      "epoch": 0.8247330296467776,
      "grad_norm": 5.7390265464782715,
      "kl": 3.7578125,
      "learning_rate": 4.526185587228867e-08,
      "loss": 0.1996,
      "reward": 0.4508928880095482,
      "reward_std": 0.18440180644392967,
      "rewards/accuracy_reward": 0.03348214481957257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4174107387661934,
      "step": 2761
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1797.2054443359375,
      "epoch": 0.8250317377342992,
      "grad_norm": 4.378124237060547,
      "kl": 4.25390625,
      "learning_rate": 4.511232930780465e-08,
      "loss": 0.2535,
      "reward": 0.4860491305589676,
      "reward_std": 0.12500005587935448,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205559372902,
      "step": 2762
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1779.680908203125,
      "epoch": 0.8253304458218206,
      "grad_norm": 2.2467453479766846,
      "kl": 2.603515625,
      "learning_rate": 4.496302564102522e-08,
      "loss": 0.1524,
      "reward": 0.4916294813156128,
      "reward_std": 0.20204727724194527,
      "rewards/accuracy_reward": 0.05803571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937798023224,
      "step": 2763
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1822.1451416015625,
      "epoch": 0.8256291539093421,
      "grad_norm": 13.633927345275879,
      "kl": 2.81640625,
      "learning_rate": 4.4813945034378075e-08,
      "loss": 0.1703,
      "reward": 0.5613839402794838,
      "reward_std": 0.1623811051249504,
      "rewards/accuracy_reward": 0.1339285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4274553805589676,
      "step": 2764
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1785.76123046875,
      "epoch": 0.8259278619968635,
      "grad_norm": 18.230432510375977,
      "kl": 2.8984375,
      "learning_rate": 4.4665087650048336e-08,
      "loss": 0.1808,
      "reward": 0.4799107387661934,
      "reward_std": 0.13493396900594234,
      "rewards/accuracy_reward": 0.051339288242161274,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428571455180645,
      "step": 2765
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1704.9197387695312,
      "epoch": 0.826226570084385,
      "grad_norm": 5.058344841003418,
      "kl": 3.64453125,
      "learning_rate": 4.4516453649978304e-08,
      "loss": 0.2266,
      "reward": 0.5011161044239998,
      "reward_std": 0.16021873615682125,
      "rewards/accuracy_reward": 0.08258928917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185267984867096,
      "step": 2766
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1856.7590026855469,
      "epoch": 0.8265252781719065,
      "grad_norm": 20.13213539123535,
      "kl": 4.75,
      "learning_rate": 4.436804319586709e-08,
      "loss": 0.2462,
      "reward": 0.428013414144516,
      "reward_std": 0.1505781952291727,
      "rewards/accuracy_reward": 0.022321429336443543,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4056919813156128,
      "step": 2767
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1811.3193054199219,
      "epoch": 0.826823986259428,
      "grad_norm": 4.587947368621826,
      "kl": 3.54296875,
      "learning_rate": 4.421985644917078e-08,
      "loss": 0.2035,
      "reward": 0.4687500223517418,
      "reward_std": 0.19177458062767982,
      "rewards/accuracy_reward": 0.053571431431919336,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4151785895228386,
      "step": 2768
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1758.3393859863281,
      "epoch": 0.8271226943469494,
      "grad_norm": 15.049426078796387,
      "kl": 3.84765625,
      "learning_rate": 4.4071893571102106e-08,
      "loss": 0.2578,
      "reward": 0.5803571715950966,
      "reward_std": 0.1423000916838646,
      "rewards/accuracy_reward": 0.15848214784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 2769
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1789.8193054199219,
      "epoch": 0.8274214024344709,
      "grad_norm": 12.692499160766602,
      "kl": 4.83984375,
      "learning_rate": 4.392415472263003e-08,
      "loss": 0.2733,
      "reward": 0.5178571790456772,
      "reward_std": 0.16736486367881298,
      "rewards/accuracy_reward": 0.1049107164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4129464477300644,
      "step": 2770
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1776.5447082519531,
      "epoch": 0.8277201105219923,
      "grad_norm": 8.108077049255371,
      "kl": 4.140625,
      "learning_rate": 4.3776640064480006e-08,
      "loss": 0.2426,
      "reward": 0.4877232313156128,
      "reward_std": 0.17130259796977043,
      "rewards/accuracy_reward": 0.07142857578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4162946566939354,
      "step": 2771
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1760.24560546875,
      "epoch": 0.8280188186095139,
      "grad_norm": 10.073029518127441,
      "kl": 4.69140625,
      "learning_rate": 4.362934975713362e-08,
      "loss": 0.2737,
      "reward": 0.4860491305589676,
      "reward_std": 0.19546386413276196,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4235491305589676,
      "step": 2772
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1746.1741943359375,
      "epoch": 0.8283175266970353,
      "grad_norm": 5.034422874450684,
      "kl": 3.4921875,
      "learning_rate": 4.348228396082809e-08,
      "loss": 0.199,
      "reward": 0.6054687649011612,
      "reward_std": 0.23008305206894875,
      "rewards/accuracy_reward": 0.1763392947614193,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294813156128,
      "step": 2773
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1800.4644165039062,
      "epoch": 0.8286162347845568,
      "grad_norm": 6.503471374511719,
      "kl": 2.771484375,
      "learning_rate": 4.3335442835556735e-08,
      "loss": 0.1608,
      "reward": 0.5022321566939354,
      "reward_std": 0.15359157510101795,
      "rewards/accuracy_reward": 0.07589286006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263392984867096,
      "step": 2774
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1776.6005554199219,
      "epoch": 0.8289149428720782,
      "grad_norm": 6.779937744140625,
      "kl": 2.984375,
      "learning_rate": 4.318882654106834e-08,
      "loss": 0.1681,
      "reward": 0.5005580484867096,
      "reward_std": 0.1902486328035593,
      "rewards/accuracy_reward": 0.06919643096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4313616305589676,
      "step": 2775
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1700.7991943359375,
      "epoch": 0.8292136509595998,
      "grad_norm": 8.174077987670898,
      "kl": 3.84375,
      "learning_rate": 4.3042435236866906e-08,
      "loss": 0.224,
      "reward": 0.6004464477300644,
      "reward_std": 0.1744564939290285,
      "rewards/accuracy_reward": 0.17410715110599995,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263393059372902,
      "step": 2776
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1752.5960388183594,
      "epoch": 0.8295123590471212,
      "grad_norm": 7.453596591949463,
      "kl": 3.22265625,
      "learning_rate": 4.2896269082212e-08,
      "loss": 0.196,
      "reward": 0.5200893133878708,
      "reward_std": 0.1831824593245983,
      "rewards/accuracy_reward": 0.08705357671715319,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357387661934,
      "step": 2777
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1715.3795471191406,
      "epoch": 0.8298110671346427,
      "grad_norm": 9.621438026428223,
      "kl": 3.1953125,
      "learning_rate": 4.2750328236117906e-08,
      "loss": 0.1982,
      "reward": 0.6339286044239998,
      "reward_std": 0.18803077191114426,
      "rewards/accuracy_reward": 0.1941964328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4397321566939354,
      "step": 2778
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1774.2054443359375,
      "epoch": 0.8301097752221641,
      "grad_norm": 7.126408576965332,
      "kl": 3.06640625,
      "learning_rate": 4.2604612857354036e-08,
      "loss": 0.1858,
      "reward": 0.475446455180645,
      "reward_std": 0.16569902002811432,
      "rewards/accuracy_reward": 0.03794643096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000223517418,
      "step": 2779
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1811.5000915527344,
      "epoch": 0.8304084833096856,
      "grad_norm": 23.061695098876953,
      "kl": 4.48046875,
      "learning_rate": 4.245912310444452e-08,
      "loss": 0.2537,
      "reward": 0.4458705559372902,
      "reward_std": 0.18903403356671333,
      "rewards/accuracy_reward": 0.04241071571595967,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4034598395228386,
      "step": 2780
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1842.8125915527344,
      "epoch": 0.8307071913972071,
      "grad_norm": 6.052119731903076,
      "kl": 3.13671875,
      "learning_rate": 4.231385913566782e-08,
      "loss": 0.1792,
      "reward": 0.522879496216774,
      "reward_std": 0.16933592408895493,
      "rewards/accuracy_reward": 0.09598214668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4268973395228386,
      "step": 2781
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1835.8415832519531,
      "epoch": 0.8310058994847286,
      "grad_norm": 7.794976711273193,
      "kl": 3.8828125,
      "learning_rate": 4.2168821109056926e-08,
      "loss": 0.2124,
      "reward": 0.4709821715950966,
      "reward_std": 0.15543937124311924,
      "rewards/accuracy_reward": 0.05357143119908869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4174107387661934,
      "step": 2782
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1753.6050109863281,
      "epoch": 0.83130460757225,
      "grad_norm": 8.775994300842285,
      "kl": 4.0234375,
      "learning_rate": 4.202400918239904e-08,
      "loss": 0.2569,
      "reward": 0.5429687723517418,
      "reward_std": 0.14894486032426357,
      "rewards/accuracy_reward": 0.1183035783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4246651977300644,
      "step": 2783
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1803.9755249023438,
      "epoch": 0.8316033156597715,
      "grad_norm": 10.189074516296387,
      "kl": 4.318359375,
      "learning_rate": 4.1879423513235315e-08,
      "loss": 0.2311,
      "reward": 0.4447544813156128,
      "reward_std": 0.12989217415452003,
      "rewards/accuracy_reward": 0.0223214291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330484867096,
      "step": 2784
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1739.3460693359375,
      "epoch": 0.8319020237472929,
      "grad_norm": 6.270489692687988,
      "kl": 3.0,
      "learning_rate": 4.173506425886086e-08,
      "loss": 0.1655,
      "reward": 0.5781250298023224,
      "reward_std": 0.14418099261820316,
      "rewards/accuracy_reward": 0.1361607201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4419643059372902,
      "step": 2785
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1716.6697692871094,
      "epoch": 0.8322007318348145,
      "grad_norm": 8.07413101196289,
      "kl": 4.03125,
      "learning_rate": 4.159093157632429e-08,
      "loss": 0.2421,
      "reward": 0.5011160895228386,
      "reward_std": 0.17074968665838242,
      "rewards/accuracy_reward": 0.07142857508733869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875223517418,
      "step": 2786
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1861.7702026367188,
      "epoch": 0.8324994399223359,
      "grad_norm": 14.408561706542969,
      "kl": 4.7265625,
      "learning_rate": 4.144702562242785e-08,
      "loss": 0.2497,
      "reward": 0.4843750149011612,
      "reward_std": 0.16984355077147484,
      "rewards/accuracy_reward": 0.08258928963914514,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4017857313156128,
      "step": 2787
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1718.3036499023438,
      "epoch": 0.8327981480098574,
      "grad_norm": 4.0564141273498535,
      "kl": 3.41796875,
      "learning_rate": 4.130334655372722e-08,
      "loss": 0.1969,
      "reward": 0.487723246216774,
      "reward_std": 0.14724065363407135,
      "rewards/accuracy_reward": 0.06919643003493547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268059372902,
      "step": 2788
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1767.4732971191406,
      "epoch": 0.8330968560973788,
      "grad_norm": 13.340109825134277,
      "kl": 3.0,
      "learning_rate": 4.115989452653101e-08,
      "loss": 0.1906,
      "reward": 0.5005580559372902,
      "reward_std": 0.15062959492206573,
      "rewards/accuracy_reward": 0.08035714458674192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4202009066939354,
      "step": 2789
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1713.8036804199219,
      "epoch": 0.8333955641849004,
      "grad_norm": 11.27281665802002,
      "kl": 3.28515625,
      "learning_rate": 4.101666969690104e-08,
      "loss": 0.2106,
      "reward": 0.5479911044239998,
      "reward_std": 0.15904320031404495,
      "rewards/accuracy_reward": 0.12053572107106447,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.427455373108387,
      "step": 2790
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.5804443359375,
      "epoch": 0.8336942722724218,
      "grad_norm": 7.325322151184082,
      "kl": 4.3203125,
      "learning_rate": 4.087367222065191e-08,
      "loss": 0.2469,
      "reward": 0.5111607313156128,
      "reward_std": 0.1578002106398344,
      "rewards/accuracy_reward": 0.1049107201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500149011612,
      "step": 2791
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1805.24560546875,
      "epoch": 0.8339929803599433,
      "grad_norm": 8.508357048034668,
      "kl": 3.8671875,
      "learning_rate": 4.073090225335077e-08,
      "loss": 0.1999,
      "reward": 0.4916294813156128,
      "reward_std": 0.16134170815348625,
      "rewards/accuracy_reward": 0.07142857438884676,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.420200914144516,
      "step": 2792
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1820.8237609863281,
      "epoch": 0.8342916884474647,
      "grad_norm": 6.702682018280029,
      "kl": 4.33203125,
      "learning_rate": 4.058835995031742e-08,
      "loss": 0.2384,
      "reward": 0.4838169887661934,
      "reward_std": 0.1821067165583372,
      "rewards/accuracy_reward": 0.06919643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205559372902,
      "step": 2793
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1753.2835693359375,
      "epoch": 0.8345903965349862,
      "grad_norm": 7.991125583648682,
      "kl": 3.328125,
      "learning_rate": 4.044604546662392e-08,
      "loss": 0.204,
      "reward": 0.550781287252903,
      "reward_std": 0.16032158955931664,
      "rewards/accuracy_reward": 0.12053572200238705,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455484867096,
      "step": 2794
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1772.5625610351562,
      "epoch": 0.8348891046225076,
      "grad_norm": 13.692147254943848,
      "kl": 2.90625,
      "learning_rate": 4.030395895709443e-08,
      "loss": 0.195,
      "reward": 0.5083705559372902,
      "reward_std": 0.14044577442109585,
      "rewards/accuracy_reward": 0.06919643213041127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4391741305589676,
      "step": 2795
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1813.9487609863281,
      "epoch": 0.8351878127100292,
      "grad_norm": 4.373557090759277,
      "kl": 3.44140625,
      "learning_rate": 4.016210057630523e-08,
      "loss": 0.1953,
      "reward": 0.502232164144516,
      "reward_std": 0.1702708825469017,
      "rewards/accuracy_reward": 0.07366071920841932,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428571455180645,
      "step": 2796
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1740.2947082519531,
      "epoch": 0.8354865207975506,
      "grad_norm": 12.968836784362793,
      "kl": 2.59375,
      "learning_rate": 4.00204704785842e-08,
      "loss": 0.1494,
      "reward": 0.4988839477300644,
      "reward_std": 0.17823892831802368,
      "rewards/accuracy_reward": 0.05580357275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443080373108387,
      "step": 2797
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.4085388183594,
      "epoch": 0.8357852288850721,
      "grad_norm": 18.100200653076172,
      "kl": 4.03515625,
      "learning_rate": 3.987906881801109e-08,
      "loss": 0.2161,
      "reward": 0.5558035969734192,
      "reward_std": 0.2106059268116951,
      "rewards/accuracy_reward": 0.1339285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750149011612,
      "step": 2798
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1752.9531860351562,
      "epoch": 0.8360839369725935,
      "grad_norm": 24.076189041137695,
      "kl": 4.9765625,
      "learning_rate": 3.97378957484171e-08,
      "loss": 0.2729,
      "reward": 0.6188616305589676,
      "reward_std": 0.15001879073679447,
      "rewards/accuracy_reward": 0.20312500931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.415736623108387,
      "step": 2799
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1757.7009887695312,
      "epoch": 0.836382645060115,
      "grad_norm": 25.359663009643555,
      "kl": 4.5859375,
      "learning_rate": 3.9596951423384596e-08,
      "loss": 0.2659,
      "reward": 0.4832589477300644,
      "reward_std": 0.19660427793860435,
      "rewards/accuracy_reward": 0.06919643143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625223517418,
      "step": 2800
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1767.0156860351562,
      "epoch": 0.8366813531476365,
      "grad_norm": 13.452777862548828,
      "kl": 4.53515625,
      "learning_rate": 3.94562359962472e-08,
      "loss": 0.2648,
      "reward": 0.718191996216774,
      "reward_std": 0.19460217095911503,
      "rewards/accuracy_reward": 0.2879464440047741,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455559372902,
      "step": 2801
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1863.6697692871094,
      "epoch": 0.8369800612351579,
      "grad_norm": 28.666614532470703,
      "kl": 4.2109375,
      "learning_rate": 3.93157496200896e-08,
      "loss": 0.2109,
      "reward": 0.5429687798023224,
      "reward_std": 0.16688236221671104,
      "rewards/accuracy_reward": 0.12946429220028222,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4135044813156128,
      "step": 2802
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1801.5000610351562,
      "epoch": 0.8372787693226794,
      "grad_norm": 4.676569938659668,
      "kl": 3.70703125,
      "learning_rate": 3.917549244774709e-08,
      "loss": 0.2008,
      "reward": 0.4893973469734192,
      "reward_std": 0.10330197587609291,
      "rewards/accuracy_reward": 0.07589286053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4135044813156128,
      "step": 2803
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1796.9732971191406,
      "epoch": 0.8375774774102008,
      "grad_norm": 9.16559886932373,
      "kl": 3.6953125,
      "learning_rate": 3.903546463180577e-08,
      "loss": 0.2251,
      "reward": 0.4408482313156128,
      "reward_std": 0.15753020904958248,
      "rewards/accuracy_reward": 0.033482145285233855,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4073660895228386,
      "step": 2804
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1743.8750610351562,
      "epoch": 0.8378761854977224,
      "grad_norm": 29.317298889160156,
      "kl": 2.93359375,
      "learning_rate": 3.889566632460223e-08,
      "loss": 0.1926,
      "reward": 0.5407366380095482,
      "reward_std": 0.14402149058878422,
      "rewards/accuracy_reward": 0.1116071492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294813156128,
      "step": 2805
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1784.9710388183594,
      "epoch": 0.8381748935852438,
      "grad_norm": 10.296802520751953,
      "kl": 3.52734375,
      "learning_rate": 3.8756097678223246e-08,
      "loss": 0.197,
      "reward": 0.521205373108387,
      "reward_std": 0.20996534451842308,
      "rewards/accuracy_reward": 0.11160714738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4095982313156128,
      "step": 2806
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1732.69873046875,
      "epoch": 0.8384736016727653,
      "grad_norm": 16.911495208740234,
      "kl": 3.23828125,
      "learning_rate": 3.8616758844505854e-08,
      "loss": 0.2049,
      "reward": 0.535156287252903,
      "reward_std": 0.14113422855734825,
      "rewards/accuracy_reward": 0.1116071455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4235491305589676,
      "step": 2807
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1731.1116638183594,
      "epoch": 0.8387723097602867,
      "grad_norm": 14.176715850830078,
      "kl": 2.4052734375,
      "learning_rate": 3.847764997503708e-08,
      "loss": 0.1675,
      "reward": 0.537388414144516,
      "reward_std": 0.18824812024831772,
      "rewards/accuracy_reward": 0.10267857275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098395228386,
      "step": 2808
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1775.8348999023438,
      "epoch": 0.8390710178478082,
      "grad_norm": 10.756903648376465,
      "kl": 3.265625,
      "learning_rate": 3.83387712211537e-08,
      "loss": 0.1964,
      "reward": 0.5111607387661934,
      "reward_std": 0.15385663695633411,
      "rewards/accuracy_reward": 0.08482143469154835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263393059372902,
      "step": 2809
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1758.2701416015625,
      "epoch": 0.8393697259353297,
      "grad_norm": 17.201772689819336,
      "kl": 3.79296875,
      "learning_rate": 3.820012273394224e-08,
      "loss": 0.2336,
      "reward": 0.5636160969734192,
      "reward_std": 0.14474987797439098,
      "rewards/accuracy_reward": 0.13839286286383867,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232387661934,
      "step": 2810
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1759.4465026855469,
      "epoch": 0.8396684340228512,
      "grad_norm": 11.889358520507812,
      "kl": 3.53125,
      "learning_rate": 3.806170466423855e-08,
      "loss": 0.1892,
      "reward": 0.5385044887661934,
      "reward_std": 0.20358259975910187,
      "rewards/accuracy_reward": 0.1071428619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4313616305589676,
      "step": 2811
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1690.9152526855469,
      "epoch": 0.8399671421103726,
      "grad_norm": 8.393016815185547,
      "kl": 3.30859375,
      "learning_rate": 3.792351716262804e-08,
      "loss": 0.2163,
      "reward": 0.5770089477300644,
      "reward_std": 0.12579200975596905,
      "rewards/accuracy_reward": 0.1406250074505806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4363839477300644,
      "step": 2812
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1777.9956359863281,
      "epoch": 0.8402658501978941,
      "grad_norm": 8.034296035766602,
      "kl": 4.0703125,
      "learning_rate": 3.778556037944516e-08,
      "loss": 0.2399,
      "reward": 0.4854910969734192,
      "reward_std": 0.11718124151229858,
      "rewards/accuracy_reward": 0.05357143096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4319196715950966,
      "step": 2813
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1811.7746276855469,
      "epoch": 0.8405645582854155,
      "grad_norm": 15.394757270812988,
      "kl": 4.9765625,
      "learning_rate": 3.764783446477329e-08,
      "loss": 0.2787,
      "reward": 0.4637276977300644,
      "reward_std": 0.12020159512758255,
      "rewards/accuracy_reward": 0.0491071455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205633878708,
      "step": 2814
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1747.3772888183594,
      "epoch": 0.8408632663729371,
      "grad_norm": 6.204975605010986,
      "kl": 3.70703125,
      "learning_rate": 3.7510339568444766e-08,
      "loss": 0.23,
      "reward": 0.4570312723517418,
      "reward_std": 0.13439291529357433,
      "rewards/accuracy_reward": 0.0267857164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455484867096,
      "step": 2815
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1736.1116943359375,
      "epoch": 0.8411619744604585,
      "grad_norm": 8.071277618408203,
      "kl": 4.376953125,
      "learning_rate": 3.7373075840040625e-08,
      "loss": 0.2444,
      "reward": 0.5892857313156128,
      "reward_std": 0.1566626150161028,
      "rewards/accuracy_reward": 0.1629464365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263393059372902,
      "step": 2816
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1754.7500610351562,
      "epoch": 0.84146068254798,
      "grad_norm": 21.16781997680664,
      "kl": 5.0234375,
      "learning_rate": 3.723604342889022e-08,
      "loss": 0.2835,
      "reward": 0.558035746216774,
      "reward_std": 0.15717510506510735,
      "rewards/accuracy_reward": 0.14285714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4151785895228386,
      "step": 2817
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1827.1161499023438,
      "epoch": 0.8417593906355014,
      "grad_norm": 26.719785690307617,
      "kl": 5.19140625,
      "learning_rate": 3.709924248407148e-08,
      "loss": 0.2564,
      "reward": 0.4453125149011612,
      "reward_std": 0.1519513614475727,
      "rewards/accuracy_reward": 0.026785715948790312,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185267984867096,
      "step": 2818
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1738.41748046875,
      "epoch": 0.842058098723023,
      "grad_norm": 6.47824764251709,
      "kl": 3.75390625,
      "learning_rate": 3.6962673154410457e-08,
      "loss": 0.2191,
      "reward": 0.4843750223517418,
      "reward_std": 0.11692117527127266,
      "rewards/accuracy_reward": 0.0491071455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4352678805589676,
      "step": 2819
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1782.1630249023438,
      "epoch": 0.8423568068105444,
      "grad_norm": 22.394481658935547,
      "kl": 4.875,
      "learning_rate": 3.682633558848108e-08,
      "loss": 0.2574,
      "reward": 0.4821428805589676,
      "reward_std": 0.15902639366686344,
      "rewards/accuracy_reward": 0.06919643515720963,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4129464477300644,
      "step": 2820
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1785.6719665527344,
      "epoch": 0.8426555148980659,
      "grad_norm": 25.251895904541016,
      "kl": 2.78125,
      "learning_rate": 3.66902299346053e-08,
      "loss": 0.1716,
      "reward": 0.5440848469734192,
      "reward_std": 0.1367931067943573,
      "rewards/accuracy_reward": 0.1026785783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4414062723517418,
      "step": 2821
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1811.1719665527344,
      "epoch": 0.8429542229855873,
      "grad_norm": 5.394992828369141,
      "kl": 4.23828125,
      "learning_rate": 3.6554356340852826e-08,
      "loss": 0.2367,
      "reward": 0.490513414144516,
      "reward_std": 0.1593309435993433,
      "rewards/accuracy_reward": 0.066964291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.423549123108387,
      "step": 2822
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1695.4576416015625,
      "epoch": 0.8432529310731088,
      "grad_norm": 15.126260757446289,
      "kl": 2.8359375,
      "learning_rate": 3.6418714955040644e-08,
      "loss": 0.1952,
      "reward": 0.628348246216774,
      "reward_std": 0.1594865694642067,
      "rewards/accuracy_reward": 0.18526786845177412,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4430803880095482,
      "step": 2823
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1687.1741638183594,
      "epoch": 0.8435516391606303,
      "grad_norm": 13.725850105285645,
      "kl": 2.72265625,
      "learning_rate": 3.628330592473347e-08,
      "loss": 0.1807,
      "reward": 0.565290205180645,
      "reward_std": 0.13737407699227333,
      "rewards/accuracy_reward": 0.1160714365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4492187723517418,
      "step": 2824
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1661.8639221191406,
      "epoch": 0.8438503472481518,
      "grad_norm": 27.825334548950195,
      "kl": 2.48828125,
      "learning_rate": 3.614812939724288e-08,
      "loss": 0.1973,
      "reward": 0.5982142984867096,
      "reward_std": 0.1384568866342306,
      "rewards/accuracy_reward": 0.1540178656578064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.444196455180645,
      "step": 2825
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1680.8371276855469,
      "epoch": 0.8441490553356732,
      "grad_norm": 8.884342193603516,
      "kl": 2.513671875,
      "learning_rate": 3.601318551962779e-08,
      "loss": 0.1639,
      "reward": 0.5167411044239998,
      "reward_std": 0.13020514883100986,
      "rewards/accuracy_reward": 0.082589291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341518133878708,
      "step": 2826
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1795.41748046875,
      "epoch": 0.8444477634231947,
      "grad_norm": 4.7163262367248535,
      "kl": 3.07421875,
      "learning_rate": 3.587847443869393e-08,
      "loss": 0.1752,
      "reward": 0.602678582072258,
      "reward_std": 0.24865194782614708,
      "rewards/accuracy_reward": 0.1674107238650322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435267873108387,
      "step": 2827
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1760.5134582519531,
      "epoch": 0.8447464715107161,
      "grad_norm": 18.098613739013672,
      "kl": 4.6015625,
      "learning_rate": 3.574399630099367e-08,
      "loss": 0.2707,
      "reward": 0.4843750223517418,
      "reward_std": 0.15744599886238575,
      "rewards/accuracy_reward": 0.06696428824216127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4174107387661934,
      "step": 2828
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1845.9063110351562,
      "epoch": 0.8450451795982377,
      "grad_norm": 17.663509368896484,
      "kl": 5.265625,
      "learning_rate": 3.560975125282609e-08,
      "loss": 0.2827,
      "reward": 0.5094866380095482,
      "reward_std": 0.1355590522289276,
      "rewards/accuracy_reward": 0.09598214784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4135044887661934,
      "step": 2829
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1727.9197387695312,
      "epoch": 0.8453438876857591,
      "grad_norm": 6.348749160766602,
      "kl": 3.4765625,
      "learning_rate": 3.5475739440236685e-08,
      "loss": 0.2117,
      "reward": 0.5479910895228386,
      "reward_std": 0.1451041642576456,
      "rewards/accuracy_reward": 0.11830357578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875223517418,
      "step": 2830
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1815.4085998535156,
      "epoch": 0.8456425957732806,
      "grad_norm": 3.880139112472534,
      "kl": 3.54296875,
      "learning_rate": 3.534196100901709e-08,
      "loss": 0.2059,
      "reward": 0.4720982313156128,
      "reward_std": 0.12230434268712997,
      "rewards/accuracy_reward": 0.04687500116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232313156128,
      "step": 2831
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1724.9688110351562,
      "epoch": 0.845941303860802,
      "grad_norm": 14.58448314666748,
      "kl": 4.65625,
      "learning_rate": 3.520841610470518e-08,
      "loss": 0.2733,
      "reward": 0.556919664144516,
      "reward_std": 0.17150320298969746,
      "rewards/accuracy_reward": 0.1473214365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4095982313156128,
      "step": 2832
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1770.3840026855469,
      "epoch": 0.8462400119483235,
      "grad_norm": 4.627825736999512,
      "kl": 4.23046875,
      "learning_rate": 3.507510487258475e-08,
      "loss": 0.2299,
      "reward": 0.5563616305589676,
      "reward_std": 0.14239880815148354,
      "rewards/accuracy_reward": 0.12723215040750802,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294813156128,
      "step": 2833
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1763.6005249023438,
      "epoch": 0.846538720035845,
      "grad_norm": 24.971054077148438,
      "kl": 5.4296875,
      "learning_rate": 3.494202745768526e-08,
      "loss": 0.3124,
      "reward": 0.4771205559372902,
      "reward_std": 0.19081398472189903,
      "rewards/accuracy_reward": 0.06696428917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4101562649011612,
      "step": 2834
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1818.9710693359375,
      "epoch": 0.8468374281233665,
      "grad_norm": 14.570950508117676,
      "kl": 4.47265625,
      "learning_rate": 3.480918400478194e-08,
      "loss": 0.2367,
      "reward": 0.5072544887661934,
      "reward_std": 0.1360940933227539,
      "rewards/accuracy_reward": 0.08705357578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.420200914144516,
      "step": 2835
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1762.0157165527344,
      "epoch": 0.8471361362108879,
      "grad_norm": 10.195008277893066,
      "kl": 3.46484375,
      "learning_rate": 3.4676574658395543e-08,
      "loss": 0.1947,
      "reward": 0.4458705484867096,
      "reward_std": 0.12237017042934895,
      "rewards/accuracy_reward": 0.015625000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455484867096,
      "step": 2836
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1820.0425109863281,
      "epoch": 0.8474348442984094,
      "grad_norm": 12.358247756958008,
      "kl": 4.20703125,
      "learning_rate": 3.4544199562791866e-08,
      "loss": 0.2186,
      "reward": 0.5530134215950966,
      "reward_std": 0.18798187002539635,
      "rewards/accuracy_reward": 0.12276785937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455484867096,
      "step": 2837
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1843.7009582519531,
      "epoch": 0.8477335523859308,
      "grad_norm": 9.467040061950684,
      "kl": 4.46484375,
      "learning_rate": 3.44120588619822e-08,
      "loss": 0.2407,
      "reward": 0.5770089402794838,
      "reward_std": 0.18057376332581043,
      "rewards/accuracy_reward": 0.1674107201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4095982313156128,
      "step": 2838
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1781.6741638183594,
      "epoch": 0.8480322604734524,
      "grad_norm": 4.170590877532959,
      "kl": 3.71875,
      "learning_rate": 3.428015269972254e-08,
      "loss": 0.1974,
      "reward": 0.573102705180645,
      "reward_std": 0.1913245003670454,
      "rewards/accuracy_reward": 0.1584821492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205559372902,
      "step": 2839
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1763.8728637695312,
      "epoch": 0.8483309685609738,
      "grad_norm": 4.940290451049805,
      "kl": 3.72265625,
      "learning_rate": 3.4148481219513916e-08,
      "loss": 0.2111,
      "reward": 0.5033482387661934,
      "reward_std": 0.17690503410995007,
      "rewards/accuracy_reward": 0.08482143096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185267984867096,
      "step": 2840
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1738.4777526855469,
      "epoch": 0.8486296766484953,
      "grad_norm": 14.314323425292969,
      "kl": 2.587890625,
      "learning_rate": 3.401704456460208e-08,
      "loss": 0.1491,
      "reward": 0.5680803805589676,
      "reward_std": 0.14791902713477612,
      "rewards/accuracy_reward": 0.13616072060540318,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4319196566939354,
      "step": 2841
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1750.9130249023438,
      "epoch": 0.8489283847360167,
      "grad_norm": 7.777742385864258,
      "kl": 3.05078125,
      "learning_rate": 3.388584287797708e-08,
      "loss": 0.1697,
      "reward": 0.5212053805589676,
      "reward_std": 0.12548474594950676,
      "rewards/accuracy_reward": 0.08705357508733869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341517984867096,
      "step": 2842
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1792.2322082519531,
      "epoch": 0.8492270928235381,
      "grad_norm": 10.728815078735352,
      "kl": 3.3046875,
      "learning_rate": 3.3754876302373584e-08,
      "loss": 0.1954,
      "reward": 0.5401786044239998,
      "reward_std": 0.15113292448222637,
      "rewards/accuracy_reward": 0.1116071492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714402794838,
      "step": 2843
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1740.18310546875,
      "epoch": 0.8495258009110597,
      "grad_norm": 14.258487701416016,
      "kl": 2.84765625,
      "learning_rate": 3.362414498027041e-08,
      "loss": 0.1908,
      "reward": 0.5703125223517418,
      "reward_std": 0.159937035292387,
      "rewards/accuracy_reward": 0.13169643515720963,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4386160895228386,
      "step": 2844
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1727.9420471191406,
      "epoch": 0.8498245089985811,
      "grad_norm": 9.724454879760742,
      "kl": 3.046875,
      "learning_rate": 3.349364905389032e-08,
      "loss": 0.1951,
      "reward": 0.4598214477300644,
      "reward_std": 0.16079909354448318,
      "rewards/accuracy_reward": 0.03348214505240321,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263393059372902,
      "step": 2845
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1811.3304443359375,
      "epoch": 0.8501232170861026,
      "grad_norm": 6.999212741851807,
      "kl": 4.05859375,
      "learning_rate": 3.336338866520019e-08,
      "loss": 0.245,
      "reward": 0.4475446715950966,
      "reward_std": 0.1660367138683796,
      "rewards/accuracy_reward": 0.026785715948790312,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.420758955180645,
      "step": 2846
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1747.8282165527344,
      "epoch": 0.850421925173624,
      "grad_norm": 6.5364837646484375,
      "kl": 3.38671875,
      "learning_rate": 3.323336395591056e-08,
      "loss": 0.2158,
      "reward": 0.5396205559372902,
      "reward_std": 0.21204089932143688,
      "rewards/accuracy_reward": 0.10267857578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419813156128,
      "step": 2847
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1749.9353637695312,
      "epoch": 0.8507206332611456,
      "grad_norm": 3.7056779861450195,
      "kl": 3.658203125,
      "learning_rate": 3.310357506747546e-08,
      "loss": 0.2161,
      "reward": 0.5479910969734192,
      "reward_std": 0.17284457385540009,
      "rewards/accuracy_reward": 0.11607143376022577,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4319196715950966,
      "step": 2848
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1706.5558776855469,
      "epoch": 0.851019341348667,
      "grad_norm": 10.318049430847168,
      "kl": 3.67578125,
      "learning_rate": 3.297402214109263e-08,
      "loss": 0.2286,
      "reward": 0.5664062649011612,
      "reward_std": 0.1484462097287178,
      "rewards/accuracy_reward": 0.12723215017467737,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4391741305589676,
      "step": 2849
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1779.9688110351562,
      "epoch": 0.8513180494361885,
      "grad_norm": 16.316965103149414,
      "kl": 3.81640625,
      "learning_rate": 3.284470531770281e-08,
      "loss": 0.1954,
      "reward": 0.5792410969734192,
      "reward_std": 0.1879999078810215,
      "rewards/accuracy_reward": 0.1562500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910969734192,
      "step": 2850
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1698.9465026855469,
      "epoch": 0.8516167575237099,
      "grad_norm": 11.02546215057373,
      "kl": 4.48828125,
      "learning_rate": 3.2715624737990114e-08,
      "loss": 0.2647,
      "reward": 0.6227678880095482,
      "reward_std": 0.16439899429678917,
      "rewards/accuracy_reward": 0.1897321529686451,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357313156128,
      "step": 2851
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1700.6563110351562,
      "epoch": 0.8519154656112314,
      "grad_norm": 14.703664779663086,
      "kl": 4.44140625,
      "learning_rate": 3.258678054238156e-08,
      "loss": 0.2763,
      "reward": 0.513950914144516,
      "reward_std": 0.16689816117286682,
      "rewards/accuracy_reward": 0.09151785890571773,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330484867096,
      "step": 2852
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1759.7388916015625,
      "epoch": 0.8522141736987529,
      "grad_norm": 8.985806465148926,
      "kl": 3.41796875,
      "learning_rate": 3.2458172871046993e-08,
      "loss": 0.1969,
      "reward": 0.5998884290456772,
      "reward_std": 0.15873457677662373,
      "rewards/accuracy_reward": 0.16294643841683865,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419813156128,
      "step": 2853
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1737.6116943359375,
      "epoch": 0.8525128817862744,
      "grad_norm": 7.6839470863342285,
      "kl": 3.392578125,
      "learning_rate": 3.2329801863898914e-08,
      "loss": 0.2112,
      "reward": 0.5133928880095482,
      "reward_std": 0.22248199954628944,
      "rewards/accuracy_reward": 0.07812500186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4352678805589676,
      "step": 2854
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1737.8013916015625,
      "epoch": 0.8528115898737958,
      "grad_norm": 11.61851692199707,
      "kl": 3.09375,
      "learning_rate": 3.2201667660592514e-08,
      "loss": 0.2034,
      "reward": 0.4994419887661934,
      "reward_std": 0.13821317628026009,
      "rewards/accuracy_reward": 0.06250000302679837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419887661934,
      "step": 2855
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1794.0782470703125,
      "epoch": 0.8531102979613173,
      "grad_norm": 5.741254806518555,
      "kl": 4.39453125,
      "learning_rate": 3.207377040052509e-08,
      "loss": 0.2476,
      "reward": 0.5664062649011612,
      "reward_std": 0.1770616639405489,
      "rewards/accuracy_reward": 0.15401786682195961,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.412388414144516,
      "step": 2856
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1815.8795471191406,
      "epoch": 0.8534090060488387,
      "grad_norm": 13.1563081741333,
      "kl": 4.5390625,
      "learning_rate": 3.194611022283642e-08,
      "loss": 0.2439,
      "reward": 0.4308035895228386,
      "reward_std": 0.1460218634456396,
      "rewards/accuracy_reward": 0.020089287078008056,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4107143059372902,
      "step": 2857
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1799.8326721191406,
      "epoch": 0.8537077141363603,
      "grad_norm": 18.441537857055664,
      "kl": 5.6953125,
      "learning_rate": 3.18186872664083e-08,
      "loss": 0.2954,
      "reward": 0.4575893059372902,
      "reward_std": 0.12582561746239662,
      "rewards/accuracy_reward": 0.0446428582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4129464402794838,
      "step": 2858
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1758.341552734375,
      "epoch": 0.8540064222238817,
      "grad_norm": 6.884352684020996,
      "kl": 3.1015625,
      "learning_rate": 3.169150166986429e-08,
      "loss": 0.1871,
      "reward": 0.5318080633878708,
      "reward_std": 0.14498044177889824,
      "rewards/accuracy_reward": 0.09821428963914514,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937723517418,
      "step": 2859
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.0536499023438,
      "epoch": 0.8543051303114032,
      "grad_norm": 6.953408718109131,
      "kl": 3.6796875,
      "learning_rate": 3.156455357156995e-08,
      "loss": 0.2183,
      "reward": 0.4972098469734192,
      "reward_std": 0.09995691105723381,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812649011612,
      "step": 2860
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1785.52685546875,
      "epoch": 0.8546038383989246,
      "grad_norm": 11.905069351196289,
      "kl": 4.20703125,
      "learning_rate": 3.143784310963241e-08,
      "loss": 0.2276,
      "reward": 0.4681919738650322,
      "reward_std": 0.13415836542844772,
      "rewards/accuracy_reward": 0.04910714388824999,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4190848395228386,
      "step": 2861
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1751.0134887695312,
      "epoch": 0.8549025464864461,
      "grad_norm": 27.463790893554688,
      "kl": 3.7578125,
      "learning_rate": 3.1311370421900104e-08,
      "loss": 0.241,
      "reward": 0.5329241380095482,
      "reward_std": 0.14076276309788227,
      "rewards/accuracy_reward": 0.1004464328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776977300644,
      "step": 2862
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1793.9933776855469,
      "epoch": 0.8552012545739676,
      "grad_norm": 11.871695518493652,
      "kl": 3.796875,
      "learning_rate": 3.118513564596306e-08,
      "loss": 0.2236,
      "reward": 0.4983259215950966,
      "reward_std": 0.17828672751784325,
      "rewards/accuracy_reward": 0.0848214328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4135044813156128,
      "step": 2863
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1809.4130249023438,
      "epoch": 0.8554999626614891,
      "grad_norm": 11.549400329589844,
      "kl": 3.748046875,
      "learning_rate": 3.105913891915221e-08,
      "loss": 0.2179,
      "reward": 0.5530134066939354,
      "reward_std": 0.19054916314780712,
      "rewards/accuracy_reward": 0.12723214644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812723517418,
      "step": 2864
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1831.8572387695312,
      "epoch": 0.8557986707490105,
      "grad_norm": 6.425656795501709,
      "kl": 5.0703125,
      "learning_rate": 3.093338037853971e-08,
      "loss": 0.2816,
      "reward": 0.482142873108387,
      "reward_std": 0.14176494255661964,
      "rewards/accuracy_reward": 0.08482143376022577,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3973214477300644,
      "step": 2865
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1749.2634887695312,
      "epoch": 0.856097378836532,
      "grad_norm": 16.93515968322754,
      "kl": 4.54296875,
      "learning_rate": 3.0807860160938585e-08,
      "loss": 0.2579,
      "reward": 0.4793526977300644,
      "reward_std": 0.14044139534235,
      "rewards/accuracy_reward": 0.06250000349245965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4168526977300644,
      "step": 2866
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1799.22998046875,
      "epoch": 0.8563960869240534,
      "grad_norm": 10.220853805541992,
      "kl": 5.01171875,
      "learning_rate": 3.0682578402902394e-08,
      "loss": 0.2834,
      "reward": 0.4933035969734192,
      "reward_std": 0.15539340302348137,
      "rewards/accuracy_reward": 0.06919643236324191,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.424107164144516,
      "step": 2867
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1763.6161804199219,
      "epoch": 0.856694795011575,
      "grad_norm": 7.331393718719482,
      "kl": 3.234375,
      "learning_rate": 3.055753524072549e-08,
      "loss": 0.2141,
      "reward": 0.6060268208384514,
      "reward_std": 0.15215518698096275,
      "rewards/accuracy_reward": 0.17857143399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4274553805589676,
      "step": 2868
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1775.1451721191406,
      "epoch": 0.8569935030990964,
      "grad_norm": 11.532556533813477,
      "kl": 3.38671875,
      "learning_rate": 3.043273081044262e-08,
      "loss": 0.2169,
      "reward": 0.5200893059372902,
      "reward_std": 0.16754340194165707,
      "rewards/accuracy_reward": 0.09151785913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 2869
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1804.7165832519531,
      "epoch": 0.8572922111866179,
      "grad_norm": 11.502321243286133,
      "kl": 4.51171875,
      "learning_rate": 3.030816524782867e-08,
      "loss": 0.2482,
      "reward": 0.4397321715950966,
      "reward_std": 0.15233152359724045,
      "rewards/accuracy_reward": 0.022321429569274187,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4174107387661934,
      "step": 2870
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1762.6473999023438,
      "epoch": 0.8575909192741393,
      "grad_norm": 18.157146453857422,
      "kl": 2.869140625,
      "learning_rate": 3.0183838688398826e-08,
      "loss": 0.1877,
      "reward": 0.5100446566939354,
      "reward_std": 0.1675581280142069,
      "rewards/accuracy_reward": 0.07589286006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341518059372902,
      "step": 2871
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.7567749023438,
      "epoch": 0.8578896273616609,
      "grad_norm": 11.073336601257324,
      "kl": 2.91015625,
      "learning_rate": 3.005975126740823e-08,
      "loss": 0.165,
      "reward": 0.571986623108387,
      "reward_std": 0.16309433057904243,
      "rewards/accuracy_reward": 0.13169643562287092,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.440290205180645,
      "step": 2872
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1818.5759582519531,
      "epoch": 0.8581883354491823,
      "grad_norm": 6.701773166656494,
      "kl": 3.693359375,
      "learning_rate": 2.993590311985175e-08,
      "loss": 0.2021,
      "reward": 0.5156250223517418,
      "reward_std": 0.15023531764745712,
      "rewards/accuracy_reward": 0.09151786146685481,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.424107164144516,
      "step": 2873
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1700.0781860351562,
      "epoch": 0.8584870435367038,
      "grad_norm": 11.576151847839355,
      "kl": 2.4609375,
      "learning_rate": 2.981229438046406e-08,
      "loss": 0.1528,
      "reward": 0.6099330633878708,
      "reward_std": 0.13702805899083614,
      "rewards/accuracy_reward": 0.16741072246804833,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4425223469734192,
      "step": 2874
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1724.3170471191406,
      "epoch": 0.8587857516242252,
      "grad_norm": 6.459187984466553,
      "kl": 3.1953125,
      "learning_rate": 2.968892518371946e-08,
      "loss": 0.2099,
      "reward": 0.5546875149011612,
      "reward_std": 0.1941454131156206,
      "rewards/accuracy_reward": 0.11160714738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443080373108387,
      "step": 2875
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1776.1719360351562,
      "epoch": 0.8590844597117467,
      "grad_norm": 8.698086738586426,
      "kl": 4.51171875,
      "learning_rate": 2.9565795663831357e-08,
      "loss": 0.2484,
      "reward": 0.4620535969734192,
      "reward_std": 0.15839644148945808,
      "rewards/accuracy_reward": 0.03348214481957257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428571455180645,
      "step": 2876
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1791.9353332519531,
      "epoch": 0.8593831677992682,
      "grad_norm": 24.997295379638672,
      "kl": 4.76953125,
      "learning_rate": 2.9442905954752768e-08,
      "loss": 0.2623,
      "reward": 0.5993303880095482,
      "reward_std": 0.18251998722553253,
      "rewards/accuracy_reward": 0.18526786006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625223517418,
      "step": 2877
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1822.5514221191406,
      "epoch": 0.8596818758867897,
      "grad_norm": 8.625914573669434,
      "kl": 4.06640625,
      "learning_rate": 2.932025619017553e-08,
      "loss": 0.2276,
      "reward": 0.5987723618745804,
      "reward_std": 0.19428959116339684,
      "rewards/accuracy_reward": 0.1785714402794838,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.420200914144516,
      "step": 2878
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1765.0402526855469,
      "epoch": 0.8599805839743111,
      "grad_norm": 9.86147689819336,
      "kl": 3.6171875,
      "learning_rate": 2.9197846503530632e-08,
      "loss": 0.1936,
      "reward": 0.4419643059372902,
      "reward_std": 0.12080085463821888,
      "rewards/accuracy_reward": 0.020089285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 2879
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1773.3817749023438,
      "epoch": 0.8602792920618326,
      "grad_norm": 14.128868103027344,
      "kl": 4.4921875,
      "learning_rate": 2.9075677027987826e-08,
      "loss": 0.2685,
      "reward": 0.4799107387661934,
      "reward_std": 0.15419276431202888,
      "rewards/accuracy_reward": 0.06919643259607255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4107142984867096,
      "step": 2880
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1748.8080749511719,
      "epoch": 0.860578000149354,
      "grad_norm": 6.208010673522949,
      "kl": 4.17578125,
      "learning_rate": 2.8953747896455465e-08,
      "loss": 0.2376,
      "reward": 0.4693080484867096,
      "reward_std": 0.16070923022925854,
      "rewards/accuracy_reward": 0.06250000488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4068080559372902,
      "step": 2881
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1847.79248046875,
      "epoch": 0.8608767082368756,
      "grad_norm": 3.9581429958343506,
      "kl": 4.9140625,
      "learning_rate": 2.883205924158055e-08,
      "loss": 0.2563,
      "reward": 0.412946455180645,
      "reward_std": 0.14471461437642574,
      "rewards/accuracy_reward": 0.013392857741564512,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3995535895228386,
      "step": 2882
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1755.2947082519531,
      "epoch": 0.861175416324397,
      "grad_norm": 9.693257331848145,
      "kl": 3.033203125,
      "learning_rate": 2.8710611195748414e-08,
      "loss": 0.189,
      "reward": 0.5680803805589676,
      "reward_std": 0.1811600998044014,
      "rewards/accuracy_reward": 0.1316964365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4363839477300644,
      "step": 2883
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1762.8661499023438,
      "epoch": 0.8614741244119185,
      "grad_norm": 13.252032279968262,
      "kl": 2.912109375,
      "learning_rate": 2.858940389108258e-08,
      "loss": 0.1773,
      "reward": 0.550223246216774,
      "reward_std": 0.1731241550296545,
      "rewards/accuracy_reward": 0.12276786146685481,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.427455373108387,
      "step": 2884
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1700.6630249023438,
      "epoch": 0.8617728324994399,
      "grad_norm": 9.025867462158203,
      "kl": 1.9765625,
      "learning_rate": 2.8468437459444717e-08,
      "loss": 0.1166,
      "reward": 0.5664062798023224,
      "reward_std": 0.23437238484621048,
      "rewards/accuracy_reward": 0.1205357201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4458705559372902,
      "step": 2885
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1798.5134887695312,
      "epoch": 0.8620715405869613,
      "grad_norm": 12.436566352844238,
      "kl": 3.51171875,
      "learning_rate": 2.834771203243455e-08,
      "loss": 0.2076,
      "reward": 0.431361623108387,
      "reward_std": 0.1361066121608019,
      "rewards/accuracy_reward": 0.01562500116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4157366305589676,
      "step": 2886
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1788.5693054199219,
      "epoch": 0.8623702486744829,
      "grad_norm": 6.05305290222168,
      "kl": 2.984375,
      "learning_rate": 2.822722774138936e-08,
      "loss": 0.1635,
      "reward": 0.502232164144516,
      "reward_std": 0.12791924364864826,
      "rewards/accuracy_reward": 0.08482143143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4174107313156128,
      "step": 2887
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1692.63623046875,
      "epoch": 0.8626689567620043,
      "grad_norm": 22.986787796020508,
      "kl": 2.8125,
      "learning_rate": 2.810698471738432e-08,
      "loss": 0.1917,
      "reward": 0.6216518133878708,
      "reward_std": 0.14770063944160938,
      "rewards/accuracy_reward": 0.1785714365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443080373108387,
      "step": 2888
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1770.7322082519531,
      "epoch": 0.8629676648495258,
      "grad_norm": 5.892788410186768,
      "kl": 3.8671875,
      "learning_rate": 2.7986983091232098e-08,
      "loss": 0.2131,
      "reward": 0.4581473395228386,
      "reward_std": 0.15718774870038033,
      "rewards/accuracy_reward": 0.040178573690354824,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4179687649011612,
      "step": 2889
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1766.7813110351562,
      "epoch": 0.8632663729370472,
      "grad_norm": 15.965812683105469,
      "kl": 2.9765625,
      "learning_rate": 2.7867222993482626e-08,
      "loss": 0.1681,
      "reward": 0.5306919887661934,
      "reward_std": 0.15369248017668724,
      "rewards/accuracy_reward": 0.10044643562287092,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455484867096,
      "step": 2890
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1797.2902526855469,
      "epoch": 0.8635650810245687,
      "grad_norm": 6.335684776306152,
      "kl": 3.974609375,
      "learning_rate": 2.774770455442324e-08,
      "loss": 0.2259,
      "reward": 0.5502232387661934,
      "reward_std": 0.18266365304589272,
      "rewards/accuracy_reward": 0.12276786658912897,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4274553805589676,
      "step": 2891
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1708.0134582519531,
      "epoch": 0.8638637891120902,
      "grad_norm": 11.674386024475098,
      "kl": 4.46875,
      "learning_rate": 2.7628427904078156e-08,
      "loss": 0.263,
      "reward": 0.4720982313156128,
      "reward_std": 0.1682984046638012,
      "rewards/accuracy_reward": 0.04017857392318547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.431919664144516,
      "step": 2892
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1746.9933776855469,
      "epoch": 0.8641624971996117,
      "grad_norm": 10.590655326843262,
      "kl": 4.69140625,
      "learning_rate": 2.7509393172208794e-08,
      "loss": 0.2688,
      "reward": 0.478794664144516,
      "reward_std": 0.20007147267460823,
      "rewards/accuracy_reward": 0.0558035746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 2893
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1807.8482971191406,
      "epoch": 0.8644612052871331,
      "grad_norm": 13.005739212036133,
      "kl": 4.26171875,
      "learning_rate": 2.7390600488313265e-08,
      "loss": 0.2341,
      "reward": 0.4347098395228386,
      "reward_std": 0.12821108847856522,
      "rewards/accuracy_reward": 0.015625000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4190848469734192,
      "step": 2894
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1756.9308776855469,
      "epoch": 0.8647599133746546,
      "grad_norm": 9.230589866638184,
      "kl": 2.9375,
      "learning_rate": 2.727204998162627e-08,
      "loss": 0.176,
      "reward": 0.6004464626312256,
      "reward_std": 0.20939970389008522,
      "rewards/accuracy_reward": 0.1540178656578064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4464285969734192,
      "step": 2895
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1810.5246276855469,
      "epoch": 0.865058621462176,
      "grad_norm": 13.729401588439941,
      "kl": 2.658203125,
      "learning_rate": 2.7153741781119232e-08,
      "loss": 0.1406,
      "reward": 0.490513414144516,
      "reward_std": 0.10783707350492477,
      "rewards/accuracy_reward": 0.0580357164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776977300644,
      "step": 2896
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1802.5425109863281,
      "epoch": 0.8653573295496976,
      "grad_norm": 11.336731910705566,
      "kl": 3.828125,
      "learning_rate": 2.7035676015499913e-08,
      "loss": 0.2178,
      "reward": 0.5245535895228386,
      "reward_std": 0.15631429478526115,
      "rewards/accuracy_reward": 0.10044643143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4241071566939354,
      "step": 2897
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1787.0001220703125,
      "epoch": 0.865656037637219,
      "grad_norm": 16.975807189941406,
      "kl": 4.53125,
      "learning_rate": 2.691785281321218e-08,
      "loss": 0.2668,
      "reward": 0.4933035895228386,
      "reward_std": 0.1127147227525711,
      "rewards/accuracy_reward": 0.07366071757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.419642873108387,
      "step": 2898
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1756.33935546875,
      "epoch": 0.8659547457247405,
      "grad_norm": 9.351993560791016,
      "kl": 3.51171875,
      "learning_rate": 2.6800272302436174e-08,
      "loss": 0.2228,
      "reward": 0.5675223544239998,
      "reward_std": 0.1866442058235407,
      "rewards/accuracy_reward": 0.129464291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4380580484867096,
      "step": 2899
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1708.5134582519531,
      "epoch": 0.8662534538122619,
      "grad_norm": 7.9692254066467285,
      "kl": 3.8515625,
      "learning_rate": 2.6682934611088032e-08,
      "loss": 0.2288,
      "reward": 0.5212053805589676,
      "reward_std": 0.15833407267928123,
      "rewards/accuracy_reward": 0.08928571990691125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.431919664144516,
      "step": 2900
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1715.5245971679688,
      "epoch": 0.8665521618997835,
      "grad_norm": 14.396929740905762,
      "kl": 4.49609375,
      "learning_rate": 2.6565839866819567e-08,
      "loss": 0.2526,
      "reward": 0.5295759215950966,
      "reward_std": 0.16348044201731682,
      "rewards/accuracy_reward": 0.1049107201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4246651977300644,
      "step": 2901
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1812.9375610351562,
      "epoch": 0.8668508699873049,
      "grad_norm": 10.073188781738281,
      "kl": 3.697265625,
      "learning_rate": 2.6448988197018434e-08,
      "loss": 0.2222,
      "reward": 0.493861623108387,
      "reward_std": 0.14102050475776196,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330559372902,
      "step": 2902
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1711.2143859863281,
      "epoch": 0.8671495780748264,
      "grad_norm": 8.379871368408203,
      "kl": 3.6015625,
      "learning_rate": 2.6332379728807806e-08,
      "loss": 0.235,
      "reward": 0.4860491305589676,
      "reward_std": 0.1665329448878765,
      "rewards/accuracy_reward": 0.05133928870782256,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098395228386,
      "step": 2903
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1742.7701721191406,
      "epoch": 0.8674482861623478,
      "grad_norm": 9.887556076049805,
      "kl": 4.150390625,
      "learning_rate": 2.6216014589046235e-08,
      "loss": 0.2633,
      "reward": 0.5089285895228386,
      "reward_std": 0.11807173304259777,
      "rewards/accuracy_reward": 0.082589291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263393059372902,
      "step": 2904
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1808.384033203125,
      "epoch": 0.8677469942498693,
      "grad_norm": 13.65540885925293,
      "kl": 4.48046875,
      "learning_rate": 2.6099892904327653e-08,
      "loss": 0.2657,
      "reward": 0.4659598395228386,
      "reward_std": 0.13506022468209267,
      "rewards/accuracy_reward": 0.04687500209547579,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.419084832072258,
      "step": 2905
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1735.9532165527344,
      "epoch": 0.8680457023373908,
      "grad_norm": 10.001546859741211,
      "kl": 3.91015625,
      "learning_rate": 2.5984014800980948e-08,
      "loss": 0.2438,
      "reward": 0.5323660895228386,
      "reward_std": 0.13430923223495483,
      "rewards/accuracy_reward": 0.09598214738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4363839477300644,
      "step": 2906
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1790.6630249023438,
      "epoch": 0.8683444104249123,
      "grad_norm": 23.342836380004883,
      "kl": 5.203125,
      "learning_rate": 2.5868380405070223e-08,
      "loss": 0.2715,
      "reward": 0.4860491305589676,
      "reward_std": 0.11992811970412731,
      "rewards/accuracy_reward": 0.0758928619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4101562723517418,
      "step": 2907
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1817.872802734375,
      "epoch": 0.8686431185124337,
      "grad_norm": 31.480501174926758,
      "kl": 6.1875,
      "learning_rate": 2.575298984239441e-08,
      "loss": 0.317,
      "reward": 0.4257812649011612,
      "reward_std": 0.1364359501749277,
      "rewards/accuracy_reward": 0.024553573224693537,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4012276977300644,
      "step": 2908
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1829.2121276855469,
      "epoch": 0.8689418265999552,
      "grad_norm": 24.802799224853516,
      "kl": 5.359375,
      "learning_rate": 2.5637843238487027e-08,
      "loss": 0.267,
      "reward": 0.5117187723517418,
      "reward_std": 0.18450995348393917,
      "rewards/accuracy_reward": 0.0892857164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330559372902,
      "step": 2909
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1787.22998046875,
      "epoch": 0.8692405346874766,
      "grad_norm": 7.211831569671631,
      "kl": 4.0703125,
      "learning_rate": 2.5522940718616354e-08,
      "loss": 0.2249,
      "reward": 0.4866071790456772,
      "reward_std": 0.13803520798683167,
      "rewards/accuracy_reward": 0.06250000349245965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4241071715950966,
      "step": 2910
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1766.055908203125,
      "epoch": 0.8695392427749982,
      "grad_norm": 10.406194686889648,
      "kl": 5.12890625,
      "learning_rate": 2.5408282407785085e-08,
      "loss": 0.2952,
      "reward": 0.4520089402794838,
      "reward_std": 0.13141670264303684,
      "rewards/accuracy_reward": 0.037946431431919336,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625149011612,
      "step": 2911
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1778.7991943359375,
      "epoch": 0.8698379508625196,
      "grad_norm": 3.437451124191284,
      "kl": 3.673828125,
      "learning_rate": 2.52938684307302e-08,
      "loss": 0.2092,
      "reward": 0.5066964402794838,
      "reward_std": 0.13929622434079647,
      "rewards/accuracy_reward": 0.07366071734577417,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357313156128,
      "step": 2912
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1801.1563415527344,
      "epoch": 0.8701366589500411,
      "grad_norm": 23.609464645385742,
      "kl": 5.4765625,
      "learning_rate": 2.5179698911922842e-08,
      "loss": 0.3168,
      "reward": 0.5167411044239998,
      "reward_std": 0.1806628257036209,
      "rewards/accuracy_reward": 0.1183035746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375223517418,
      "step": 2913
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1829.7388916015625,
      "epoch": 0.8704353670375625,
      "grad_norm": 17.717758178710938,
      "kl": 5.77734375,
      "learning_rate": 2.506577397556839e-08,
      "loss": 0.296,
      "reward": 0.487165205180645,
      "reward_std": 0.13587961718440056,
      "rewards/accuracy_reward": 0.08258928591385484,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.404575914144516,
      "step": 2914
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1782.0848999023438,
      "epoch": 0.870734075125084,
      "grad_norm": 89.76165771484375,
      "kl": 3.30859375,
      "learning_rate": 2.495209374560589e-08,
      "loss": 0.2081,
      "reward": 0.5736607536673546,
      "reward_std": 0.19215238094329834,
      "rewards/accuracy_reward": 0.1428571492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035895228386,
      "step": 2915
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1709.2166137695312,
      "epoch": 0.8710327832126055,
      "grad_norm": 3.1122586727142334,
      "kl": 2.77734375,
      "learning_rate": 2.4838658345708307e-08,
      "loss": 0.1557,
      "reward": 0.5150669813156128,
      "reward_std": 0.14744308032095432,
      "rewards/accuracy_reward": 0.08035714365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098395228386,
      "step": 2916
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1782.3125610351562,
      "epoch": 0.871331491300127,
      "grad_norm": 8.200718879699707,
      "kl": 2.6875,
      "learning_rate": 2.4725467899282315e-08,
      "loss": 0.1557,
      "reward": 0.561383955180645,
      "reward_std": 0.17512557283043861,
      "rewards/accuracy_reward": 0.13169643376022577,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875149011612,
      "step": 2917
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1821.0893859863281,
      "epoch": 0.8716301993876484,
      "grad_norm": 13.514201164245605,
      "kl": 2.96484375,
      "learning_rate": 2.461252252946791e-08,
      "loss": 0.1747,
      "reward": 0.4648437723517418,
      "reward_std": 0.1687844954431057,
      "rewards/accuracy_reward": 0.0491071455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4157366156578064,
      "step": 2918
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1734.6451721191406,
      "epoch": 0.8719289074751699,
      "grad_norm": 17.268817901611328,
      "kl": 2.423828125,
      "learning_rate": 2.449982235913872e-08,
      "loss": 0.1662,
      "reward": 0.5323660969734192,
      "reward_std": 0.20988789200782776,
      "rewards/accuracy_reward": 0.09375000465661287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4386160969734192,
      "step": 2919
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1888.1094665527344,
      "epoch": 0.8722276155626913,
      "grad_norm": 5.91530704498291,
      "kl": 2.869140625,
      "learning_rate": 2.4387367510901342e-08,
      "loss": 0.1492,
      "reward": 0.4453125149011612,
      "reward_std": 0.11745469272136688,
      "rewards/accuracy_reward": 0.0290178582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4162946566939354,
      "step": 2920
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1662.2411499023438,
      "epoch": 0.8725263236502129,
      "grad_norm": 20.541122436523438,
      "kl": 1.78515625,
      "learning_rate": 2.427515810709571e-08,
      "loss": 0.123,
      "reward": 0.5094866380095482,
      "reward_std": 0.12080631777644157,
      "rewards/accuracy_reward": 0.055803574388846755,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4536830559372902,
      "step": 2921
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1790.8482971191406,
      "epoch": 0.8728250317377343,
      "grad_norm": 14.063139915466309,
      "kl": 3.62109375,
      "learning_rate": 2.4163194269794686e-08,
      "loss": 0.2145,
      "reward": 0.5200893059372902,
      "reward_std": 0.15847296826541424,
      "rewards/accuracy_reward": 0.10714286286383867,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4129464477300644,
      "step": 2922
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1619.0625610351562,
      "epoch": 0.8731237398252558,
      "grad_norm": 18.12051010131836,
      "kl": 2.103515625,
      "learning_rate": 2.4051476120803883e-08,
      "loss": 0.1477,
      "reward": 0.5602678954601288,
      "reward_std": 0.13968565501272678,
      "rewards/accuracy_reward": 0.11160714644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4486607387661934,
      "step": 2923
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1784.4777526855469,
      "epoch": 0.8734224479127772,
      "grad_norm": 18.13631248474121,
      "kl": 2.9140625,
      "learning_rate": 2.3940003781661693e-08,
      "loss": 0.1683,
      "reward": 0.487723246216774,
      "reward_std": 0.16260675713419914,
      "rewards/accuracy_reward": 0.058035718742758036,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875223517418,
      "step": 2924
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1788.4532470703125,
      "epoch": 0.8737211560002988,
      "grad_norm": 4.15519905090332,
      "kl": 3.7265625,
      "learning_rate": 2.3828777373639196e-08,
      "loss": 0.2088,
      "reward": 0.4821428880095482,
      "reward_std": 0.15781277790665627,
      "rewards/accuracy_reward": 0.06250000209547579,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.419642873108387,
      "step": 2925
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1731.6764221191406,
      "epoch": 0.8740198640878202,
      "grad_norm": 11.197689056396484,
      "kl": 2.5078125,
      "learning_rate": 2.371779701773971e-08,
      "loss": 0.162,
      "reward": 0.5217634290456772,
      "reward_std": 0.17779066413640976,
      "rewards/accuracy_reward": 0.07589286006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4458705559372902,
      "step": 2926
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1761.0848999023438,
      "epoch": 0.8743185721753417,
      "grad_norm": 5.045070648193359,
      "kl": 3.57421875,
      "learning_rate": 2.3607062834699017e-08,
      "loss": 0.2252,
      "reward": 0.5401785969734192,
      "reward_std": 0.17989646829664707,
      "rewards/accuracy_reward": 0.10937500838190317,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035969734192,
      "step": 2927
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1755.65185546875,
      "epoch": 0.8746172802628631,
      "grad_norm": 14.022778511047363,
      "kl": 3.19921875,
      "learning_rate": 2.3496574944985136e-08,
      "loss": 0.2116,
      "reward": 0.4843750298023224,
      "reward_std": 0.1548378560692072,
      "rewards/accuracy_reward": 0.04241071711294353,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4419643059372902,
      "step": 2928
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1613.7300109863281,
      "epoch": 0.8749159883503845,
      "grad_norm": 8.716017723083496,
      "kl": 3.453125,
      "learning_rate": 2.3386333468797924e-08,
      "loss": 0.234,
      "reward": 0.5368303880095482,
      "reward_std": 0.17432688735425472,
      "rewards/accuracy_reward": 0.0959821455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4408482387661934,
      "step": 2929
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1815.9286499023438,
      "epoch": 0.8752146964379061,
      "grad_norm": 43.31866455078125,
      "kl": 5.8515625,
      "learning_rate": 2.3276338526069417e-08,
      "loss": 0.3139,
      "reward": 0.534598246216774,
      "reward_std": 0.16836016438901424,
      "rewards/accuracy_reward": 0.1316964365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4029017984867096,
      "step": 2930
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1798.8728637695312,
      "epoch": 0.8755134045254275,
      "grad_norm": 9.349672317504883,
      "kl": 4.46875,
      "learning_rate": 2.3166590236463253e-08,
      "loss": 0.248,
      "reward": 0.479352705180645,
      "reward_std": 0.1662589255720377,
      "rewards/accuracy_reward": 0.04687500116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776977300644,
      "step": 2931
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1771.3259582519531,
      "epoch": 0.875812112612949,
      "grad_norm": 45.313385009765625,
      "kl": 5.58203125,
      "learning_rate": 2.305708871937484e-08,
      "loss": 0.3028,
      "reward": 0.5664062723517418,
      "reward_std": 0.1782206017524004,
      "rewards/accuracy_reward": 0.1428571492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4235491305589676,
      "step": 2932
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1778.3951721191406,
      "epoch": 0.8761108207004704,
      "grad_norm": 22.255882263183594,
      "kl": 5.15234375,
      "learning_rate": 2.294783409393114e-08,
      "loss": 0.2913,
      "reward": 0.4760044813156128,
      "reward_std": 0.20515748113393784,
      "rewards/accuracy_reward": 0.06250000488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4135044813156128,
      "step": 2933
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1802.6116638183594,
      "epoch": 0.8764095287879919,
      "grad_norm": 19.747251510620117,
      "kl": 4.20703125,
      "learning_rate": 2.283882647899041e-08,
      "loss": 0.218,
      "reward": 0.4771205633878708,
      "reward_std": 0.1563674509525299,
      "rewards/accuracy_reward": 0.04687500232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455484867096,
      "step": 2934
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1783.9063415527344,
      "epoch": 0.8767082368755134,
      "grad_norm": 7.149814605712891,
      "kl": 4.01953125,
      "learning_rate": 2.2730065993142283e-08,
      "loss": 0.2336,
      "reward": 0.482142873108387,
      "reward_std": 0.1735913883894682,
      "rewards/accuracy_reward": 0.046875000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435267873108387,
      "step": 2935
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1735.1295471191406,
      "epoch": 0.8770069449630349,
      "grad_norm": 8.458356857299805,
      "kl": 4.57421875,
      "learning_rate": 2.2621552754707562e-08,
      "loss": 0.2651,
      "reward": 0.6294643133878708,
      "reward_std": 0.14252236299216747,
      "rewards/accuracy_reward": 0.20089286286383867,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 2936
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1792.9978332519531,
      "epoch": 0.8773056530505563,
      "grad_norm": 16.847572326660156,
      "kl": 4.9375,
      "learning_rate": 2.25132868817379e-08,
      "loss": 0.2557,
      "reward": 0.4681919887661934,
      "reward_std": 0.11617675051093102,
      "rewards/accuracy_reward": 0.04687500186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4213169813156128,
      "step": 2937
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1801.0089721679688,
      "epoch": 0.8776043611380778,
      "grad_norm": 9.531877517700195,
      "kl": 4.625,
      "learning_rate": 2.2405268492016015e-08,
      "loss": 0.2494,
      "reward": 0.463169664144516,
      "reward_std": 0.1300626490265131,
      "rewards/accuracy_reward": 0.04464285937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268059372902,
      "step": 2938
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1779.6920166015625,
      "epoch": 0.8779030692255992,
      "grad_norm": 4.441956996917725,
      "kl": 4.31640625,
      "learning_rate": 2.229749770305539e-08,
      "loss": 0.242,
      "reward": 0.455915205180645,
      "reward_std": 0.1365038137882948,
      "rewards/accuracy_reward": 0.0357142873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.420200914144516,
      "step": 2939
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1797.0826721191406,
      "epoch": 0.8782017773131208,
      "grad_norm": 13.035404205322266,
      "kl": 2.236328125,
      "learning_rate": 2.2189974632099955e-08,
      "loss": 0.1537,
      "reward": 0.4810268059372902,
      "reward_std": 0.09315787442028522,
      "rewards/accuracy_reward": 0.03794643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4430803805589676,
      "step": 2940
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1745.415283203125,
      "epoch": 0.8785004854006422,
      "grad_norm": 9.836873054504395,
      "kl": 2.74609375,
      "learning_rate": 2.2082699396124364e-08,
      "loss": 0.1659,
      "reward": 0.663504496216774,
      "reward_std": 0.1744316667318344,
      "rewards/accuracy_reward": 0.22321429289877415,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4402901977300644,
      "step": 2941
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1776.7925109863281,
      "epoch": 0.8787991934881637,
      "grad_norm": 13.220605850219727,
      "kl": 3.07421875,
      "learning_rate": 2.197567211183357e-08,
      "loss": 0.1766,
      "reward": 0.4927455708384514,
      "reward_std": 0.1520634312182665,
      "rewards/accuracy_reward": 0.0580357164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098469734192,
      "step": 2942
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1749.2121276855469,
      "epoch": 0.8790979015756851,
      "grad_norm": 6.716597080230713,
      "kl": 3.1767578125,
      "learning_rate": 2.186889289566268e-08,
      "loss": 0.1721,
      "reward": 0.4960937723517418,
      "reward_std": 0.1546249557286501,
      "rewards/accuracy_reward": 0.05357143026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4425223395228386,
      "step": 2943
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1778.3861999511719,
      "epoch": 0.8793966096632067,
      "grad_norm": 14.6886625289917,
      "kl": 3.05078125,
      "learning_rate": 2.1762361863777114e-08,
      "loss": 0.1776,
      "reward": 0.5708705633878708,
      "reward_std": 0.16768885403871536,
      "rewards/accuracy_reward": 0.1383928619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776977300644,
      "step": 2944
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1756.1540832519531,
      "epoch": 0.8796953177507281,
      "grad_norm": 5.546716690063477,
      "kl": 3.890625,
      "learning_rate": 2.1656079132072098e-08,
      "loss": 0.2314,
      "reward": 0.5089285895228386,
      "reward_std": 0.13526790775358677,
      "rewards/accuracy_reward": 0.08258928777649999,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263393059372902,
      "step": 2945
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.1943054199219,
      "epoch": 0.8799940258382496,
      "grad_norm": 9.530869483947754,
      "kl": 3.150390625,
      "learning_rate": 2.155004481617287e-08,
      "loss": 0.1896,
      "reward": 0.5580357238650322,
      "reward_std": 0.1627616547048092,
      "rewards/accuracy_reward": 0.12500000558793545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357313156128,
      "step": 2946
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1775.8661499023438,
      "epoch": 0.880292733925771,
      "grad_norm": 4.572495937347412,
      "kl": 3.53125,
      "learning_rate": 2.1444259031434458e-08,
      "loss": 0.2076,
      "reward": 0.4866071790456772,
      "reward_std": 0.1618131473660469,
      "rewards/accuracy_reward": 0.05803571757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714402794838,
      "step": 2947
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1798.9911193847656,
      "epoch": 0.8805914420132925,
      "grad_norm": 6.047021865844727,
      "kl": 4.08984375,
      "learning_rate": 2.133872189294128e-08,
      "loss": 0.2408,
      "reward": 0.521205373108387,
      "reward_std": 0.1670054979622364,
      "rewards/accuracy_reward": 0.0892857164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4319196566939354,
      "step": 2948
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1685.97998046875,
      "epoch": 0.880890150100814,
      "grad_norm": 3.665703058242798,
      "kl": 1.806640625,
      "learning_rate": 2.1233433515507466e-08,
      "loss": 0.1244,
      "reward": 0.550781287252903,
      "reward_std": 0.1722656711935997,
      "rewards/accuracy_reward": 0.10267857555299997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4481026902794838,
      "step": 2949
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1736.3170471191406,
      "epoch": 0.8811888581883355,
      "grad_norm": 7.223992347717285,
      "kl": 3.658203125,
      "learning_rate": 2.1128394013676497e-08,
      "loss": 0.2365,
      "reward": 0.5725446790456772,
      "reward_std": 0.1554953046143055,
      "rewards/accuracy_reward": 0.13616072246804833,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4363839477300644,
      "step": 2950
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1719.5670471191406,
      "epoch": 0.8814875662758569,
      "grad_norm": 4.55910587310791,
      "kl": 3.94140625,
      "learning_rate": 2.1023603501721e-08,
      "loss": 0.2597,
      "reward": 0.5390625223517418,
      "reward_std": 0.17795638740062714,
      "rewards/accuracy_reward": 0.10491072200238705,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341518059372902,
      "step": 2951
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1800.57373046875,
      "epoch": 0.8817862743633784,
      "grad_norm": 17.533878326416016,
      "kl": 4.78125,
      "learning_rate": 2.0919062093642748e-08,
      "loss": 0.2545,
      "reward": 0.486049123108387,
      "reward_std": 0.1829831525683403,
      "rewards/accuracy_reward": 0.08258928777649999,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.403459832072258,
      "step": 2952
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1799.2389221191406,
      "epoch": 0.8820849824508998,
      "grad_norm": 3.896245002746582,
      "kl": 3.525390625,
      "learning_rate": 2.0814769903172657e-08,
      "loss": 0.1931,
      "reward": 0.5786830708384514,
      "reward_std": 0.20268763601779938,
      "rewards/accuracy_reward": 0.14955357508733869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294887661934,
      "step": 2953
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1863.2031860351562,
      "epoch": 0.8823836905384214,
      "grad_norm": 9.193558692932129,
      "kl": 4.49609375,
      "learning_rate": 2.0710727043770303e-08,
      "loss": 0.2302,
      "reward": 0.4285714477300644,
      "reward_std": 0.1481450628489256,
      "rewards/accuracy_reward": 0.013392857741564512,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.415178582072258,
      "step": 2954
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1763.9531860351562,
      "epoch": 0.8826823986259428,
      "grad_norm": 18.595230102539062,
      "kl": 5.09375,
      "learning_rate": 2.0606933628624174e-08,
      "loss": 0.301,
      "reward": 0.525111623108387,
      "reward_std": 0.17531871423125267,
      "rewards/accuracy_reward": 0.11160714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4135044887661934,
      "step": 2955
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1756.9107971191406,
      "epoch": 0.8829811067134643,
      "grad_norm": 6.613658905029297,
      "kl": 4.76953125,
      "learning_rate": 2.0503389770651357e-08,
      "loss": 0.269,
      "reward": 0.4910714477300644,
      "reward_std": 0.1360656190663576,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428571455180645,
      "step": 2956
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1774.3304443359375,
      "epoch": 0.8832798148009857,
      "grad_norm": 17.44745635986328,
      "kl": 4.703125,
      "learning_rate": 2.0400095582497333e-08,
      "loss": 0.2429,
      "reward": 0.5050223618745804,
      "reward_std": 0.1283576302230358,
      "rewards/accuracy_reward": 0.08482143143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.420200914144516,
      "step": 2957
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1838.9085693359375,
      "epoch": 0.8835785228885072,
      "grad_norm": 35.27434539794922,
      "kl": 6.1328125,
      "learning_rate": 2.0297051176536158e-08,
      "loss": 0.3377,
      "reward": 0.4375000223517418,
      "reward_std": 0.15447713807225227,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500074505806,
      "step": 2958
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1861.2188415527344,
      "epoch": 0.8838772309760287,
      "grad_norm": 6.645206451416016,
      "kl": 4.0078125,
      "learning_rate": 2.019425666486993e-08,
      "loss": 0.2221,
      "reward": 0.5597098544239998,
      "reward_std": 0.21596502512693405,
      "rewards/accuracy_reward": 0.14508929592557251,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205559372902,
      "step": 2959
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1795.9755249023438,
      "epoch": 0.8841759390635502,
      "grad_norm": 6.7406134605407715,
      "kl": 3.65234375,
      "learning_rate": 2.0091712159329087e-08,
      "loss": 0.1953,
      "reward": 0.5117187723517418,
      "reward_std": 0.162877157330513,
      "rewards/accuracy_reward": 0.08482143096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4268973395228386,
      "step": 2960
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1876.0982971191406,
      "epoch": 0.8844746471510716,
      "grad_norm": 15.827983856201172,
      "kl": 4.734375,
      "learning_rate": 1.9989417771472035e-08,
      "loss": 0.2644,
      "reward": 0.4921875149011612,
      "reward_std": 0.24312212690711021,
      "rewards/accuracy_reward": 0.09598214458674192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.396205373108387,
      "step": 2961
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1739.0157165527344,
      "epoch": 0.8847733552385931,
      "grad_norm": 6.16668701171875,
      "kl": 3.732421875,
      "learning_rate": 1.9887373612584912e-08,
      "loss": 0.2107,
      "reward": 0.5491071790456772,
      "reward_std": 0.12249704450368881,
      "rewards/accuracy_reward": 0.12946429289877415,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4196428805589676,
      "step": 2962
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1812.7813415527344,
      "epoch": 0.8850720633261145,
      "grad_norm": 3.481977701187134,
      "kl": 3.9765625,
      "learning_rate": 1.9785579793681866e-08,
      "loss": 0.2334,
      "reward": 0.4503348469734192,
      "reward_std": 0.17221926152706146,
      "rewards/accuracy_reward": 0.042410714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.407924123108387,
      "step": 2963
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1646.5759582519531,
      "epoch": 0.8853707714136361,
      "grad_norm": 31.167213439941406,
      "kl": 2.60546875,
      "learning_rate": 1.968403642550459e-08,
      "loss": 0.1945,
      "reward": 0.5954241305589676,
      "reward_std": 0.15118969045579433,
      "rewards/accuracy_reward": 0.15178571874275804,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443638414144516,
      "step": 2964
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1787.9197387695312,
      "epoch": 0.8856694795011575,
      "grad_norm": 7.259477615356445,
      "kl": 3.046875,
      "learning_rate": 1.9582743618522273e-08,
      "loss": 0.1485,
      "reward": 0.4520089477300644,
      "reward_std": 0.1397387720644474,
      "rewards/accuracy_reward": 0.029017859371379018,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 2965
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1790.8639221191406,
      "epoch": 0.885968187588679,
      "grad_norm": 17.066267013549805,
      "kl": 2.126953125,
      "learning_rate": 1.9481701482931546e-08,
      "loss": 0.1373,
      "reward": 0.5050223544239998,
      "reward_std": 0.14415013790130615,
      "rewards/accuracy_reward": 0.0736607164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4313616305589676,
      "step": 2966
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1790.665283203125,
      "epoch": 0.8862668956762004,
      "grad_norm": 9.304194450378418,
      "kl": 3.41796875,
      "learning_rate": 1.9380910128656448e-08,
      "loss": 0.1993,
      "reward": 0.4994419887661934,
      "reward_std": 0.13237342424690723,
      "rewards/accuracy_reward": 0.08482143399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205559372902,
      "step": 2967
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1732.9978637695312,
      "epoch": 0.886565603763722,
      "grad_norm": 25.842185974121094,
      "kl": 2.55078125,
      "learning_rate": 1.928036966534799e-08,
      "loss": 0.1695,
      "reward": 0.4453125223517418,
      "reward_std": 0.13349316455423832,
      "rewards/accuracy_reward": 0.020089286845177412,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232313156128,
      "step": 2968
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1807.9242248535156,
      "epoch": 0.8868643118512434,
      "grad_norm": 16.512422561645508,
      "kl": 3.30078125,
      "learning_rate": 1.9180080202384374e-08,
      "loss": 0.179,
      "reward": 0.5000000223517418,
      "reward_std": 0.12078810669481754,
      "rewards/accuracy_reward": 0.08035714668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.419642873108387,
      "step": 2969
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1701.9710693359375,
      "epoch": 0.8871630199387649,
      "grad_norm": 13.941060066223145,
      "kl": 2.203125,
      "learning_rate": 1.908004184887077e-08,
      "loss": 0.1268,
      "reward": 0.4642857313156128,
      "reward_std": 0.11777893453836441,
      "rewards/accuracy_reward": 0.017857143422588706,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4464285895228386,
      "step": 2970
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1748.9532165527344,
      "epoch": 0.8874617280262863,
      "grad_norm": 17.219240188598633,
      "kl": 2.8828125,
      "learning_rate": 1.8980254713638983e-08,
      "loss": 0.1782,
      "reward": 0.5279017984867096,
      "reward_std": 0.14571836963295937,
      "rewards/accuracy_reward": 0.08482143119908869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.443080373108387,
      "step": 2971
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1898.57373046875,
      "epoch": 0.8877604361138077,
      "grad_norm": 6.729400157928467,
      "kl": 3.34375,
      "learning_rate": 1.8880718905247756e-08,
      "loss": 0.1674,
      "reward": 0.4564732313156128,
      "reward_std": 0.11615396291017532,
      "rewards/accuracy_reward": 0.04017857206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4162946566939354,
      "step": 2972
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1811.5045471191406,
      "epoch": 0.8880591442013293,
      "grad_norm": 7.560126304626465,
      "kl": 3.953125,
      "learning_rate": 1.8781434531982193e-08,
      "loss": 0.2412,
      "reward": 0.5636161044239998,
      "reward_std": 0.19242243096232414,
      "rewards/accuracy_reward": 0.15401786286383867,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4095982313156128,
      "step": 2973
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1770.1942749023438,
      "epoch": 0.8883578522888507,
      "grad_norm": 8.634564399719238,
      "kl": 3.75,
      "learning_rate": 1.8682401701854005e-08,
      "loss": 0.2257,
      "reward": 0.6171875298023224,
      "reward_std": 0.1864912286400795,
      "rewards/accuracy_reward": 0.1964285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4207589402794838,
      "step": 2974
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1732.1607971191406,
      "epoch": 0.8886565603763722,
      "grad_norm": 8.900465965270996,
      "kl": 3.435546875,
      "learning_rate": 1.8583620522601234e-08,
      "loss": 0.2249,
      "reward": 0.4821428805589676,
      "reward_std": 0.1603305283933878,
      "rewards/accuracy_reward": 0.04910714388824999,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357387661934,
      "step": 2975
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1720.6452026367188,
      "epoch": 0.8889552684638936,
      "grad_norm": 11.141268730163574,
      "kl": 3.39453125,
      "learning_rate": 1.8485091101688055e-08,
      "loss": 0.2218,
      "reward": 0.622209832072258,
      "reward_std": 0.197070199996233,
      "rewards/accuracy_reward": 0.1897321566939354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776977300644,
      "step": 2976
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1708.51123046875,
      "epoch": 0.8892539765514151,
      "grad_norm": 12.334859848022461,
      "kl": 4.6875,
      "learning_rate": 1.8386813546304862e-08,
      "loss": 0.2838,
      "reward": 0.5731026977300644,
      "reward_std": 0.15574834123253822,
      "rewards/accuracy_reward": 0.1517857164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4213169813156128,
      "step": 2977
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1761.8706359863281,
      "epoch": 0.8895526846389366,
      "grad_norm": 7.9743218421936035,
      "kl": 3.248046875,
      "learning_rate": 1.8288787963368053e-08,
      "loss": 0.184,
      "reward": 0.6171875298023224,
      "reward_std": 0.23580187559127808,
      "rewards/accuracy_reward": 0.1897321455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4274553805589676,
      "step": 2978
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1776.3616943359375,
      "epoch": 0.8898513927264581,
      "grad_norm": 22.566457748413086,
      "kl": 4.9140625,
      "learning_rate": 1.8191014459519733e-08,
      "loss": 0.2881,
      "reward": 0.5022321566939354,
      "reward_std": 0.16758285649120808,
      "rewards/accuracy_reward": 0.0937500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4084821566939354,
      "step": 2979
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1825.5737609863281,
      "epoch": 0.8901501008139795,
      "grad_norm": 44.56852340698242,
      "kl": 5.8828125,
      "learning_rate": 1.8093493141127984e-08,
      "loss": 0.3137,
      "reward": 0.428013414144516,
      "reward_std": 0.16742690466344357,
      "rewards/accuracy_reward": 0.03125000116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3967634066939354,
      "step": 2980
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1765.1764526367188,
      "epoch": 0.890448808901501,
      "grad_norm": 7.553714752197266,
      "kl": 4.1484375,
      "learning_rate": 1.7996224114286446e-08,
      "loss": 0.2453,
      "reward": 0.4737723395228386,
      "reward_std": 0.11272862739861012,
      "rewards/accuracy_reward": 0.042410716181620955,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.431361623108387,
      "step": 2981
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1765.3148193359375,
      "epoch": 0.8907475169890224,
      "grad_norm": 19.89854621887207,
      "kl": 4.50390625,
      "learning_rate": 1.7899207484814217e-08,
      "loss": 0.2607,
      "reward": 0.5273437574505806,
      "reward_std": 0.19599131867289543,
      "rewards/accuracy_reward": 0.1026785783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4246651977300644,
      "step": 2982
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1829.44873046875,
      "epoch": 0.891046225076544,
      "grad_norm": 17.692481994628906,
      "kl": 4.50390625,
      "learning_rate": 1.78024433582559e-08,
      "loss": 0.2467,
      "reward": 0.4832589402794838,
      "reward_std": 0.21332794427871704,
      "rewards/accuracy_reward": 0.0691964328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625149011612,
      "step": 2983
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1802.1764221191406,
      "epoch": 0.8913449331640654,
      "grad_norm": 23.558889389038086,
      "kl": 5.62109375,
      "learning_rate": 1.7705931839881476e-08,
      "loss": 0.2927,
      "reward": 0.4308035895228386,
      "reward_std": 0.15326131507754326,
      "rewards/accuracy_reward": 0.0267857164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4040178805589676,
      "step": 2984
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1766.0179443359375,
      "epoch": 0.8916436412515869,
      "grad_norm": 16.315526962280273,
      "kl": 3.86328125,
      "learning_rate": 1.7609673034685885e-08,
      "loss": 0.2083,
      "reward": 0.4849330559372902,
      "reward_std": 0.1722598746418953,
      "rewards/accuracy_reward": 0.0580357164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4268973395228386,
      "step": 2985
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1770.6116943359375,
      "epoch": 0.8919423493391083,
      "grad_norm": 5.627732753753662,
      "kl": 3.76171875,
      "learning_rate": 1.751366704738938e-08,
      "loss": 0.212,
      "reward": 0.4910714477300644,
      "reward_std": 0.17865683883428574,
      "rewards/accuracy_reward": 0.06473214598372579,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263393059372902,
      "step": 2986
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1760.9107666015625,
      "epoch": 0.8922410574266298,
      "grad_norm": 3.4575247764587402,
      "kl": 4.0546875,
      "learning_rate": 1.741791398243697e-08,
      "loss": 0.2264,
      "reward": 0.5011160969734192,
      "reward_std": 0.18788821622729301,
      "rewards/accuracy_reward": 0.07812500558793545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 2987
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1787.1407165527344,
      "epoch": 0.8925397655141513,
      "grad_norm": 6.4187541007995605,
      "kl": 4.140625,
      "learning_rate": 1.7322413943998666e-08,
      "loss": 0.2338,
      "reward": 0.5474330559372902,
      "reward_std": 0.1545313596725464,
      "rewards/accuracy_reward": 0.13616072130389512,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4112723395228386,
      "step": 2988
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1778.4822387695312,
      "epoch": 0.8928384736016728,
      "grad_norm": 6.561193466186523,
      "kl": 3.31640625,
      "learning_rate": 1.7227167035969188e-08,
      "loss": 0.1937,
      "reward": 0.4893973469734192,
      "reward_std": 0.15546206384897232,
      "rewards/accuracy_reward": 0.07812500558793545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.411272332072258,
      "step": 2989
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1749.9375610351562,
      "epoch": 0.8931371816891942,
      "grad_norm": 11.033413887023926,
      "kl": 3.0859375,
      "learning_rate": 1.7132173361967704e-08,
      "loss": 0.1851,
      "reward": 0.5418526902794838,
      "reward_std": 0.16524029150605202,
      "rewards/accuracy_reward": 0.11383929289877415,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428013414144516,
      "step": 2990
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1816.8103637695312,
      "epoch": 0.8934358897767157,
      "grad_norm": 10.531332969665527,
      "kl": 4.046875,
      "learning_rate": 1.7037433025338138e-08,
      "loss": 0.2373,
      "reward": 0.4988839477300644,
      "reward_std": 0.1308812964707613,
      "rewards/accuracy_reward": 0.0892857201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4095982313156128,
      "step": 2991
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1735.6764221191406,
      "epoch": 0.8937345978642371,
      "grad_norm": 3.7154836654663086,
      "kl": 3.07421875,
      "learning_rate": 1.6942946129148676e-08,
      "loss": 0.1677,
      "reward": 0.4637276977300644,
      "reward_std": 0.15698243957012892,
      "rewards/accuracy_reward": 0.04017857392318547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.423549123108387,
      "step": 2992
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1754.1942443847656,
      "epoch": 0.8940333059517587,
      "grad_norm": 20.348743438720703,
      "kl": 3.02734375,
      "learning_rate": 1.6848712776191765e-08,
      "loss": 0.1971,
      "reward": 0.4441964477300644,
      "reward_std": 0.14076597429811954,
      "rewards/accuracy_reward": 0.013392857741564512,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035895228386,
      "step": 2993
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1851.2701721191406,
      "epoch": 0.8943320140392801,
      "grad_norm": 8.984735488891602,
      "kl": 3.22265625,
      "learning_rate": 1.6754733068984035e-08,
      "loss": 0.1707,
      "reward": 0.4648437649011612,
      "reward_std": 0.13831565342843533,
      "rewards/accuracy_reward": 0.04910714412108064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.415736623108387,
      "step": 2994
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1720.7210693359375,
      "epoch": 0.8946307221268016,
      "grad_norm": 9.068717956542969,
      "kl": 3.7734375,
      "learning_rate": 1.6661007109766283e-08,
      "loss": 0.2381,
      "reward": 0.4849330559372902,
      "reward_std": 0.17593863233923912,
      "rewards/accuracy_reward": 0.05357143213041127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4313616305589676,
      "step": 2995
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1759.6295166015625,
      "epoch": 0.894929430214323,
      "grad_norm": 6.387885093688965,
      "kl": 3.314453125,
      "learning_rate": 1.6567535000503092e-08,
      "loss": 0.1854,
      "reward": 0.580915205180645,
      "reward_std": 0.20527694001793861,
      "rewards/accuracy_reward": 0.15625000558793545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4246651902794838,
      "step": 2996
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1803.1764221191406,
      "epoch": 0.8952281383018446,
      "grad_norm": 3.8826239109039307,
      "kl": 4.03125,
      "learning_rate": 1.6474316842882973e-08,
      "loss": 0.2262,
      "reward": 0.4609375223517418,
      "reward_std": 0.17871228978037834,
      "rewards/accuracy_reward": 0.042410716880112886,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268133878708,
      "step": 2997
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1864.72998046875,
      "epoch": 0.895526846389366,
      "grad_norm": 5.681728839874268,
      "kl": 4.2421875,
      "learning_rate": 1.63813527383182e-08,
      "loss": 0.2383,
      "reward": 0.4497767984867096,
      "reward_std": 0.16325656697154045,
      "rewards/accuracy_reward": 0.04687500232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4029017984867096,
      "step": 2998
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1703.1116943359375,
      "epoch": 0.8958255544768875,
      "grad_norm": 8.530016899108887,
      "kl": 3.26171875,
      "learning_rate": 1.6288642787944507e-08,
      "loss": 0.1854,
      "reward": 0.4676339477300644,
      "reward_std": 0.14518877491354942,
      "rewards/accuracy_reward": 0.037946431431919336,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875298023224,
      "step": 2999
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1781.8952026367188,
      "epoch": 0.8961242625644089,
      "grad_norm": 7.317563533782959,
      "kl": 3.83203125,
      "learning_rate": 1.6196187092621327e-08,
      "loss": 0.2195,
      "reward": 0.5145089477300644,
      "reward_std": 0.1594829149544239,
      "rewards/accuracy_reward": 0.1004464328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625149011612,
      "step": 3000
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1784.9063415527344,
      "epoch": 0.8964229706519304,
      "grad_norm": 24.33642578125,
      "kl": 5.44140625,
      "learning_rate": 1.6103985752931314e-08,
      "loss": 0.3198,
      "reward": 0.5122768133878708,
      "reward_std": 0.16183251701295376,
      "rewards/accuracy_reward": 0.10267857578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4095982313156128,
      "step": 3001
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1724.8973999023438,
      "epoch": 0.8967216787394519,
      "grad_norm": 7.091845989227295,
      "kl": 3.2333984375,
      "learning_rate": 1.601203886918054e-08,
      "loss": 0.1751,
      "reward": 0.506138414144516,
      "reward_std": 0.11216121260076761,
      "rewards/accuracy_reward": 0.0691964328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419813156128,
      "step": 3002
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1765.618408203125,
      "epoch": 0.8970203868269734,
      "grad_norm": 21.163101196289062,
      "kl": 3.357421875,
      "learning_rate": 1.5920346541398205e-08,
      "loss": 0.2151,
      "reward": 0.5133928805589676,
      "reward_std": 0.17487531155347824,
      "rewards/accuracy_reward": 0.07812500232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435267873108387,
      "step": 3003
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1825.7456359863281,
      "epoch": 0.8973190949144948,
      "grad_norm": 5.242889881134033,
      "kl": 4.80078125,
      "learning_rate": 1.582890886933655e-08,
      "loss": 0.2813,
      "reward": 0.4743303805589676,
      "reward_std": 0.12895838357508183,
      "rewards/accuracy_reward": 0.05580357275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268059372902,
      "step": 3004
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1696.6183471679688,
      "epoch": 0.8976178030020163,
      "grad_norm": 15.135379791259766,
      "kl": 4.1640625,
      "learning_rate": 1.5737725952470805e-08,
      "loss": 0.2561,
      "reward": 0.4765625298023224,
      "reward_std": 0.12551194243133068,
      "rewards/accuracy_reward": 0.04017857275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.436383955180645,
      "step": 3005
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1744.9197387695312,
      "epoch": 0.8979165110895377,
      "grad_norm": 5.449563980102539,
      "kl": 3.953125,
      "learning_rate": 1.5646797889999093e-08,
      "loss": 0.2307,
      "reward": 0.5429687574505806,
      "reward_std": 0.1960749812424183,
      "rewards/accuracy_reward": 0.11607143376022577,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4268973469734192,
      "step": 3006
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1808.3036804199219,
      "epoch": 0.8982152191770593,
      "grad_norm": 40.83533477783203,
      "kl": 6.3828125,
      "learning_rate": 1.5556124780842144e-08,
      "loss": 0.3468,
      "reward": 0.4520089477300644,
      "reward_std": 0.16220754757523537,
      "rewards/accuracy_reward": 0.05803571757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3939732313156128,
      "step": 3007
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1789.8572387695312,
      "epoch": 0.8985139272645807,
      "grad_norm": 16.653467178344727,
      "kl": 4.16796875,
      "learning_rate": 1.5465706723643474e-08,
      "loss": 0.23,
      "reward": 0.5379464626312256,
      "reward_std": 0.1597521211951971,
      "rewards/accuracy_reward": 0.11607143096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 3008
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1767.9130554199219,
      "epoch": 0.8988126353521022,
      "grad_norm": 8.859665870666504,
      "kl": 3.5703125,
      "learning_rate": 1.5375543816769087e-08,
      "loss": 0.2082,
      "reward": 0.4838169887661934,
      "reward_std": 0.11428386904299259,
      "rewards/accuracy_reward": 0.04910714668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098395228386,
      "step": 3009
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1773.87060546875,
      "epoch": 0.8991113434396236,
      "grad_norm": 13.51382064819336,
      "kl": 3.103515625,
      "learning_rate": 1.528563615830733e-08,
      "loss": 0.1675,
      "reward": 0.4893973469734192,
      "reward_std": 0.15845802798867226,
      "rewards/accuracy_reward": 0.06026785937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294887661934,
      "step": 3010
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1756.0670471191406,
      "epoch": 0.8994100515271451,
      "grad_norm": 14.859163284301758,
      "kl": 5.16015625,
      "learning_rate": 1.519598384606893e-08,
      "loss": 0.3,
      "reward": 0.4966518208384514,
      "reward_std": 0.13064716011285782,
      "rewards/accuracy_reward": 0.08928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4073660895228386,
      "step": 3011
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1805.6786499023438,
      "epoch": 0.8997087596146666,
      "grad_norm": 12.079204559326172,
      "kl": 4.96484375,
      "learning_rate": 1.5106586977586904e-08,
      "loss": 0.2839,
      "reward": 0.5368303880095482,
      "reward_std": 0.16945725865662098,
      "rewards/accuracy_reward": 0.1205357201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4162946715950966,
      "step": 3012
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1767.6429443359375,
      "epoch": 0.9000074677021881,
      "grad_norm": 6.974257946014404,
      "kl": 2.837890625,
      "learning_rate": 1.501744565011617e-08,
      "loss": 0.1681,
      "reward": 0.6277901977300644,
      "reward_std": 0.17926384136080742,
      "rewards/accuracy_reward": 0.19196429336443543,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4358259066939354,
      "step": 3013
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1811.9509582519531,
      "epoch": 0.9003061757897095,
      "grad_norm": 20.885343551635742,
      "kl": 5.484375,
      "learning_rate": 1.492855996063386e-08,
      "loss": 0.3143,
      "reward": 0.4743303805589676,
      "reward_std": 0.1958291307091713,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4029018059372902,
      "step": 3014
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1822.6697082519531,
      "epoch": 0.9006048838772309,
      "grad_norm": 11.043693542480469,
      "kl": 3.26953125,
      "learning_rate": 1.4839930005838775e-08,
      "loss": 0.1846,
      "reward": 0.5206473544239998,
      "reward_std": 0.14596332795917988,
      "rewards/accuracy_reward": 0.09151786309666932,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294887661934,
      "step": 3015
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1799.7523193359375,
      "epoch": 0.9009035919647524,
      "grad_norm": 7.681408405303955,
      "kl": 4.0546875,
      "learning_rate": 1.4751555882151683e-08,
      "loss": 0.2197,
      "reward": 0.5641741305589676,
      "reward_std": 0.20423424988985062,
      "rewards/accuracy_reward": 0.15178572572767735,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.412388414144516,
      "step": 3016
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1744.52685546875,
      "epoch": 0.9012023000522739,
      "grad_norm": 11.086936950683594,
      "kl": 3.40625,
      "learning_rate": 1.4663437685715014e-08,
      "loss": 0.2169,
      "reward": 0.545758955180645,
      "reward_std": 0.18009774573147297,
      "rewards/accuracy_reward": 0.12053571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232387661934,
      "step": 3017
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1757.9487609863281,
      "epoch": 0.9015010081397954,
      "grad_norm": 6.577908992767334,
      "kl": 4.16015625,
      "learning_rate": 1.4575575512392608e-08,
      "loss": 0.2548,
      "reward": 0.5831473544239998,
      "reward_std": 0.2279868759214878,
      "rewards/accuracy_reward": 0.16517858067527413,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4179687723517418,
      "step": 3018
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1802.2076721191406,
      "epoch": 0.9017997162273168,
      "grad_norm": 10.551751136779785,
      "kl": 4.45703125,
      "learning_rate": 1.4487969457770005e-08,
      "loss": 0.2568,
      "reward": 0.4715401977300644,
      "reward_std": 0.12084257788956165,
      "rewards/accuracy_reward": 0.0535714328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4179687649011612,
      "step": 3019
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1745.399658203125,
      "epoch": 0.9020984243148383,
      "grad_norm": 21.559125900268555,
      "kl": 2.9453125,
      "learning_rate": 1.4400619617153998e-08,
      "loss": 0.197,
      "reward": 0.601562537252903,
      "reward_std": 0.18469862639904022,
      "rewards/accuracy_reward": 0.17187500558793545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875223517418,
      "step": 3020
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1797.2076721191406,
      "epoch": 0.9023971324023597,
      "grad_norm": 7.436116695404053,
      "kl": 3.30859375,
      "learning_rate": 1.4313526085572596e-08,
      "loss": 0.1834,
      "reward": 0.561941996216774,
      "reward_std": 0.1673073936253786,
      "rewards/accuracy_reward": 0.1406250037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4213169887661934,
      "step": 3021
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1719.8281860351562,
      "epoch": 0.9026958404898813,
      "grad_norm": 12.858040809631348,
      "kl": 3.8125,
      "learning_rate": 1.4226688957775068e-08,
      "loss": 0.2291,
      "reward": 0.5027901977300644,
      "reward_std": 0.1566948276013136,
      "rewards/accuracy_reward": 0.0758928582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4268973395228386,
      "step": 3022
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1715.0067749023438,
      "epoch": 0.9029945485774027,
      "grad_norm": 8.969083786010742,
      "kl": 4.052734375,
      "learning_rate": 1.4140108328231703e-08,
      "loss": 0.2399,
      "reward": 0.546316996216774,
      "reward_std": 0.15149638429284096,
      "rewards/accuracy_reward": 0.11830358020961285,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428013414144516,
      "step": 3023
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1759.7523193359375,
      "epoch": 0.9032932566649242,
      "grad_norm": 11.13490104675293,
      "kl": 2.9375,
      "learning_rate": 1.4053784291133686e-08,
      "loss": 0.1927,
      "reward": 0.4854910895228386,
      "reward_std": 0.1499284766614437,
      "rewards/accuracy_reward": 0.0468750037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4386160895228386,
      "step": 3024
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1778.4978332519531,
      "epoch": 0.9035919647524456,
      "grad_norm": 3.653440237045288,
      "kl": 3.8359375,
      "learning_rate": 1.3967716940393198e-08,
      "loss": 0.2544,
      "reward": 0.530133955180645,
      "reward_std": 0.18014714866876602,
      "rewards/accuracy_reward": 0.11160715040750802,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185267984867096,
      "step": 3025
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1806.9085388183594,
      "epoch": 0.9038906728399672,
      "grad_norm": 4.947422981262207,
      "kl": 4.1796875,
      "learning_rate": 1.3881906369642982e-08,
      "loss": 0.2415,
      "reward": 0.4609375223517418,
      "reward_std": 0.1501184031367302,
      "rewards/accuracy_reward": 0.04017857392318547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4207589402794838,
      "step": 3026
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1709.4688110351562,
      "epoch": 0.9041893809274886,
      "grad_norm": 7.598712921142578,
      "kl": 3.859375,
      "learning_rate": 1.3796352672236556e-08,
      "loss": 0.232,
      "reward": 0.4603794887661934,
      "reward_std": 0.14632592536509037,
      "rewards/accuracy_reward": 0.03348214365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4268973395228386,
      "step": 3027
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1767.2098999023438,
      "epoch": 0.9044880890150101,
      "grad_norm": 9.425708770751953,
      "kl": 3.85546875,
      "learning_rate": 1.371105594124805e-08,
      "loss": 0.2343,
      "reward": 0.5066964477300644,
      "reward_std": 0.12787888757884502,
      "rewards/accuracy_reward": 0.08035714644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263392984867096,
      "step": 3028
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1792.0358276367188,
      "epoch": 0.9047867971025315,
      "grad_norm": 7.950504779815674,
      "kl": 4.50390625,
      "learning_rate": 1.3626016269471823e-08,
      "loss": 0.2516,
      "reward": 0.483258955180645,
      "reward_std": 0.1965232789516449,
      "rewards/accuracy_reward": 0.06473214598372579,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268059372902,
      "step": 3029
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1667.6987609863281,
      "epoch": 0.905085505190053,
      "grad_norm": 7.137703895568848,
      "kl": 2.84765625,
      "learning_rate": 1.3541233749422732e-08,
      "loss": 0.1731,
      "reward": 0.5083705559372902,
      "reward_std": 0.12266553938388824,
      "rewards/accuracy_reward": 0.06250000302679837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4458705484867096,
      "step": 3030
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1749.6942749023438,
      "epoch": 0.9053842132775745,
      "grad_norm": 10.648491859436035,
      "kl": 4.376953125,
      "learning_rate": 1.3456708473335936e-08,
      "loss": 0.2488,
      "reward": 0.6004464626312256,
      "reward_std": 0.19203465059399605,
      "rewards/accuracy_reward": 0.16964286379516125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035895228386,
      "step": 3031
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1747.9063415527344,
      "epoch": 0.905682921365096,
      "grad_norm": 12.397019386291504,
      "kl": 3.0234375,
      "learning_rate": 1.3372440533166551e-08,
      "loss": 0.175,
      "reward": 0.4827009215950966,
      "reward_std": 0.1601248849183321,
      "rewards/accuracy_reward": 0.03571428661234677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.446986623108387,
      "step": 3032
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1771.4286499023438,
      "epoch": 0.9059816294526174,
      "grad_norm": 9.668246269226074,
      "kl": 3.3359375,
      "learning_rate": 1.3288430020589853e-08,
      "loss": 0.2089,
      "reward": 0.502790205180645,
      "reward_std": 0.1742841601371765,
      "rewards/accuracy_reward": 0.0647321455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4380580559372902,
      "step": 3033
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1750.419677734375,
      "epoch": 0.9062803375401389,
      "grad_norm": 15.599602699279785,
      "kl": 2.484375,
      "learning_rate": 1.3204677027001122e-08,
      "loss": 0.162,
      "reward": 0.5122768059372902,
      "reward_std": 0.1495446301996708,
      "rewards/accuracy_reward": 0.06250000232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4497768059372902,
      "step": 3034
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1680.7031860351562,
      "epoch": 0.9065790456276603,
      "grad_norm": 6.781495094299316,
      "kl": 2.71484375,
      "learning_rate": 1.3121181643515306e-08,
      "loss": 0.1947,
      "reward": 0.5859375149011612,
      "reward_std": 0.16371503099799156,
      "rewards/accuracy_reward": 0.13839286286383867,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4475446715950966,
      "step": 3035
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1825.415283203125,
      "epoch": 0.9068777537151819,
      "grad_norm": 11.519110679626465,
      "kl": 4.69921875,
      "learning_rate": 1.3037943960967245e-08,
      "loss": 0.2582,
      "reward": 0.5390625298023224,
      "reward_std": 0.16831624880433083,
      "rewards/accuracy_reward": 0.1227678656578064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4162946566939354,
      "step": 3036
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1725.13623046875,
      "epoch": 0.9071764618027033,
      "grad_norm": 4.192883491516113,
      "kl": 3.50390625,
      "learning_rate": 1.2954964069911396e-08,
      "loss": 0.2139,
      "reward": 0.5452009215950966,
      "reward_std": 0.15532734617590904,
      "rewards/accuracy_reward": 0.11607143376022577,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294887661934,
      "step": 3037
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1805.3326721191406,
      "epoch": 0.9074751698902248,
      "grad_norm": 11.080338478088379,
      "kl": 4.55078125,
      "learning_rate": 1.2872242060621707e-08,
      "loss": 0.2447,
      "reward": 0.5541295036673546,
      "reward_std": 0.13940712064504623,
      "rewards/accuracy_reward": 0.1316964365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330633878708,
      "step": 3038
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1740.2745971679688,
      "epoch": 0.9077738779777462,
      "grad_norm": 9.880365371704102,
      "kl": 5.671875,
      "learning_rate": 1.278977802309167e-08,
      "loss": 0.3287,
      "reward": 0.506138414144516,
      "reward_std": 0.12769347988069057,
      "rewards/accuracy_reward": 0.0870535746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4190848395228386,
      "step": 3039
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1696.2322082519531,
      "epoch": 0.9080725860652678,
      "grad_norm": 7.809533596038818,
      "kl": 4.68359375,
      "learning_rate": 1.2707572047034015e-08,
      "loss": 0.2759,
      "reward": 0.5156250223517418,
      "reward_std": 0.14715279079973698,
      "rewards/accuracy_reward": 0.08705357578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 3040
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1754.4442749023438,
      "epoch": 0.9083712941527892,
      "grad_norm": 28.321964263916016,
      "kl": 5.5078125,
      "learning_rate": 1.262562422188082e-08,
      "loss": 0.3082,
      "reward": 0.494419664144516,
      "reward_std": 0.16195397078990936,
      "rewards/accuracy_reward": 0.0870535746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4073660895228386,
      "step": 3041
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1783.243408203125,
      "epoch": 0.9086700022403107,
      "grad_norm": 18.60158920288086,
      "kl": 4.6484375,
      "learning_rate": 1.25439346367833e-08,
      "loss": 0.2489,
      "reward": 0.5781250298023224,
      "reward_std": 0.1846358235925436,
      "rewards/accuracy_reward": 0.1584821529686451,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4196428805589676,
      "step": 3042
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1759.8884887695312,
      "epoch": 0.9089687103278321,
      "grad_norm": 13.930624008178711,
      "kl": 4.5078125,
      "learning_rate": 1.2462503380611667e-08,
      "loss": 0.2436,
      "reward": 0.5306919887661934,
      "reward_std": 0.09980147518217564,
      "rewards/accuracy_reward": 0.1093750037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4213169813156128,
      "step": 3043
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1729.2098999023438,
      "epoch": 0.9092674184153536,
      "grad_norm": 7.3565874099731445,
      "kl": 3.76171875,
      "learning_rate": 1.2381330541955198e-08,
      "loss": 0.2193,
      "reward": 0.5223214402794838,
      "reward_std": 0.19376079738140106,
      "rewards/accuracy_reward": 0.08928571757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357313156128,
      "step": 3044
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1697.0313415527344,
      "epoch": 0.909566126502875,
      "grad_norm": 6.694425106048584,
      "kl": 4.1171875,
      "learning_rate": 1.2300416209121977e-08,
      "loss": 0.2357,
      "reward": 0.541852705180645,
      "reward_std": 0.15285667032003403,
      "rewards/accuracy_reward": 0.10937500488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776977300644,
      "step": 3045
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1810.3148498535156,
      "epoch": 0.9098648345903966,
      "grad_norm": 5.282022476196289,
      "kl": 4.31640625,
      "learning_rate": 1.221976047013884e-08,
      "loss": 0.2247,
      "reward": 0.4536830484867096,
      "reward_std": 0.15884272940456867,
      "rewards/accuracy_reward": 0.0290178582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4246651977300644,
      "step": 3046
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1796.7188415527344,
      "epoch": 0.910163542677918,
      "grad_norm": 25.243209838867188,
      "kl": 2.900390625,
      "learning_rate": 1.2139363412751347e-08,
      "loss": 0.2018,
      "reward": 0.4419643059372902,
      "reward_std": 0.13291450589895248,
      "rewards/accuracy_reward": 0.013392857508733869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 3047
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1701.4844360351562,
      "epoch": 0.9104622507654395,
      "grad_norm": 8.00151252746582,
      "kl": 4.0234375,
      "learning_rate": 1.2059225124423617e-08,
      "loss": 0.2593,
      "reward": 0.565848246216774,
      "reward_std": 0.17314192466437817,
      "rewards/accuracy_reward": 0.1361607164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875149011612,
      "step": 3048
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.62060546875,
      "epoch": 0.9107609588529609,
      "grad_norm": 5.522336006164551,
      "kl": 4.046875,
      "learning_rate": 1.1979345692338245e-08,
      "loss": 0.2419,
      "reward": 0.5033482313156128,
      "reward_std": 0.11757203377783298,
      "rewards/accuracy_reward": 0.08035714668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 3049
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1800.149658203125,
      "epoch": 0.9110596669404825,
      "grad_norm": 6.179238319396973,
      "kl": 4.08203125,
      "learning_rate": 1.1899725203396216e-08,
      "loss": 0.2348,
      "reward": 0.6026785969734192,
      "reward_std": 0.17641312070190907,
      "rewards/accuracy_reward": 0.180803582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 3050
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1830.51123046875,
      "epoch": 0.9113583750280039,
      "grad_norm": 6.5983734130859375,
      "kl": 4.78125,
      "learning_rate": 1.1820363744216883e-08,
      "loss": 0.2758,
      "reward": 0.4492187649011612,
      "reward_std": 0.13487210683524609,
      "rewards/accuracy_reward": 0.04910714365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.400111623108387,
      "step": 3051
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1785.0313415527344,
      "epoch": 0.9116570831155254,
      "grad_norm": 4.358333587646484,
      "kl": 3.80078125,
      "learning_rate": 1.1741261401137626e-08,
      "loss": 0.2286,
      "reward": 0.4453125149011612,
      "reward_std": 0.1487284917384386,
      "rewards/accuracy_reward": 0.029017859138548374,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.416294664144516,
      "step": 3052
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1799.5536499023438,
      "epoch": 0.9119557912030468,
      "grad_norm": 4.497030258178711,
      "kl": 4.21484375,
      "learning_rate": 1.1662418260214136e-08,
      "loss": 0.2276,
      "reward": 0.5513393133878708,
      "reward_std": 0.16147803515195847,
      "rewards/accuracy_reward": 0.13392857206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4174107313156128,
      "step": 3053
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1747.040283203125,
      "epoch": 0.9122544992905683,
      "grad_norm": 7.4948410987854,
      "kl": 3.5390625,
      "learning_rate": 1.1583834407219966e-08,
      "loss": 0.2219,
      "reward": 0.5005580559372902,
      "reward_std": 0.1525675356388092,
      "rewards/accuracy_reward": 0.06919643329456449,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.431361623108387,
      "step": 3054
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1801.0960388183594,
      "epoch": 0.9125532073780898,
      "grad_norm": 4.306365489959717,
      "kl": 3.921875,
      "learning_rate": 1.150550992764665e-08,
      "loss": 0.2151,
      "reward": 0.585937537252903,
      "reward_std": 0.17312992364168167,
      "rewards/accuracy_reward": 0.16964286379516125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.416294664144516,
      "step": 3055
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1640.2902526855469,
      "epoch": 0.9128519154656113,
      "grad_norm": 11.753826141357422,
      "kl": 3.318359375,
      "learning_rate": 1.1427444906703609e-08,
      "loss": 0.2259,
      "reward": 0.5664062798023224,
      "reward_std": 0.1418941468000412,
      "rewards/accuracy_reward": 0.12500000651925802,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4414062649011612,
      "step": 3056
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1771.5045166015625,
      "epoch": 0.9131506235531327,
      "grad_norm": 12.52258014678955,
      "kl": 2.4765625,
      "learning_rate": 1.1349639429317825e-08,
      "loss": 0.1535,
      "reward": 0.6238839626312256,
      "reward_std": 0.19463452324271202,
      "rewards/accuracy_reward": 0.18750000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4363839477300644,
      "step": 3057
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1791.9331665039062,
      "epoch": 0.9134493316406541,
      "grad_norm": 3.472938060760498,
      "kl": 3.248046875,
      "learning_rate": 1.1272093580134118e-08,
      "loss": 0.1694,
      "reward": 0.4793526977300644,
      "reward_std": 0.12507658638060093,
      "rewards/accuracy_reward": 0.051339289639145136,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428013414144516,
      "step": 3058
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1815.5291137695312,
      "epoch": 0.9137480397281756,
      "grad_norm": 4.629681587219238,
      "kl": 4.2578125,
      "learning_rate": 1.1194807443514781e-08,
      "loss": 0.2538,
      "reward": 0.4146205559372902,
      "reward_std": 0.13787997514009476,
      "rewards/accuracy_reward": 0.02455357275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3900669813156128,
      "step": 3059
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1745.524658203125,
      "epoch": 0.9140467478156971,
      "grad_norm": 16.71782112121582,
      "kl": 3.63671875,
      "learning_rate": 1.1117781103539475e-08,
      "loss": 0.2186,
      "reward": 0.5518973395228386,
      "reward_std": 0.15606126189231873,
      "rewards/accuracy_reward": 0.12276786309666932,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294813156128,
      "step": 3060
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1699.4420166015625,
      "epoch": 0.9143454559032186,
      "grad_norm": 10.47619915008545,
      "kl": 3.46875,
      "learning_rate": 1.1041014644005364e-08,
      "loss": 0.2168,
      "reward": 0.498883955180645,
      "reward_std": 0.180588249117136,
      "rewards/accuracy_reward": 0.06696428777649999,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.431919664144516,
      "step": 3061
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1749.5023193359375,
      "epoch": 0.91464416399074,
      "grad_norm": 16.9040584564209,
      "kl": 3.40234375,
      "learning_rate": 1.0964508148426837e-08,
      "loss": 0.2171,
      "reward": 0.462611623108387,
      "reward_std": 0.1347369384020567,
      "rewards/accuracy_reward": 0.0290178582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937723517418,
      "step": 3062
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1708.7255249023438,
      "epoch": 0.9149428720782615,
      "grad_norm": 9.712556838989258,
      "kl": 3.4453125,
      "learning_rate": 1.0888261700035428e-08,
      "loss": 0.2222,
      "reward": 0.517857164144516,
      "reward_std": 0.16814947500824928,
      "rewards/accuracy_reward": 0.0803571455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000149011612,
      "step": 3063
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1722.6808776855469,
      "epoch": 0.9152415801657829,
      "grad_norm": 5.283228874206543,
      "kl": 2.953125,
      "learning_rate": 1.0812275381779811e-08,
      "loss": 0.1658,
      "reward": 0.478794664144516,
      "reward_std": 0.15902943909168243,
      "rewards/accuracy_reward": 0.044642860535532236,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341517984867096,
      "step": 3064
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1808.7991943359375,
      "epoch": 0.9155402882533045,
      "grad_norm": 5.598512172698975,
      "kl": 3.57421875,
      "learning_rate": 1.0736549276325724e-08,
      "loss": 0.202,
      "reward": 0.5485491380095482,
      "reward_std": 0.18903005868196487,
      "rewards/accuracy_reward": 0.1272321492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4213169813156128,
      "step": 3065
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.1451721191406,
      "epoch": 0.9158389963408259,
      "grad_norm": 18.885473251342773,
      "kl": 2.52734375,
      "learning_rate": 1.0661083466055627e-08,
      "loss": 0.1556,
      "reward": 0.5452009066939354,
      "reward_std": 0.18969298340380192,
      "rewards/accuracy_reward": 0.1026785746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4425223395228386,
      "step": 3066
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1743.0290832519531,
      "epoch": 0.9161377044283474,
      "grad_norm": 9.304407119750977,
      "kl": 4.61328125,
      "learning_rate": 1.0585878033069012e-08,
      "loss": 0.2862,
      "reward": 0.5987723469734192,
      "reward_std": 0.1754293590784073,
      "rewards/accuracy_reward": 0.17633929592557251,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330559372902,
      "step": 3067
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1749.4197387695312,
      "epoch": 0.9164364125158688,
      "grad_norm": 8.330451965332031,
      "kl": 4.33203125,
      "learning_rate": 1.0510933059181993e-08,
      "loss": 0.2321,
      "reward": 0.5016741305589676,
      "reward_std": 0.1721808724105358,
      "rewards/accuracy_reward": 0.0758928598370403,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812723517418,
      "step": 3068
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1680.8706359863281,
      "epoch": 0.9167351206033904,
      "grad_norm": 12.119317054748535,
      "kl": 3.59765625,
      "learning_rate": 1.043624862592732e-08,
      "loss": 0.2109,
      "reward": 0.5251116305589676,
      "reward_std": 0.1410869061946869,
      "rewards/accuracy_reward": 0.09151786053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937723517418,
      "step": 3069
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1756.1563415527344,
      "epoch": 0.9170338286909118,
      "grad_norm": 5.231842994689941,
      "kl": 3.50390625,
      "learning_rate": 1.0361824814554416e-08,
      "loss": 0.2004,
      "reward": 0.5078125223517418,
      "reward_std": 0.14818345196545124,
      "rewards/accuracy_reward": 0.07812500582076609,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875223517418,
      "step": 3070
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1776.3304443359375,
      "epoch": 0.9173325367784333,
      "grad_norm": 3.4943671226501465,
      "kl": 3.2890625,
      "learning_rate": 1.0287661706029016e-08,
      "loss": 0.1785,
      "reward": 0.5435268208384514,
      "reward_std": 0.1916392520070076,
      "rewards/accuracy_reward": 0.11160714738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4319196566939354,
      "step": 3071
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1795.1853637695312,
      "epoch": 0.9176312448659547,
      "grad_norm": 21.077880859375,
      "kl": 4.32421875,
      "learning_rate": 1.021375938103336e-08,
      "loss": 0.2095,
      "reward": 0.4536830484867096,
      "reward_std": 0.11673583835363388,
      "rewards/accuracy_reward": 0.0334821455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.420200914144516,
      "step": 3072
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1856.1942749023438,
      "epoch": 0.9179299529534762,
      "grad_norm": 4.496991157531738,
      "kl": 3.3828125,
      "learning_rate": 1.0140117919965913e-08,
      "loss": 0.1775,
      "reward": 0.5474330633878708,
      "reward_std": 0.1377830784767866,
      "rewards/accuracy_reward": 0.11830357648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294887661934,
      "step": 3073
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1732.1317749023438,
      "epoch": 0.9182286610409977,
      "grad_norm": 9.806090354919434,
      "kl": 4.6328125,
      "learning_rate": 1.0066737402941367e-08,
      "loss": 0.2868,
      "reward": 0.5418526977300644,
      "reward_std": 0.12095485255122185,
      "rewards/accuracy_reward": 0.113839291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4280134066939354,
      "step": 3074
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1828.0491943359375,
      "epoch": 0.9185273691285192,
      "grad_norm": 19.60527992248535,
      "kl": 4.4609375,
      "learning_rate": 9.993617909790558e-09,
      "loss": 0.2367,
      "reward": 0.5094866305589676,
      "reward_std": 0.1649342216551304,
      "rewards/accuracy_reward": 0.0937500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4157366305589676,
      "step": 3075
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1771.8505249023438,
      "epoch": 0.9188260772160406,
      "grad_norm": 14.581271171569824,
      "kl": 3.98046875,
      "learning_rate": 9.9207595200603e-09,
      "loss": 0.2017,
      "reward": 0.6121652126312256,
      "reward_std": 0.18209047242999077,
      "rewards/accuracy_reward": 0.1830357201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294813156128,
      "step": 3076
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1752.3951416015625,
      "epoch": 0.9191247853035621,
      "grad_norm": 21.229734420776367,
      "kl": 4.875,
      "learning_rate": 9.848162313013386e-09,
      "loss": 0.2846,
      "reward": 0.5362723618745804,
      "reward_std": 0.19161529652774334,
      "rewards/accuracy_reward": 0.1160714365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4202009066939354,
      "step": 3077
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1698.8750610351562,
      "epoch": 0.9194234933910835,
      "grad_norm": 6.557315349578857,
      "kl": 3.1796875,
      "learning_rate": 9.77582636762847e-09,
      "loss": 0.1881,
      "reward": 0.506138414144516,
      "reward_std": 0.1553447749465704,
      "rewards/accuracy_reward": 0.07812500419095159,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428013414144516,
      "step": 3078
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1744.9621276855469,
      "epoch": 0.9197222014786051,
      "grad_norm": 9.445967674255371,
      "kl": 3.66796875,
      "learning_rate": 9.703751762599994e-09,
      "loss": 0.2223,
      "reward": 0.5524553805589676,
      "reward_std": 0.16282760351896286,
      "rewards/accuracy_reward": 0.12500000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.427455373108387,
      "step": 3079
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1771.3907165527344,
      "epoch": 0.9200209095661265,
      "grad_norm": 18.23233413696289,
      "kl": 4.734375,
      "learning_rate": 9.631938576337984e-09,
      "loss": 0.2743,
      "reward": 0.5479910895228386,
      "reward_std": 0.14629370346665382,
      "rewards/accuracy_reward": 0.1428571455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4051339402794838,
      "step": 3080
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1814.0491943359375,
      "epoch": 0.920319617653648,
      "grad_norm": 5.184340953826904,
      "kl": 4.21875,
      "learning_rate": 9.560386886968253e-09,
      "loss": 0.2364,
      "reward": 0.4453125223517418,
      "reward_std": 0.14160324074327946,
      "rewards/accuracy_reward": 0.029017858672887087,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4162946715950966,
      "step": 3081
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1717.8125915527344,
      "epoch": 0.9206183257411694,
      "grad_norm": 6.77448034286499,
      "kl": 2.98828125,
      "learning_rate": 9.489096772331951e-09,
      "loss": 0.1924,
      "reward": 0.5340401977300644,
      "reward_std": 0.1878398396074772,
      "rewards/accuracy_reward": 0.1049107201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294887661934,
      "step": 3082
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1778.1250915527344,
      "epoch": 0.920917033828691,
      "grad_norm": 10.25493049621582,
      "kl": 2.974609375,
      "learning_rate": 9.418068309985733e-09,
      "loss": 0.1866,
      "reward": 0.5345982387661934,
      "reward_std": 0.1333506666123867,
      "rewards/accuracy_reward": 0.10044643213041127,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341517984867096,
      "step": 3083
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1834.1183776855469,
      "epoch": 0.9212157419162124,
      "grad_norm": 12.673382759094238,
      "kl": 4.8359375,
      "learning_rate": 9.347301577201678e-09,
      "loss": 0.2593,
      "reward": 0.5574776977300644,
      "reward_std": 0.21058500185608864,
      "rewards/accuracy_reward": 0.1428571492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205559372902,
      "step": 3084
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1763.618408203125,
      "epoch": 0.9215144500037339,
      "grad_norm": 29.40384292602539,
      "kl": 4.73046875,
      "learning_rate": 9.276796650966979e-09,
      "loss": 0.2436,
      "reward": 0.5691964477300644,
      "reward_std": 0.1542373187839985,
      "rewards/accuracy_reward": 0.15848214738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4107142984867096,
      "step": 3085
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1711.9040832519531,
      "epoch": 0.9218131580912553,
      "grad_norm": 9.678021430969238,
      "kl": 3.7734375,
      "learning_rate": 9.206553607984086e-09,
      "loss": 0.2356,
      "reward": 0.5323660895228386,
      "reward_std": 0.15971754118800163,
      "rewards/accuracy_reward": 0.1093750074505806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910969734192,
      "step": 3086
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1828.6607971191406,
      "epoch": 0.9221118661787768,
      "grad_norm": 2.939305543899536,
      "kl": 3.140625,
      "learning_rate": 9.13657252467062e-09,
      "loss": 0.1652,
      "reward": 0.474888414144516,
      "reward_std": 0.180317971855402,
      "rewards/accuracy_reward": 0.051339288242161274,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.423549123108387,
      "step": 3087
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1776.3996276855469,
      "epoch": 0.9224105742662982,
      "grad_norm": 11.6106595993042,
      "kl": 2.98046875,
      "learning_rate": 9.066853477159076e-09,
      "loss": 0.1752,
      "reward": 0.4486607313156128,
      "reward_std": 0.12987824715673923,
      "rewards/accuracy_reward": 0.015625000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357387661934,
      "step": 3088
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1814.6563110351562,
      "epoch": 0.9227092823538198,
      "grad_norm": 9.244889259338379,
      "kl": 3.361328125,
      "learning_rate": 8.99739654129697e-09,
      "loss": 0.2032,
      "reward": 0.4799107238650322,
      "reward_std": 0.16903270967304707,
      "rewards/accuracy_reward": 0.06026786030270159,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.419642873108387,
      "step": 3089
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1766.0781860351562,
      "epoch": 0.9230079904413412,
      "grad_norm": 13.433330535888672,
      "kl": 4.20703125,
      "learning_rate": 8.92820179264664e-09,
      "loss": 0.2559,
      "reward": 0.487165205180645,
      "reward_std": 0.14917483367025852,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4157366305589676,
      "step": 3090
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1827.9398193359375,
      "epoch": 0.9233066985288627,
      "grad_norm": 19.02574348449707,
      "kl": 3.81640625,
      "learning_rate": 8.859269306485174e-09,
      "loss": 0.1995,
      "reward": 0.4782366305589676,
      "reward_std": 0.14759855717420578,
      "rewards/accuracy_reward": 0.0736607201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.404575914144516,
      "step": 3091
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1778.9264221191406,
      "epoch": 0.9236054066163841,
      "grad_norm": 10.65204906463623,
      "kl": 4.765625,
      "learning_rate": 8.790599157804363e-09,
      "loss": 0.2756,
      "reward": 0.465959832072258,
      "reward_std": 0.18430043384432793,
      "rewards/accuracy_reward": 0.0602678619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4056919738650322,
      "step": 3092
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1741.4063415527344,
      "epoch": 0.9239041147039057,
      "grad_norm": 6.63496732711792,
      "kl": 3.9765625,
      "learning_rate": 8.722191421310614e-09,
      "loss": 0.2362,
      "reward": 0.5625000223517418,
      "reward_std": 0.16316584311425686,
      "rewards/accuracy_reward": 0.1406250111758709,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 3093
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1609.1340026855469,
      "epoch": 0.9242028227914271,
      "grad_norm": 19.142261505126953,
      "kl": 2.734375,
      "learning_rate": 8.654046171424817e-09,
      "loss": 0.187,
      "reward": 0.5251116454601288,
      "reward_std": 0.09467749111354351,
      "rewards/accuracy_reward": 0.0781250037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4469866305589676,
      "step": 3094
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1746.68310546875,
      "epoch": 0.9245015308789486,
      "grad_norm": 6.223680019378662,
      "kl": 3.29296875,
      "learning_rate": 8.586163482282388e-09,
      "loss": 0.2091,
      "reward": 0.5022321715950966,
      "reward_std": 0.15067282505333424,
      "rewards/accuracy_reward": 0.07812500302679837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.424107164144516,
      "step": 3095
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1801.8728332519531,
      "epoch": 0.92480023896647,
      "grad_norm": 20.50689125061035,
      "kl": 4.59375,
      "learning_rate": 8.518543427732949e-09,
      "loss": 0.2264,
      "reward": 0.4414062723517418,
      "reward_std": 0.15418160147964954,
      "rewards/accuracy_reward": 0.03571428661234677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4056919813156128,
      "step": 3096
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1763.696533203125,
      "epoch": 0.9250989470539915,
      "grad_norm": 14.881399154663086,
      "kl": 2.71875,
      "learning_rate": 8.451186081340516e-09,
      "loss": 0.1606,
      "reward": 0.5011160969734192,
      "reward_std": 0.14176259748637676,
      "rewards/accuracy_reward": 0.06473214668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4363839477300644,
      "step": 3097
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1813.4331359863281,
      "epoch": 0.925397655141513,
      "grad_norm": 4.612854480743408,
      "kl": 4.1796875,
      "learning_rate": 8.384091516383363e-09,
      "loss": 0.2359,
      "reward": 0.4531250223517418,
      "reward_std": 0.16692296788096428,
      "rewards/accuracy_reward": 0.04464286006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4084821566939354,
      "step": 3098
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1691.0179443359375,
      "epoch": 0.9256963632290345,
      "grad_norm": 14.503144264221191,
      "kl": 3.51171875,
      "learning_rate": 8.317259805853682e-09,
      "loss": 0.2401,
      "reward": 0.617745578289032,
      "reward_std": 0.16978689655661583,
      "rewards/accuracy_reward": 0.1897321529686451,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428013414144516,
      "step": 3099
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1752.8773498535156,
      "epoch": 0.9259950713165559,
      "grad_norm": 5.220859050750732,
      "kl": 3.9453125,
      "learning_rate": 8.250691022457872e-09,
      "loss": 0.2274,
      "reward": 0.5731026977300644,
      "reward_std": 0.12308023124933243,
      "rewards/accuracy_reward": 0.1540178619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4190848395228386,
      "step": 3100
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1750.0202026367188,
      "epoch": 0.9262937794040773,
      "grad_norm": 3.6601431369781494,
      "kl": 3.54296875,
      "learning_rate": 8.18438523861631e-09,
      "loss": 0.2011,
      "reward": 0.5223214477300644,
      "reward_std": 0.15881632268428802,
      "rewards/accuracy_reward": 0.09598214877769351,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263393059372902,
      "step": 3101
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1783.2634887695312,
      "epoch": 0.9265924874915988,
      "grad_norm": 18.882341384887695,
      "kl": 3.4375,
      "learning_rate": 8.118342526463068e-09,
      "loss": 0.2169,
      "reward": 0.524553582072258,
      "reward_std": 0.1694175899028778,
      "rewards/accuracy_reward": 0.098214291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263393059372902,
      "step": 3102
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1798.60498046875,
      "epoch": 0.9268911955791203,
      "grad_norm": 8.143572807312012,
      "kl": 3.80859375,
      "learning_rate": 8.052562957846231e-09,
      "loss": 0.2106,
      "reward": 0.4693080559372902,
      "reward_std": 0.17779669910669327,
      "rewards/accuracy_reward": 0.044642859138548374,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4246651977300644,
      "step": 3103
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1819.1072082519531,
      "epoch": 0.9271899036666418,
      "grad_norm": 8.65472412109375,
      "kl": 3.60546875,
      "learning_rate": 7.98704660432753e-09,
      "loss": 0.2099,
      "reward": 0.5619419887661934,
      "reward_std": 0.1586289443075657,
      "rewards/accuracy_reward": 0.1383928656578064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.423549123108387,
      "step": 3104
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1720.1273193359375,
      "epoch": 0.9274886117541632,
      "grad_norm": 5.983371734619141,
      "kl": 4.0703125,
      "learning_rate": 7.921793537182253e-09,
      "loss": 0.2433,
      "reward": 0.4676339477300644,
      "reward_std": 0.16578348353505135,
      "rewards/accuracy_reward": 0.044642859138548374,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 3105
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1735.9308776855469,
      "epoch": 0.9277873198416847,
      "grad_norm": 6.4985761642456055,
      "kl": 3.88671875,
      "learning_rate": 7.856803827399422e-09,
      "loss": 0.234,
      "reward": 0.5491071790456772,
      "reward_std": 0.17311876267194748,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.424107164144516,
      "step": 3106
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1773.8706359863281,
      "epoch": 0.9280860279292061,
      "grad_norm": 8.45417594909668,
      "kl": 3.65625,
      "learning_rate": 7.792077545681452e-09,
      "loss": 0.2069,
      "reward": 0.5100446566939354,
      "reward_std": 0.12487944401800632,
      "rewards/accuracy_reward": 0.0848214328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232313156128,
      "step": 3107
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1744.1719665527344,
      "epoch": 0.9283847360167277,
      "grad_norm": 19.50294303894043,
      "kl": 3.54296875,
      "learning_rate": 7.727614762444157e-09,
      "loss": 0.2339,
      "reward": 0.4715401977300644,
      "reward_std": 0.12681268900632858,
      "rewards/accuracy_reward": 0.044642860535532236,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.426897332072258,
      "step": 3108
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1741.77685546875,
      "epoch": 0.9286834441042491,
      "grad_norm": 5.543459892272949,
      "kl": 3.41015625,
      "learning_rate": 7.66341554781677e-09,
      "loss": 0.1953,
      "reward": 0.514508955180645,
      "reward_std": 0.16556296683847904,
      "rewards/accuracy_reward": 0.08035714412108064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341518133878708,
      "step": 3109
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1782.1875610351562,
      "epoch": 0.9289821521917706,
      "grad_norm": 8.373510360717773,
      "kl": 4.515625,
      "learning_rate": 7.59947997164173e-09,
      "loss": 0.2663,
      "reward": 0.4927455559372902,
      "reward_std": 0.1440216265618801,
      "rewards/accuracy_reward": 0.08258929033763707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4101562723517418,
      "step": 3110
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1699.5335693359375,
      "epoch": 0.929280860279292,
      "grad_norm": 9.046926498413086,
      "kl": 3.68359375,
      "learning_rate": 7.53580810347465e-09,
      "loss": 0.2108,
      "reward": 0.5178571566939354,
      "reward_std": 0.14192751049995422,
      "rewards/accuracy_reward": 0.08928571688011289,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 3111
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1775.0982971191406,
      "epoch": 0.9295795683668135,
      "grad_norm": 6.791934490203857,
      "kl": 3.8671875,
      "learning_rate": 7.47240001258434e-09,
      "loss": 0.2312,
      "reward": 0.5797991305589676,
      "reward_std": 0.1971987523138523,
      "rewards/accuracy_reward": 0.16517857648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205559372902,
      "step": 3112
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1777.4620971679688,
      "epoch": 0.929878276454335,
      "grad_norm": 20.01893424987793,
      "kl": 4.6796875,
      "learning_rate": 7.409255767952538e-09,
      "loss": 0.277,
      "reward": 0.5295759290456772,
      "reward_std": 0.16408658772706985,
      "rewards/accuracy_reward": 0.1183035746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4112723469734192,
      "step": 3113
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1777.0268859863281,
      "epoch": 0.9301769845418565,
      "grad_norm": 10.58565616607666,
      "kl": 2.490234375,
      "learning_rate": 7.3463754382740165e-09,
      "loss": 0.1567,
      "reward": 0.5552455633878708,
      "reward_std": 0.1887827143073082,
      "rewards/accuracy_reward": 0.11830358020961285,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419813156128,
      "step": 3114
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1721.7210388183594,
      "epoch": 0.9304756926293779,
      "grad_norm": 5.684086322784424,
      "kl": 2.814453125,
      "learning_rate": 7.283759091956443e-09,
      "loss": 0.1601,
      "reward": 0.4827009066939354,
      "reward_std": 0.13451972976326942,
      "rewards/accuracy_reward": 0.0602678619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330484867096,
      "step": 3115
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1728.8438415527344,
      "epoch": 0.9307744007168994,
      "grad_norm": 4.802729606628418,
      "kl": 3.37890625,
      "learning_rate": 7.2214067971201885e-09,
      "loss": 0.1962,
      "reward": 0.6389509290456772,
      "reward_std": 0.21386107802391052,
      "rewards/accuracy_reward": 0.2075892984867096,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4313616156578064,
      "step": 3116
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1669.9085388183594,
      "epoch": 0.9310731088044208,
      "grad_norm": 17.05530548095703,
      "kl": 3.48046875,
      "learning_rate": 7.159318621598465e-09,
      "loss": 0.2257,
      "reward": 0.6227678880095482,
      "reward_std": 0.15726906433701515,
      "rewards/accuracy_reward": 0.1852678693830967,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000149011612,
      "step": 3117
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1734.29248046875,
      "epoch": 0.9313718168919424,
      "grad_norm": 13.835975646972656,
      "kl": 3.3828125,
      "learning_rate": 7.097494632937184e-09,
      "loss": 0.2097,
      "reward": 0.5128348395228386,
      "reward_std": 0.13877625204622746,
      "rewards/accuracy_reward": 0.07142857578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4414062723517418,
      "step": 3118
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1813.4755249023438,
      "epoch": 0.9316705249794638,
      "grad_norm": 4.7541327476501465,
      "kl": 3.427734375,
      "learning_rate": 7.035934898394713e-09,
      "loss": 0.1887,
      "reward": 0.5758928805589676,
      "reward_std": 0.18983046524226665,
      "rewards/accuracy_reward": 0.14285714738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357313156128,
      "step": 3119
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1725.618408203125,
      "epoch": 0.9319692330669853,
      "grad_norm": 12.552999496459961,
      "kl": 2.859375,
      "learning_rate": 6.974639484942063e-09,
      "loss": 0.1765,
      "reward": 0.5189732387661934,
      "reward_std": 0.10459255427122116,
      "rewards/accuracy_reward": 0.07812500488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.440848246216774,
      "step": 3120
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1796.5960388183594,
      "epoch": 0.9322679411545067,
      "grad_norm": 8.727331161499023,
      "kl": 4.015625,
      "learning_rate": 6.91360845926256e-09,
      "loss": 0.2201,
      "reward": 0.4497768059372902,
      "reward_std": 0.13696794025599957,
      "rewards/accuracy_reward": 0.022321430034935474,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.427455373108387,
      "step": 3121
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1729.83935546875,
      "epoch": 0.9325666492420283,
      "grad_norm": 9.591011047363281,
      "kl": 4.203125,
      "learning_rate": 6.852841887752037e-09,
      "loss": 0.2477,
      "reward": 0.5357143059372902,
      "reward_std": 0.18371436186134815,
      "rewards/accuracy_reward": 0.11607143469154835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4196428656578064,
      "step": 3122
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1759.2545471191406,
      "epoch": 0.9328653573295497,
      "grad_norm": 22.989971160888672,
      "kl": 4.265625,
      "learning_rate": 6.7923398365185844e-09,
      "loss": 0.229,
      "reward": 0.4977678805589676,
      "reward_std": 0.14584042876958847,
      "rewards/accuracy_reward": 0.0669642873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035895228386,
      "step": 3123
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1756.4465026855469,
      "epoch": 0.9331640654170712,
      "grad_norm": 9.116214752197266,
      "kl": 4.5,
      "learning_rate": 6.732102371382442e-09,
      "loss": 0.2603,
      "reward": 0.5089285895228386,
      "reward_std": 0.17124420404434204,
      "rewards/accuracy_reward": 0.08705357578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750149011612,
      "step": 3124
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1755.65185546875,
      "epoch": 0.9334627735045926,
      "grad_norm": 22.127580642700195,
      "kl": 5.46875,
      "learning_rate": 6.672129557876133e-09,
      "loss": 0.3201,
      "reward": 0.5697544887661934,
      "reward_std": 0.16058344766497612,
      "rewards/accuracy_reward": 0.15625000605359674,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4135044813156128,
      "step": 3125
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1812.8594665527344,
      "epoch": 0.9337614815921141,
      "grad_norm": 6.615753173828125,
      "kl": 3.001953125,
      "learning_rate": 6.612421461244161e-09,
      "loss": 0.1564,
      "reward": 0.5569196715950966,
      "reward_std": 0.16876626014709473,
      "rewards/accuracy_reward": 0.129464291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.427455373108387,
      "step": 3126
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1736.3505249023438,
      "epoch": 0.9340601896796356,
      "grad_norm": 12.076683044433594,
      "kl": 3.9609375,
      "learning_rate": 6.552978146443094e-09,
      "loss": 0.2644,
      "reward": 0.4631696566939354,
      "reward_std": 0.1319701299071312,
      "rewards/accuracy_reward": 0.03348214295692742,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875149011612,
      "step": 3127
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1780.7813110351562,
      "epoch": 0.9343588977671571,
      "grad_norm": 35.0097770690918,
      "kl": 5.09765625,
      "learning_rate": 6.493799678141426e-09,
      "loss": 0.2522,
      "reward": 0.6395089626312256,
      "reward_std": 0.15091378800570965,
      "rewards/accuracy_reward": 0.2209821492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268059372902,
      "step": 3128
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1768.3148193359375,
      "epoch": 0.9346576058546785,
      "grad_norm": 14.694561958312988,
      "kl": 2.734375,
      "learning_rate": 6.434886120719546e-09,
      "loss": 0.1725,
      "reward": 0.5117187649011612,
      "reward_std": 0.15856457501649857,
      "rewards/accuracy_reward": 0.0758928582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4358259066939354,
      "step": 3129
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1761.3550109863281,
      "epoch": 0.9349563139422,
      "grad_norm": 7.165038108825684,
      "kl": 4.32421875,
      "learning_rate": 6.376237538269602e-09,
      "loss": 0.264,
      "reward": 0.4760044887661934,
      "reward_std": 0.1332947574555874,
      "rewards/accuracy_reward": 0.05357143236324191,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330633878708,
      "step": 3130
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1797.0804443359375,
      "epoch": 0.9352550220297214,
      "grad_norm": 17.504234313964844,
      "kl": 4.69921875,
      "learning_rate": 6.317853994595529e-09,
      "loss": 0.2485,
      "reward": 0.431361623108387,
      "reward_std": 0.13290254212915897,
      "rewards/accuracy_reward": 0.02008928661234677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.411272332072258,
      "step": 3131
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1735.5134582519531,
      "epoch": 0.935553730117243,
      "grad_norm": 11.88955307006836,
      "kl": 4.0078125,
      "learning_rate": 6.259735553212908e-09,
      "loss": 0.208,
      "reward": 0.4654018059372902,
      "reward_std": 0.16931617259979248,
      "rewards/accuracy_reward": 0.03794643026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4274553805589676,
      "step": 3132
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1741.9822387695312,
      "epoch": 0.9358524382047644,
      "grad_norm": 3.674797296524048,
      "kl": 3.75390625,
      "learning_rate": 6.201882277348885e-09,
      "loss": 0.1923,
      "reward": 0.5580357313156128,
      "reward_std": 0.1989150382578373,
      "rewards/accuracy_reward": 0.13616071827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750149011612,
      "step": 3133
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1712.0313110351562,
      "epoch": 0.9361511462922859,
      "grad_norm": 21.303447723388672,
      "kl": 4.01171875,
      "learning_rate": 6.144294229942171e-09,
      "loss": 0.261,
      "reward": 0.502790205180645,
      "reward_std": 0.11032152362167835,
      "rewards/accuracy_reward": 0.0736607164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294887661934,
      "step": 3134
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1724.8438110351562,
      "epoch": 0.9364498543798073,
      "grad_norm": 8.405097961425781,
      "kl": 3.533203125,
      "learning_rate": 6.086971473642871e-09,
      "loss": 0.2215,
      "reward": 0.5541294813156128,
      "reward_std": 0.14196583814918995,
      "rewards/accuracy_reward": 0.1160714328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4380580559372902,
      "step": 3135
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1806.8125915527344,
      "epoch": 0.9367485624673288,
      "grad_norm": 17.944316864013672,
      "kl": 4.68359375,
      "learning_rate": 6.029914070812575e-09,
      "loss": 0.249,
      "reward": 0.5385044813156128,
      "reward_std": 0.1912555918097496,
      "rewards/accuracy_reward": 0.12946429289877415,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4090401977300644,
      "step": 3136
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1732.7255249023438,
      "epoch": 0.9370472705548503,
      "grad_norm": 6.294620990753174,
      "kl": 3.93359375,
      "learning_rate": 5.973122083524157e-09,
      "loss": 0.2346,
      "reward": 0.5625000223517418,
      "reward_std": 0.14301216043531895,
      "rewards/accuracy_reward": 0.1339285783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 3137
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1776.2366943359375,
      "epoch": 0.9373459786423718,
      "grad_norm": 15.142308235168457,
      "kl": 3.6484375,
      "learning_rate": 5.916595573561667e-09,
      "loss": 0.2258,
      "reward": 0.5050223469734192,
      "reward_std": 0.17098475247621536,
      "rewards/accuracy_reward": 0.07589285960420966,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294887661934,
      "step": 3138
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1749.7255249023438,
      "epoch": 0.9376446867298932,
      "grad_norm": 6.567526817321777,
      "kl": 3.203125,
      "learning_rate": 5.8603346024204694e-09,
      "loss": 0.194,
      "reward": 0.4743303656578064,
      "reward_std": 0.12915113940835,
      "rewards/accuracy_reward": 0.051339288242161274,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 3139
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1831.8572082519531,
      "epoch": 0.9379433948174147,
      "grad_norm": 12.231660842895508,
      "kl": 3.6015625,
      "learning_rate": 5.804339231306993e-09,
      "loss": 0.2138,
      "reward": 0.4860491305589676,
      "reward_std": 0.13884839043021202,
      "rewards/accuracy_reward": 0.053571430034935474,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.432477705180645,
      "step": 3140
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1788.8862609863281,
      "epoch": 0.9382421029049361,
      "grad_norm": 8.83573055267334,
      "kl": 4.6328125,
      "learning_rate": 5.748609521138648e-09,
      "loss": 0.2445,
      "reward": 0.5089285969734192,
      "reward_std": 0.1527208685874939,
      "rewards/accuracy_reward": 0.0937500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4151785895228386,
      "step": 3141
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1702.3371276855469,
      "epoch": 0.9385408109924577,
      "grad_norm": 7.195098876953125,
      "kl": 3.83984375,
      "learning_rate": 5.693145532543936e-09,
      "loss": 0.239,
      "reward": 0.5959821715950966,
      "reward_std": 0.14992650970816612,
      "rewards/accuracy_reward": 0.1718750074505806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.424107164144516,
      "step": 3142
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1869.4532165527344,
      "epoch": 0.9388395190799791,
      "grad_norm": 5.760509490966797,
      "kl": 3.953125,
      "learning_rate": 5.637947325862258e-09,
      "loss": 0.2181,
      "reward": 0.4174107313156128,
      "reward_std": 0.13676492683589458,
      "rewards/accuracy_reward": 0.008928572060540318,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4084821566939354,
      "step": 3143
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1748.0916137695312,
      "epoch": 0.9391382271675005,
      "grad_norm": 16.554542541503906,
      "kl": 2.84375,
      "learning_rate": 5.5830149611438286e-09,
      "loss": 0.1715,
      "reward": 0.5496651977300644,
      "reward_std": 0.15513592213392258,
      "rewards/accuracy_reward": 0.11607143701985478,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937649011612,
      "step": 3144
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1800.3772888183594,
      "epoch": 0.939436935255022,
      "grad_norm": 19.81356430053711,
      "kl": 3.03515625,
      "learning_rate": 5.5283484981496484e-09,
      "loss": 0.194,
      "reward": 0.4687500223517418,
      "reward_std": 0.14997711405158043,
      "rewards/accuracy_reward": 0.026785716181620955,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4419643059372902,
      "step": 3145
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1613.5157165527344,
      "epoch": 0.9397356433425434,
      "grad_norm": 10.664583206176758,
      "kl": 2.736328125,
      "learning_rate": 5.4739479963515635e-09,
      "loss": 0.1909,
      "reward": 0.6250000298023224,
      "reward_std": 0.17824650555849075,
      "rewards/accuracy_reward": 0.1785714328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4464285895228386,
      "step": 3146
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1804.8906860351562,
      "epoch": 0.940034351430065,
      "grad_norm": 12.780122756958008,
      "kl": 4.765625,
      "learning_rate": 5.419813514931898e-09,
      "loss": 0.251,
      "reward": 0.5440848395228386,
      "reward_std": 0.22799570858478546,
      "rewards/accuracy_reward": 0.13616072107106447,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4079241305589676,
      "step": 3147
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1717.0223999023438,
      "epoch": 0.9403330595175864,
      "grad_norm": 7.646012783050537,
      "kl": 4.01171875,
      "learning_rate": 5.36594511278371e-09,
      "loss": 0.2488,
      "reward": 0.4743303805589676,
      "reward_std": 0.14721242897212505,
      "rewards/accuracy_reward": 0.053571430733427405,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4207589402794838,
      "step": 3148
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1827.399658203125,
      "epoch": 0.9406317676051079,
      "grad_norm": 7.611230373382568,
      "kl": 3.85546875,
      "learning_rate": 5.312342848510509e-09,
      "loss": 0.2185,
      "reward": 0.4960937723517418,
      "reward_std": 0.21288838610053062,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.424665205180645,
      "step": 3149
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1727.5871276855469,
      "epoch": 0.9409304756926293,
      "grad_norm": 7.610684871673584,
      "kl": 3.70703125,
      "learning_rate": 5.2590067804263175e-09,
      "loss": 0.2226,
      "reward": 0.5641741305589676,
      "reward_std": 0.22518694400787354,
      "rewards/accuracy_reward": 0.13839286309666932,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812723517418,
      "step": 3150
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1803.4197387695312,
      "epoch": 0.9412291837801509,
      "grad_norm": 10.031593322753906,
      "kl": 3.064453125,
      "learning_rate": 5.20593696655558e-09,
      "loss": 0.174,
      "reward": 0.4609375074505806,
      "reward_std": 0.1403737049549818,
      "rewards/accuracy_reward": 0.0379464291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 3151
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1793.4666137695312,
      "epoch": 0.9415278918676723,
      "grad_norm": 6.310960292816162,
      "kl": 3.25,
      "learning_rate": 5.153133464632975e-09,
      "loss": 0.2044,
      "reward": 0.550781287252903,
      "reward_std": 0.1295801568776369,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812723517418,
      "step": 3152
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1776.6384582519531,
      "epoch": 0.9418265999551938,
      "grad_norm": 10.475260734558105,
      "kl": 3.171875,
      "learning_rate": 5.100596332103635e-09,
      "loss": 0.2002,
      "reward": 0.458705373108387,
      "reward_std": 0.16390957683324814,
      "rewards/accuracy_reward": 0.03571428661234677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 3153
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1749.4219360351562,
      "epoch": 0.9421253080427152,
      "grad_norm": 4.847557067871094,
      "kl": 3.6796875,
      "learning_rate": 5.048325626122757e-09,
      "loss": 0.2235,
      "reward": 0.5139509215950966,
      "reward_std": 0.18066633492708206,
      "rewards/accuracy_reward": 0.08482143422588706,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294813156128,
      "step": 3154
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1818.08935546875,
      "epoch": 0.9424240161302367,
      "grad_norm": 5.6882171630859375,
      "kl": 4.109375,
      "learning_rate": 4.996321403555742e-09,
      "loss": 0.231,
      "reward": 0.4637276977300644,
      "reward_std": 0.1563696563243866,
      "rewards/accuracy_reward": 0.05357143236324191,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4101562649011612,
      "step": 3155
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1827.3996276855469,
      "epoch": 0.9427227242177582,
      "grad_norm": 23.35978889465332,
      "kl": 5.1953125,
      "learning_rate": 4.944583720978085e-09,
      "loss": 0.2714,
      "reward": 0.4787946715950966,
      "reward_std": 0.1769043579697609,
      "rewards/accuracy_reward": 0.07142857694998384,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4073660895228386,
      "step": 3156
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1712.5804443359375,
      "epoch": 0.9430214323052797,
      "grad_norm": 12.596489906311035,
      "kl": 3.16796875,
      "learning_rate": 4.893112634675372e-09,
      "loss": 0.2033,
      "reward": 0.573660746216774,
      "reward_std": 0.2282893806695938,
      "rewards/accuracy_reward": 0.1316964328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4419643059372902,
      "step": 3157
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1818.4777526855469,
      "epoch": 0.9433201403928011,
      "grad_norm": 16.636917114257812,
      "kl": 3.865234375,
      "learning_rate": 4.841908200643008e-09,
      "loss": 0.2199,
      "reward": 0.4960937723517418,
      "reward_std": 0.16121109575033188,
      "rewards/accuracy_reward": 0.06919643096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.426897332072258,
      "step": 3158
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1700.7120971679688,
      "epoch": 0.9436188484803226,
      "grad_norm": 6.981161117553711,
      "kl": 3.98046875,
      "learning_rate": 4.790970474586459e-09,
      "loss": 0.2366,
      "reward": 0.5719866305589676,
      "reward_std": 0.18915634788572788,
      "rewards/accuracy_reward": 0.145089291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4268973469734192,
      "step": 3159
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1741.5134887695312,
      "epoch": 0.943917556567844,
      "grad_norm": 7.836648464202881,
      "kl": 4.546875,
      "learning_rate": 4.740299511920981e-09,
      "loss": 0.2752,
      "reward": 0.5357143133878708,
      "reward_std": 0.12292356602847576,
      "rewards/accuracy_reward": 0.1160714365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4196428805589676,
      "step": 3160
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1727.7009582519531,
      "epoch": 0.9442162646553656,
      "grad_norm": 6.599543571472168,
      "kl": 3.65625,
      "learning_rate": 4.68989536777159e-09,
      "loss": 0.2202,
      "reward": 0.5234375223517418,
      "reward_std": 0.19363130629062653,
      "rewards/accuracy_reward": 0.0870535746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.436383955180645,
      "step": 3161
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1778.6965026855469,
      "epoch": 0.944514972742887,
      "grad_norm": 8.948225975036621,
      "kl": 4.51953125,
      "learning_rate": 4.639758096973062e-09,
      "loss": 0.2605,
      "reward": 0.518415205180645,
      "reward_std": 0.1995774433016777,
      "rewards/accuracy_reward": 0.10267857648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.415736623108387,
      "step": 3162
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1823.5625915527344,
      "epoch": 0.9448136808304085,
      "grad_norm": 6.4680304527282715,
      "kl": 4.5390625,
      "learning_rate": 4.589887754069849e-09,
      "loss": 0.2601,
      "reward": 0.4732143059372902,
      "reward_std": 0.14428245462477207,
      "rewards/accuracy_reward": 0.0625000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4107142984867096,
      "step": 3163
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1729.2143859863281,
      "epoch": 0.9451123889179299,
      "grad_norm": 11.300824165344238,
      "kl": 4.7421875,
      "learning_rate": 4.540284393315969e-09,
      "loss": 0.2796,
      "reward": 0.494419664144516,
      "reward_std": 0.16322331316769123,
      "rewards/accuracy_reward": 0.06919643236324191,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232313156128,
      "step": 3164
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1736.2456359863281,
      "epoch": 0.9454110970054515,
      "grad_norm": 5.113112449645996,
      "kl": 4.37109375,
      "learning_rate": 4.49094806867506e-09,
      "loss": 0.2704,
      "reward": 0.4698660895228386,
      "reward_std": 0.19220685958862305,
      "rewards/accuracy_reward": 0.049107144586741924,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4207589477300644,
      "step": 3165
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1829.5469360351562,
      "epoch": 0.9457098050929729,
      "grad_norm": 24.551511764526367,
      "kl": 4.70703125,
      "learning_rate": 4.441878833820134e-09,
      "loss": 0.2674,
      "reward": 0.4871651977300644,
      "reward_std": 0.18432844430208206,
      "rewards/accuracy_reward": 0.08258928917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4045759066939354,
      "step": 3166
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1872.2009582519531,
      "epoch": 0.9460085131804944,
      "grad_norm": 34.3686408996582,
      "kl": 6.0078125,
      "learning_rate": 4.393076742133794e-09,
      "loss": 0.3151,
      "reward": 0.4386160895228386,
      "reward_std": 0.18710462376475334,
      "rewards/accuracy_reward": 0.04464285960420966,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3939732313156128,
      "step": 3167
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1805.9911193847656,
      "epoch": 0.9463072212680158,
      "grad_norm": 14.207158088684082,
      "kl": 5.05859375,
      "learning_rate": 4.344541846707878e-09,
      "loss": 0.2826,
      "reward": 0.463727705180645,
      "reward_std": 0.14076603762805462,
      "rewards/accuracy_reward": 0.055803575087338686,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.407924123108387,
      "step": 3168
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1703.6741943359375,
      "epoch": 0.9466059293555373,
      "grad_norm": 4.945898532867432,
      "kl": 3.65625,
      "learning_rate": 4.2962742003435916e-09,
      "loss": 0.238,
      "reward": 0.5641741305589676,
      "reward_std": 0.17153637669980526,
      "rewards/accuracy_reward": 0.1250000074505806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4391741305589676,
      "step": 3169
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1794.618408203125,
      "epoch": 0.9469046374430587,
      "grad_norm": 5.989921569824219,
      "kl": 3.93359375,
      "learning_rate": 4.2482738555514596e-09,
      "loss": 0.21,
      "reward": 0.4386160969734192,
      "reward_std": 0.15121140331029892,
      "rewards/accuracy_reward": 0.029017859138548374,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4095982313156128,
      "step": 3170
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1811.1653137207031,
      "epoch": 0.9472033455305803,
      "grad_norm": 5.36673641204834,
      "kl": 3.89453125,
      "learning_rate": 4.2005408645510995e-09,
      "loss": 0.2284,
      "reward": 0.490513414144516,
      "reward_std": 0.16210117377340794,
      "rewards/accuracy_reward": 0.07142857648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4190848469734192,
      "step": 3171
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1720.30810546875,
      "epoch": 0.9475020536181017,
      "grad_norm": 7.075869083404541,
      "kl": 4.4296875,
      "learning_rate": 4.1530752792713345e-09,
      "loss": 0.2756,
      "reward": 0.550781287252903,
      "reward_std": 0.17472469806671143,
      "rewards/accuracy_reward": 0.12723214668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4235491305589676,
      "step": 3172
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1710.9777526855469,
      "epoch": 0.9478007617056232,
      "grad_norm": 6.183236122131348,
      "kl": 3.9375,
      "learning_rate": 4.105877151350079e-09,
      "loss": 0.25,
      "reward": 0.5652901977300644,
      "reward_std": 0.20641006901860237,
      "rewards/accuracy_reward": 0.1383928619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4268973469734192,
      "step": 3173
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1810.16748046875,
      "epoch": 0.9480994697931446,
      "grad_norm": 13.622652053833008,
      "kl": 3.55859375,
      "learning_rate": 4.058946532134261e-09,
      "loss": 0.2034,
      "reward": 0.4614955633878708,
      "reward_std": 0.15879039093852043,
      "rewards/accuracy_reward": 0.0379464291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4235491305589676,
      "step": 3174
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1698.0603332519531,
      "epoch": 0.9483981778806662,
      "grad_norm": 5.101284980773926,
      "kl": 3.421875,
      "learning_rate": 4.0122834726798175e-09,
      "loss": 0.2153,
      "reward": 0.5904018133878708,
      "reward_std": 0.18477614410221577,
      "rewards/accuracy_reward": 0.1517857201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4386160969734192,
      "step": 3175
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1785.2188110351562,
      "epoch": 0.9486968859681876,
      "grad_norm": 9.613402366638184,
      "kl": 3.35546875,
      "learning_rate": 3.965888023751584e-09,
      "loss": 0.2111,
      "reward": 0.5011160895228386,
      "reward_std": 0.19675280526280403,
      "rewards/accuracy_reward": 0.07142857206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875149011612,
      "step": 3176
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1790.4621276855469,
      "epoch": 0.9489955940557091,
      "grad_norm": 8.176694869995117,
      "kl": 4.2734375,
      "learning_rate": 3.919760235823216e-09,
      "loss": 0.2368,
      "reward": 0.4771205633878708,
      "reward_std": 0.16108850575983524,
      "rewards/accuracy_reward": 0.0580357164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4190848469734192,
      "step": 3177
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1786.6697082519531,
      "epoch": 0.9492943021432305,
      "grad_norm": 6.700039863586426,
      "kl": 4.3359375,
      "learning_rate": 3.873900159077264e-09,
      "loss": 0.2448,
      "reward": 0.4637276977300644,
      "reward_std": 0.12255089730024338,
      "rewards/accuracy_reward": 0.04687500232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.416852705180645,
      "step": 3178
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1768.2344360351562,
      "epoch": 0.949593010230752,
      "grad_norm": 7.902632236480713,
      "kl": 4.8125,
      "learning_rate": 3.8283078434049875e-09,
      "loss": 0.2684,
      "reward": 0.4614955559372902,
      "reward_std": 0.119241613894701,
      "rewards/accuracy_reward": 0.0535714328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4079241305589676,
      "step": 3179
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1741.6384887695312,
      "epoch": 0.9498917183182735,
      "grad_norm": 4.458237171173096,
      "kl": 3.6015625,
      "learning_rate": 3.782983338406348e-09,
      "loss": 0.2103,
      "reward": 0.5864955559372902,
      "reward_std": 0.15262810699641705,
      "rewards/accuracy_reward": 0.15625000861473382,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455559372902,
      "step": 3180
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1772.41748046875,
      "epoch": 0.950190426405795,
      "grad_norm": 5.634639263153076,
      "kl": 3.90625,
      "learning_rate": 3.7379266933900145e-09,
      "loss": 0.2275,
      "reward": 0.5848214477300644,
      "reward_std": 0.22271934151649475,
      "rewards/accuracy_reward": 0.1540178693830967,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035969734192,
      "step": 3181
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1721.69873046875,
      "epoch": 0.9504891344933164,
      "grad_norm": 10.401625633239746,
      "kl": 2.9140625,
      "learning_rate": 3.6931379573731105e-09,
      "loss": 0.1685,
      "reward": 0.6082589477300644,
      "reward_std": 0.14908975176513195,
      "rewards/accuracy_reward": 0.1718750111758709,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4363839477300644,
      "step": 3182
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1759.9844665527344,
      "epoch": 0.9507878425808379,
      "grad_norm": 13.341290473937988,
      "kl": 3.6640625,
      "learning_rate": 3.6486171790814647e-09,
      "loss": 0.2179,
      "reward": 0.5373884290456772,
      "reward_std": 0.14686298184096813,
      "rewards/accuracy_reward": 0.1116071455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812723517418,
      "step": 3183
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1674.2411804199219,
      "epoch": 0.9510865506683593,
      "grad_norm": 14.428565979003906,
      "kl": 2.9921875,
      "learning_rate": 3.6043644069493064e-09,
      "loss": 0.1993,
      "reward": 0.5239955708384514,
      "reward_std": 0.18549969606101513,
      "rewards/accuracy_reward": 0.08705357392318547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419813156128,
      "step": 3184
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1761.3014221191406,
      "epoch": 0.9513852587558809,
      "grad_norm": 8.357264518737793,
      "kl": 4.06640625,
      "learning_rate": 3.560379689119292e-09,
      "loss": 0.2379,
      "reward": 0.4726562649011612,
      "reward_std": 0.1429803017526865,
      "rewards/accuracy_reward": 0.06473214738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.407924123108387,
      "step": 3185
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1757.9643859863281,
      "epoch": 0.9516839668434023,
      "grad_norm": 4.255255222320557,
      "kl": 3.6875,
      "learning_rate": 3.5166630734425052e-09,
      "loss": 0.2305,
      "reward": 0.4771205633878708,
      "reward_std": 0.14277459122240543,
      "rewards/accuracy_reward": 0.049107146449387074,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428013414144516,
      "step": 3186
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1743.0982971191406,
      "epoch": 0.9519826749309237,
      "grad_norm": 6.341939449310303,
      "kl": 3.50390625,
      "learning_rate": 3.4732146074783477e-09,
      "loss": 0.2226,
      "reward": 0.4776785969734192,
      "reward_std": 0.17072607204318047,
      "rewards/accuracy_reward": 0.04464286006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357313156128,
      "step": 3187
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1778.85498046875,
      "epoch": 0.9522813830184452,
      "grad_norm": 19.17293357849121,
      "kl": 3.181640625,
      "learning_rate": 3.430034338494453e-09,
      "loss": 0.1825,
      "reward": 0.5312500298023224,
      "reward_std": 0.18148424476385117,
      "rewards/accuracy_reward": 0.0937500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000149011612,
      "step": 3188
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1834.7188415527344,
      "epoch": 0.9525800911059666,
      "grad_norm": 10.420117378234863,
      "kl": 4.1796875,
      "learning_rate": 3.387122313466745e-09,
      "loss": 0.2216,
      "reward": 0.5446428880095482,
      "reward_std": 0.18158254399895668,
      "rewards/accuracy_reward": 0.14062500558793545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4040178805589676,
      "step": 3189
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1749.8750610351562,
      "epoch": 0.9528787991934882,
      "grad_norm": 6.479268550872803,
      "kl": 4.19140625,
      "learning_rate": 3.344478579079296e-09,
      "loss": 0.2208,
      "reward": 0.545758955180645,
      "reward_std": 0.13298838585615158,
      "rewards/accuracy_reward": 0.12500000861473382,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4207589477300644,
      "step": 3190
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1747.2969360351562,
      "epoch": 0.9531775072810096,
      "grad_norm": 13.04605770111084,
      "kl": 4.15234375,
      "learning_rate": 3.3021031817243305e-09,
      "loss": 0.2672,
      "reward": 0.5485491305589676,
      "reward_std": 0.13957325369119644,
      "rewards/accuracy_reward": 0.12500000419095159,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.423549123108387,
      "step": 3191
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1803.9643249511719,
      "epoch": 0.9534762153685311,
      "grad_norm": 8.232499122619629,
      "kl": 4.0859375,
      "learning_rate": 3.25999616750211e-09,
      "loss": 0.2353,
      "reward": 0.5133928805589676,
      "reward_std": 0.14984129182994366,
      "rewards/accuracy_reward": 0.1071428656578064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500149011612,
      "step": 3192
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1763.87060546875,
      "epoch": 0.9537749234560525,
      "grad_norm": 5.395274639129639,
      "kl": 4.078125,
      "learning_rate": 3.2181575822209373e-09,
      "loss": 0.2314,
      "reward": 0.5418526977300644,
      "reward_std": 0.1361988317221403,
      "rewards/accuracy_reward": 0.12500000605359674,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4168526977300644,
      "step": 3193
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1772.0290832519531,
      "epoch": 0.954073631543574,
      "grad_norm": 15.65932846069336,
      "kl": 5.5078125,
      "learning_rate": 3.1765874713970685e-09,
      "loss": 0.3223,
      "reward": 0.4854910895228386,
      "reward_std": 0.17267966642975807,
      "rewards/accuracy_reward": 0.08482143096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4006696566939354,
      "step": 3194
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1790.774658203125,
      "epoch": 0.9543723396310955,
      "grad_norm": 7.965880870819092,
      "kl": 3.8046875,
      "learning_rate": 3.1352858802547734e-09,
      "loss": 0.2394,
      "reward": 0.5613839477300644,
      "reward_std": 0.1668187379837036,
      "rewards/accuracy_reward": 0.1339285783469677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4274553805589676,
      "step": 3195
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1829.0246276855469,
      "epoch": 0.954671047718617,
      "grad_norm": 9.920350074768066,
      "kl": 4.8125,
      "learning_rate": 3.0942528537260816e-09,
      "loss": 0.2439,
      "reward": 0.4960937649011612,
      "reward_std": 0.15205425955355167,
      "rewards/accuracy_reward": 0.0892857201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4068080559372902,
      "step": 3196
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1793.6808776855469,
      "epoch": 0.9549697558061384,
      "grad_norm": 12.731158256530762,
      "kl": 3.83984375,
      "learning_rate": 3.053488436450896e-09,
      "loss": 0.2385,
      "reward": 0.5256696566939354,
      "reward_std": 0.16851279698312283,
      "rewards/accuracy_reward": 0.113839291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.411830373108387,
      "step": 3197
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1784.9554138183594,
      "epoch": 0.9552684638936599,
      "grad_norm": 7.935122489929199,
      "kl": 3.79296875,
      "learning_rate": 3.0129926727769926e-09,
      "loss": 0.2218,
      "reward": 0.5106026977300644,
      "reward_std": 0.158831886947155,
      "rewards/accuracy_reward": 0.09598214644938707,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205559372902,
      "step": 3198
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1692.1630249023438,
      "epoch": 0.9555671719811814,
      "grad_norm": 3.705775737762451,
      "kl": 3.6328125,
      "learning_rate": 2.972765606759686e-09,
      "loss": 0.2013,
      "reward": 0.5781250223517418,
      "reward_std": 0.2159331515431404,
      "rewards/accuracy_reward": 0.1406250074505806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000223517418,
      "step": 3199
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1828.0156860351562,
      "epoch": 0.9558658800687029,
      "grad_norm": 25.695594787597656,
      "kl": 5.859375,
      "learning_rate": 2.932807282162136e-09,
      "loss": 0.3036,
      "reward": 0.5184151977300644,
      "reward_std": 0.14088300615549088,
      "rewards/accuracy_reward": 0.1160714365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4023437649011612,
      "step": 3200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1874.7568054199219,
      "epoch": 0.9561645881562243,
      "grad_norm": 5.909251689910889,
      "kl": 4.36328125,
      "learning_rate": 2.8931177424550703e-09,
      "loss": 0.2182,
      "reward": 0.459263414144516,
      "reward_std": 0.13483469188213348,
      "rewards/accuracy_reward": 0.04687500116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4123884066939354,
      "step": 3201
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1782.8795471191406,
      "epoch": 0.9564632962437458,
      "grad_norm": 8.672348976135254,
      "kl": 4.6484375,
      "learning_rate": 2.8536970308168375e-09,
      "loss": 0.2647,
      "reward": 0.4977678805589676,
      "reward_std": 0.1677652969956398,
      "rewards/accuracy_reward": 0.08258928917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4151785895228386,
      "step": 3202
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1720.0134887695312,
      "epoch": 0.9567620043312672,
      "grad_norm": 4.105799198150635,
      "kl": 3.91015625,
      "learning_rate": 2.814545190133272e-09,
      "loss": 0.2326,
      "reward": 0.5234375223517418,
      "reward_std": 0.16041920334100723,
      "rewards/accuracy_reward": 0.10267857275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4207589477300644,
      "step": 3203
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1759.7567749023438,
      "epoch": 0.9570607124187888,
      "grad_norm": 5.669576168060303,
      "kl": 3.69140625,
      "learning_rate": 2.7756622629978023e-09,
      "loss": 0.21,
      "reward": 0.471540205180645,
      "reward_std": 0.18292231857776642,
      "rewards/accuracy_reward": 0.04687500232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4246651977300644,
      "step": 3204
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1761.5603332519531,
      "epoch": 0.9573594205063102,
      "grad_norm": 8.462360382080078,
      "kl": 3.076171875,
      "learning_rate": 2.7370482917111747e-09,
      "loss": 0.1753,
      "reward": 0.533482164144516,
      "reward_std": 0.17462538182735443,
      "rewards/accuracy_reward": 0.09375000605359674,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.439732164144516,
      "step": 3205
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1683.3437805175781,
      "epoch": 0.9576581285938317,
      "grad_norm": 12.1578369140625,
      "kl": 3.265625,
      "learning_rate": 2.6987033182816478e-09,
      "loss": 0.2162,
      "reward": 0.4642857313156128,
      "reward_std": 0.15591027028858662,
      "rewards/accuracy_reward": 0.03125000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.433035746216774,
      "step": 3206
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1812.6607971191406,
      "epoch": 0.9579568366813531,
      "grad_norm": 6.98715353012085,
      "kl": 4.796875,
      "learning_rate": 2.6606273844247405e-09,
      "loss": 0.2678,
      "reward": 0.435825914144516,
      "reward_std": 0.16061965562403202,
      "rewards/accuracy_reward": 0.024553572991862893,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4112723395228386,
      "step": 3207
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1717.1898193359375,
      "epoch": 0.9582555447688746,
      "grad_norm": 10.775981903076172,
      "kl": 3.16796875,
      "learning_rate": 2.6228205315634023e-09,
      "loss": 0.2059,
      "reward": 0.5407366380095482,
      "reward_std": 0.17142374627292156,
      "rewards/accuracy_reward": 0.1004464328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.440290205180645,
      "step": 3208
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1776.7388916015625,
      "epoch": 0.9585542528563961,
      "grad_norm": 3.871722459793091,
      "kl": 3.81640625,
      "learning_rate": 2.5852828008277593e-09,
      "loss": 0.2029,
      "reward": 0.5507812723517418,
      "reward_std": 0.14525078795850277,
      "rewards/accuracy_reward": 0.12500000605359674,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812723517418,
      "step": 3209
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1753.7166137695312,
      "epoch": 0.9588529609439176,
      "grad_norm": 16.75496482849121,
      "kl": 5.109375,
      "learning_rate": 2.5480142330551736e-09,
      "loss": 0.2791,
      "reward": 0.5340401977300644,
      "reward_std": 0.14148659072816372,
      "rewards/accuracy_reward": 0.12500000465661287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4090401977300644,
      "step": 3210
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1855.8304443359375,
      "epoch": 0.959151669031439,
      "grad_norm": 6.700922966003418,
      "kl": 4.26171875,
      "learning_rate": 2.5110148687902123e-09,
      "loss": 0.2248,
      "reward": 0.4609375149011612,
      "reward_std": 0.14860033616423607,
      "rewards/accuracy_reward": 0.051339289639145136,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4095982313156128,
      "step": 3211
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1764.5692443847656,
      "epoch": 0.9594503771189605,
      "grad_norm": 17.808759689331055,
      "kl": 3.984375,
      "learning_rate": 2.474284748284539e-09,
      "loss": 0.2201,
      "reward": 0.5011160895228386,
      "reward_std": 0.19133322685956955,
      "rewards/accuracy_reward": 0.07589286006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232313156128,
      "step": 3212
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1834.571533203125,
      "epoch": 0.9597490852064819,
      "grad_norm": 5.078474998474121,
      "kl": 3.60546875,
      "learning_rate": 2.437823911496911e-09,
      "loss": 0.2053,
      "reward": 0.4760044813156128,
      "reward_std": 0.16334300860762596,
      "rewards/accuracy_reward": 0.06696429010480642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4090401977300644,
      "step": 3213
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1768.2813110351562,
      "epoch": 0.9600477932940035,
      "grad_norm": 3.833346128463745,
      "kl": 4.48046875,
      "learning_rate": 2.401632398093184e-09,
      "loss": 0.2626,
      "reward": 0.5072544887661934,
      "reward_std": 0.15252332389354706,
      "rewards/accuracy_reward": 0.0915178619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4157366305589676,
      "step": 3214
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1732.7634887695312,
      "epoch": 0.9603465013815249,
      "grad_norm": 18.387584686279297,
      "kl": 4.87890625,
      "learning_rate": 2.3657102474461677e-09,
      "loss": 0.2854,
      "reward": 0.5195312798023224,
      "reward_std": 0.1605258844792843,
      "rewards/accuracy_reward": 0.10491071757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205484867096,
      "step": 3215
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1786.2857971191406,
      "epoch": 0.9606452094690464,
      "grad_norm": 13.01712703704834,
      "kl": 3.76953125,
      "learning_rate": 2.330057498635629e-09,
      "loss": 0.2075,
      "reward": 0.4815848395228386,
      "reward_std": 0.12791107781231403,
      "rewards/accuracy_reward": 0.05357143026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428013414144516,
      "step": 3216
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1707.8706359863281,
      "epoch": 0.9609439175565678,
      "grad_norm": 3.9677698612213135,
      "kl": 4.42578125,
      "learning_rate": 2.2946741904482646e-09,
      "loss": 0.26,
      "reward": 0.4681919813156128,
      "reward_std": 0.13768784515559673,
      "rewards/accuracy_reward": 0.04910714505240321,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4190848395228386,
      "step": 3217
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1757.8125610351562,
      "epoch": 0.9612426256440894,
      "grad_norm": 12.316123962402344,
      "kl": 4.49609375,
      "learning_rate": 2.2595603613776713e-09,
      "loss": 0.2568,
      "reward": 0.4888393133878708,
      "reward_std": 0.16414214670658112,
      "rewards/accuracy_reward": 0.06919643143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.419642873108387,
      "step": 3218
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1777.01123046875,
      "epoch": 0.9615413337316108,
      "grad_norm": 6.586704254150391,
      "kl": 3.40234375,
      "learning_rate": 2.224716049624209e-09,
      "loss": 0.1906,
      "reward": 0.4514509215950966,
      "reward_std": 0.14094720594584942,
      "rewards/accuracy_reward": 0.03348214365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4179687723517418,
      "step": 3219
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1749.7009887695312,
      "epoch": 0.9618400418191323,
      "grad_norm": 13.039702415466309,
      "kl": 3.16015625,
      "learning_rate": 2.1901412930951103e-09,
      "loss": 0.204,
      "reward": 0.486607164144516,
      "reward_std": 0.11408872716128826,
      "rewards/accuracy_reward": 0.0468750037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.439732164144516,
      "step": 3220
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1779.9688110351562,
      "epoch": 0.9621387499066537,
      "grad_norm": 13.141895294189453,
      "kl": 4.2890625,
      "learning_rate": 2.155836129404287e-09,
      "loss": 0.2534,
      "reward": 0.4972098395228386,
      "reward_std": 0.1626892313361168,
      "rewards/accuracy_reward": 0.07142857392318547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812649011612,
      "step": 3221
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1695.446533203125,
      "epoch": 0.9624374579941752,
      "grad_norm": 10.435662269592285,
      "kl": 3.7890625,
      "learning_rate": 2.121800595872414e-09,
      "loss": 0.2309,
      "reward": 0.5580357313156128,
      "reward_std": 0.13018103316426277,
      "rewards/accuracy_reward": 0.1250000074505806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4330357313156128,
      "step": 3222
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1675.6697387695312,
      "epoch": 0.9627361660816967,
      "grad_norm": 12.00804615020752,
      "kl": 3.25390625,
      "learning_rate": 2.088034729526844e-09,
      "loss": 0.2002,
      "reward": 0.4525669738650322,
      "reward_std": 0.11714149080216885,
      "rewards/accuracy_reward": 0.020089287078008056,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776902794838,
      "step": 3223
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1662.0648193359375,
      "epoch": 0.9630348741692182,
      "grad_norm": 3.8778293132781982,
      "kl": 3.73046875,
      "learning_rate": 2.054538567101499e-09,
      "loss": 0.2364,
      "reward": 0.580357164144516,
      "reward_std": 0.18404560908675194,
      "rewards/accuracy_reward": 0.15178572479635477,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 3224
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1820.1094360351562,
      "epoch": 0.9633335822567396,
      "grad_norm": 6.830080032348633,
      "kl": 3.89453125,
      "learning_rate": 2.0213121450368963e-09,
      "loss": 0.2165,
      "reward": 0.5792410969734192,
      "reward_std": 0.19683990254998207,
      "rewards/accuracy_reward": 0.1584821455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4207589477300644,
      "step": 3225
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1782.0179138183594,
      "epoch": 0.9636322903442611,
      "grad_norm": 4.4903693199157715,
      "kl": 4.640625,
      "learning_rate": 1.9883554994802044e-09,
      "loss": 0.2569,
      "reward": 0.482700914144516,
      "reward_std": 0.15014082193374634,
      "rewards/accuracy_reward": 0.06919643143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4135044813156128,
      "step": 3226
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1838.9375915527344,
      "epoch": 0.9639309984317825,
      "grad_norm": 10.182792663574219,
      "kl": 4.56640625,
      "learning_rate": 1.9556686662849377e-09,
      "loss": 0.2401,
      "reward": 0.4732143133878708,
      "reward_std": 0.12268347665667534,
      "rewards/accuracy_reward": 0.07589286053553224,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.397321455180645,
      "step": 3227
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1753.3639221191406,
      "epoch": 0.9642297065193041,
      "grad_norm": 14.042912483215332,
      "kl": 3.1015625,
      "learning_rate": 1.9232516810112064e-09,
      "loss": 0.1831,
      "reward": 0.5675223469734192,
      "reward_std": 0.1663905456662178,
      "rewards/accuracy_reward": 0.13392857555299997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937723517418,
      "step": 3228
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1748.8616943359375,
      "epoch": 0.9645284146068255,
      "grad_norm": 8.678559303283691,
      "kl": 3.93359375,
      "learning_rate": 1.891104578925523e-09,
      "loss": 0.2227,
      "reward": 0.545200914144516,
      "reward_std": 0.19857880100607872,
      "rewards/accuracy_reward": 0.11830357741564512,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4268973469734192,
      "step": 3229
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1701.5693054199219,
      "epoch": 0.9648271226943469,
      "grad_norm": 13.495780944824219,
      "kl": 3.55859375,
      "learning_rate": 1.859227395000773e-09,
      "loss": 0.2209,
      "reward": 0.5145089626312256,
      "reward_std": 0.1673678308725357,
      "rewards/accuracy_reward": 0.08035714598372579,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341518059372902,
      "step": 3230
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1743.5157165527344,
      "epoch": 0.9651258307818684,
      "grad_norm": 13.046490669250488,
      "kl": 5.2890625,
      "learning_rate": 1.827620163916216e-09,
      "loss": 0.307,
      "reward": 0.495535746216774,
      "reward_std": 0.12875139899551868,
      "rewards/accuracy_reward": 0.08482143376022577,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4107143133878708,
      "step": 3231
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1742.5380249023438,
      "epoch": 0.9654245388693898,
      "grad_norm": 9.2525634765625,
      "kl": 2.9609375,
      "learning_rate": 1.7962829200574015e-09,
      "loss": 0.1717,
      "reward": 0.4620535895228386,
      "reward_std": 0.1367003209888935,
      "rewards/accuracy_reward": 0.02232142980210483,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.439732164144516,
      "step": 3232
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1721.3460693359375,
      "epoch": 0.9657232469569114,
      "grad_norm": 11.904914855957031,
      "kl": 4.828125,
      "learning_rate": 1.765215697516198e-09,
      "loss": 0.2804,
      "reward": 0.4849330559372902,
      "reward_std": 0.1466090828180313,
      "rewards/accuracy_reward": 0.07142857648432255,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4135044813156128,
      "step": 3233
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1771.024658203125,
      "epoch": 0.9660219550444328,
      "grad_norm": 5.6642961502075195,
      "kl": 3.875,
      "learning_rate": 1.734418530090681e-09,
      "loss": 0.2206,
      "reward": 0.5022321715950966,
      "reward_std": 0.17550724558532238,
      "rewards/accuracy_reward": 0.08482143376022577,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4174107313156128,
      "step": 3234
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1757.2009582519531,
      "epoch": 0.9663206631319543,
      "grad_norm": 6.1873626708984375,
      "kl": 3.59765625,
      "learning_rate": 1.703891451285161e-09,
      "loss": 0.2164,
      "reward": 0.5563616454601288,
      "reward_std": 0.1640193685889244,
      "rewards/accuracy_reward": 0.1316964386496693,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4246651977300644,
      "step": 3235
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1734.9107666015625,
      "epoch": 0.9666193712194757,
      "grad_norm": 11.023072242736816,
      "kl": 3.54296875,
      "learning_rate": 1.6736344943101277e-09,
      "loss": 0.2383,
      "reward": 0.510602705180645,
      "reward_std": 0.12545725144445896,
      "rewards/accuracy_reward": 0.0825892873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4280134066939354,
      "step": 3236
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1757.58935546875,
      "epoch": 0.9669180793069972,
      "grad_norm": 8.52560806274414,
      "kl": 3.9765625,
      "learning_rate": 1.6436476920821396e-09,
      "loss": 0.2214,
      "reward": 0.5535714626312256,
      "reward_std": 0.18101037666201591,
      "rewards/accuracy_reward": 0.13169643376022577,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 3237
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1788.4576416015625,
      "epoch": 0.9672167873945187,
      "grad_norm": 5.21449613571167,
      "kl": 3.515625,
      "learning_rate": 1.6139310772239346e-09,
      "loss": 0.1935,
      "reward": 0.4827009215950966,
      "reward_std": 0.1373145617544651,
      "rewards/accuracy_reward": 0.04910714412108064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937649011612,
      "step": 3238
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1839.9152526855469,
      "epoch": 0.9675154954820402,
      "grad_norm": 3.3413240909576416,
      "kl": 4.71875,
      "learning_rate": 1.5844846820642632e-09,
      "loss": 0.2622,
      "reward": 0.510044664144516,
      "reward_std": 0.20998390763998032,
      "rewards/accuracy_reward": 0.09821428777649999,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4118303805589676,
      "step": 3239
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1726.3348999023438,
      "epoch": 0.9678142035695616,
      "grad_norm": 11.348340034484863,
      "kl": 3.1796875,
      "learning_rate": 1.555308538637945e-09,
      "loss": 0.1876,
      "reward": 0.4391741305589676,
      "reward_std": 0.11797518283128738,
      "rewards/accuracy_reward": 0.008928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455633878708,
      "step": 3240
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1793.8728332519531,
      "epoch": 0.9681129116570831,
      "grad_norm": 3.342447519302368,
      "kl": 3.07421875,
      "learning_rate": 1.526402678685701e-09,
      "loss": 0.1839,
      "reward": 0.4575893059372902,
      "reward_std": 0.1572511661797762,
      "rewards/accuracy_reward": 0.026785715483129025,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035969734192,
      "step": 3241
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1768.4085388183594,
      "epoch": 0.9684116197446045,
      "grad_norm": 16.113948822021484,
      "kl": 4.01953125,
      "learning_rate": 1.4977671336543484e-09,
      "loss": 0.2376,
      "reward": 0.5217634215950966,
      "reward_std": 0.1494471412152052,
      "rewards/accuracy_reward": 0.10044643399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4213169813156128,
      "step": 3242
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1802.3281860351562,
      "epoch": 0.9687103278321261,
      "grad_norm": 7.708029747009277,
      "kl": 3.17578125,
      "learning_rate": 1.4694019346964958e-09,
      "loss": 0.1761,
      "reward": 0.5552455633878708,
      "reward_std": 0.15770412608981133,
      "rewards/accuracy_reward": 0.12723215157166123,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4280134066939354,
      "step": 3243
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1787.6563415527344,
      "epoch": 0.9690090359196475,
      "grad_norm": 7.213177680969238,
      "kl": 4.48046875,
      "learning_rate": 1.4413071126707367e-09,
      "loss": 0.2409,
      "reward": 0.5608259215950966,
      "reward_std": 0.1370740346610546,
      "rewards/accuracy_reward": 0.13616071757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4246651977300644,
      "step": 3244
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1834.4911499023438,
      "epoch": 0.969307744007169,
      "grad_norm": 6.7094831466674805,
      "kl": 4.53125,
      "learning_rate": 1.413482698141455e-09,
      "loss": 0.2549,
      "reward": 0.4849330559372902,
      "reward_std": 0.16904442012310028,
      "rewards/accuracy_reward": 0.07812500186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4068080559372902,
      "step": 3245
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1709.4241943359375,
      "epoch": 0.9696064520946904,
      "grad_norm": 11.190864562988281,
      "kl": 4.2890625,
      "learning_rate": 1.3859287213789095e-09,
      "loss": 0.265,
      "reward": 0.5083705633878708,
      "reward_std": 0.14753515273332596,
      "rewards/accuracy_reward": 0.07812500186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4302455633878708,
      "step": 3246
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1774.87060546875,
      "epoch": 0.969905160182212,
      "grad_norm": 7.2082200050354,
      "kl": 4.20703125,
      "learning_rate": 1.3586452123590941e-09,
      "loss": 0.2546,
      "reward": 0.5167410895228386,
      "reward_std": 0.148310799151659,
      "rewards/accuracy_reward": 0.09375000558793545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910895228386,
      "step": 3247
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1837.290283203125,
      "epoch": 0.9702038682697334,
      "grad_norm": 5.3650288581848145,
      "kl": 4.35546875,
      "learning_rate": 1.3316322007638491e-09,
      "loss": 0.2479,
      "reward": 0.4547991305589676,
      "reward_std": 0.12177349627017975,
      "rewards/accuracy_reward": 0.04017857206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4146205559372902,
      "step": 3248
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1763.3616943359375,
      "epoch": 0.9705025763572549,
      "grad_norm": 9.203317642211914,
      "kl": 3.91015625,
      "learning_rate": 1.3048897159806393e-09,
      "loss": 0.22,
      "reward": 0.4670759066939354,
      "reward_std": 0.14849507808685303,
      "rewards/accuracy_reward": 0.03794643050059676,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294813156128,
      "step": 3249
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1804.180908203125,
      "epoch": 0.9708012844447763,
      "grad_norm": 8.387083053588867,
      "kl": 3.52734375,
      "learning_rate": 1.2784177871026648e-09,
      "loss": 0.1944,
      "reward": 0.4486607387661934,
      "reward_std": 0.1336724292486906,
      "rewards/accuracy_reward": 0.026785715715959668,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 3250
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1791.4822387695312,
      "epoch": 0.9710999925322978,
      "grad_norm": 14.715662002563477,
      "kl": 2.8046875,
      "learning_rate": 1.2522164429288051e-09,
      "loss": 0.1831,
      "reward": 0.5546875223517418,
      "reward_std": 0.1625835169106722,
      "rewards/accuracy_reward": 0.1205357201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341518059372902,
      "step": 3251
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1733.4777526855469,
      "epoch": 0.9713987006198193,
      "grad_norm": 7.860199451446533,
      "kl": 4.01171875,
      "learning_rate": 1.226285711963565e-09,
      "loss": 0.2234,
      "reward": 0.5401785969734192,
      "reward_std": 0.11176765523850918,
      "rewards/accuracy_reward": 0.1160714328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.424107164144516,
      "step": 3252
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1676.5447387695312,
      "epoch": 0.9716974087073408,
      "grad_norm": 12.388923645019531,
      "kl": 2.9140625,
      "learning_rate": 1.2006256224170176e-09,
      "loss": 0.212,
      "reward": 0.558035746216774,
      "reward_std": 0.16197515465319157,
      "rewards/accuracy_reward": 0.1160714365541935,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4419643059372902,
      "step": 3253
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1658.4554443359375,
      "epoch": 0.9719961167948622,
      "grad_norm": 17.441303253173828,
      "kl": 2.482421875,
      "learning_rate": 1.17523620220486e-09,
      "loss": 0.1706,
      "reward": 0.5887276977300644,
      "reward_std": 0.19740157201886177,
      "rewards/accuracy_reward": 0.1473214328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4414062723517418,
      "step": 3254
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1738.1853332519531,
      "epoch": 0.9722948248823837,
      "grad_norm": 7.259647369384766,
      "kl": 3.14453125,
      "learning_rate": 1.1501174789482205e-09,
      "loss": 0.1827,
      "reward": 0.4542410969734192,
      "reward_std": 0.1370771937072277,
      "rewards/accuracy_reward": 0.02455357275903225,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875149011612,
      "step": 3255
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1804.2188415527344,
      "epoch": 0.9725935329699051,
      "grad_norm": 4.185131072998047,
      "kl": 4.08984375,
      "learning_rate": 1.1252694799738782e-09,
      "loss": 0.2213,
      "reward": 0.4815848544239998,
      "reward_std": 0.1548773217946291,
      "rewards/accuracy_reward": 0.0647321455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4168526977300644,
      "step": 3256
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1722.8170471191406,
      "epoch": 0.9728922410574267,
      "grad_norm": 10.445076942443848,
      "kl": 3.421875,
      "learning_rate": 1.1006922323140155e-09,
      "loss": 0.2177,
      "reward": 0.5133928880095482,
      "reward_std": 0.15792641043663025,
      "rewards/accuracy_reward": 0.07589286146685481,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000223517418,
      "step": 3257
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1811.1853637695312,
      "epoch": 0.9731909491449481,
      "grad_norm": 3.947209358215332,
      "kl": 3.19140625,
      "learning_rate": 1.0763857627062167e-09,
      "loss": 0.1849,
      "reward": 0.4827009215950966,
      "reward_std": 0.163326783105731,
      "rewards/accuracy_reward": 0.06250000232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4202009066939354,
      "step": 3258
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1758.7500610351562,
      "epoch": 0.9734896572324696,
      "grad_norm": 8.148619651794434,
      "kl": 4.84375,
      "learning_rate": 1.0523500975935794e-09,
      "loss": 0.2744,
      "reward": 0.4916294813156128,
      "reward_std": 0.16704277321696281,
      "rewards/accuracy_reward": 0.08035714738070965,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.411272332072258,
      "step": 3259
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1727.977783203125,
      "epoch": 0.973788365319991,
      "grad_norm": 8.718232154846191,
      "kl": 4.31640625,
      "learning_rate": 1.0285852631245485e-09,
      "loss": 0.2689,
      "reward": 0.5429687723517418,
      "reward_std": 0.1785835474729538,
      "rewards/accuracy_reward": 0.11383928847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294813156128,
      "step": 3260
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1756.68310546875,
      "epoch": 0.9740870734075125,
      "grad_norm": 18.217281341552734,
      "kl": 4.58203125,
      "learning_rate": 1.0050912851529425e-09,
      "loss": 0.2554,
      "reward": 0.5089285895228386,
      "reward_std": 0.16127764247357845,
      "rewards/accuracy_reward": 0.08928571757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.419642873108387,
      "step": 3261
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1702.5291137695312,
      "epoch": 0.974385781495034,
      "grad_norm": 11.334710121154785,
      "kl": 3.5234375,
      "learning_rate": 9.818681892378722e-10,
      "loss": 0.2235,
      "reward": 0.502790205180645,
      "reward_std": 0.18495706096291542,
      "rewards/accuracy_reward": 0.06696429010480642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4358259066939354,
      "step": 3262
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1719.0022888183594,
      "epoch": 0.9746844895825555,
      "grad_norm": 4.499971389770508,
      "kl": 2.546875,
      "learning_rate": 9.589160006438224e-10,
      "loss": 0.1475,
      "reward": 0.4737723469734192,
      "reward_std": 0.13281539268791676,
      "rewards/accuracy_reward": 0.029017859371379018,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4447544813156128,
      "step": 3263
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1704.1920166015625,
      "epoch": 0.9749831976700769,
      "grad_norm": 24.08409881591797,
      "kl": 2.90234375,
      "learning_rate": 9.362347443405138e-10,
      "loss": 0.1919,
      "reward": 0.5814732387661934,
      "reward_std": 0.17124444991350174,
      "rewards/accuracy_reward": 0.13839286006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4430803805589676,
      "step": 3264
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1644.2366943359375,
      "epoch": 0.9752819057575984,
      "grad_norm": 4.518803119659424,
      "kl": 4.01953125,
      "learning_rate": 9.138244450029309e-10,
      "loss": 0.2466,
      "reward": 0.5580357313156128,
      "reward_std": 0.1352438535541296,
      "rewards/accuracy_reward": 0.1227678619325161,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435267873108387,
      "step": 3265
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.8527526855469,
      "epoch": 0.9755806138451198,
      "grad_norm": 6.444426536560059,
      "kl": 3.298828125,
      "learning_rate": 8.916851270112935e-10,
      "loss": 0.1823,
      "reward": 0.491071455180645,
      "reward_std": 0.16471440717577934,
      "rewards/accuracy_reward": 0.06919643143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 3266
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1677.5960693359375,
      "epoch": 0.9758793219326414,
      "grad_norm": 14.600465774536133,
      "kl": 2.501953125,
      "learning_rate": 8.698168144509744e-10,
      "loss": 0.1546,
      "reward": 0.5295759215950966,
      "reward_std": 0.1610003411769867,
      "rewards/accuracy_reward": 0.0781250037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4514509215950966,
      "step": 3267
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1720.274658203125,
      "epoch": 0.9761780300201628,
      "grad_norm": 40.18429946899414,
      "kl": 3.697265625,
      "learning_rate": 8.482195311126095e-10,
      "loss": 0.2371,
      "reward": 0.4726562798023224,
      "reward_std": 0.14589036256074905,
      "rewards/accuracy_reward": 0.0357142873108387,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4369419887661934,
      "step": 3268
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1773.0379943847656,
      "epoch": 0.9764767381076843,
      "grad_norm": 6.269368648529053,
      "kl": 3.94140625,
      "learning_rate": 8.268933004918766e-10,
      "loss": 0.2166,
      "reward": 0.5066964626312256,
      "reward_std": 0.16532974503934383,
      "rewards/accuracy_reward": 0.08482143096625805,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750223517418,
      "step": 3269
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1740.6563110351562,
      "epoch": 0.9767754461952057,
      "grad_norm": 7.653167247772217,
      "kl": 5.02734375,
      "learning_rate": 8.058381457896612e-10,
      "loss": 0.31,
      "reward": 0.604910746216774,
      "reward_std": 0.15586625412106514,
      "rewards/accuracy_reward": 0.1919642947614193,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4129464477300644,
      "step": 3270
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1786.3683776855469,
      "epoch": 0.9770741542827273,
      "grad_norm": 9.851397514343262,
      "kl": 4.03125,
      "learning_rate": 7.85054089911863e-10,
      "loss": 0.2315,
      "reward": 0.5541294813156128,
      "reward_std": 0.1262793205678463,
      "rewards/accuracy_reward": 0.145089291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4090401977300644,
      "step": 3271
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1776.1429443359375,
      "epoch": 0.9773728623702487,
      "grad_norm": 10.898795127868652,
      "kl": 3.76953125,
      "learning_rate": 7.645411554695336e-10,
      "loss": 0.2023,
      "reward": 0.5290178805589676,
      "reward_std": 0.18615956231951714,
      "rewards/accuracy_reward": 0.1004464328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.428571455180645,
      "step": 3272
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1752.6719360351562,
      "epoch": 0.9776715704577701,
      "grad_norm": 3.232449531555176,
      "kl": 3.80078125,
      "learning_rate": 7.442993647787388e-10,
      "loss": 0.2239,
      "reward": 0.5440848395228386,
      "reward_std": 0.21797686256468296,
      "rewards/accuracy_reward": 0.12276786006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4213169813156128,
      "step": 3273
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1763.4866943359375,
      "epoch": 0.9779702785452916,
      "grad_norm": 24.500986099243164,
      "kl": 4.16015625,
      "learning_rate": 7.24328739860558e-10,
      "loss": 0.2141,
      "reward": 0.4860491305589676,
      "reward_std": 0.18895114213228226,
      "rewards/accuracy_reward": 0.06696428917348385,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4190848469734192,
      "step": 3274
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1713.9264221191406,
      "epoch": 0.978268986632813,
      "grad_norm": 7.536870002746582,
      "kl": 2.78125,
      "learning_rate": 7.046293024410288e-10,
      "loss": 0.1921,
      "reward": 0.5329241454601288,
      "reward_std": 0.18951557576656342,
      "rewards/accuracy_reward": 0.09821429289877415,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4347098395228386,
      "step": 3275
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1813.7880249023438,
      "epoch": 0.9785676947203346,
      "grad_norm": 16.642459869384766,
      "kl": 4.640625,
      "learning_rate": 6.852010739512859e-10,
      "loss": 0.2588,
      "reward": 0.4821428805589676,
      "reward_std": 0.14625650271773338,
      "rewards/accuracy_reward": 0.07142857578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4107143059372902,
      "step": 3276
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1771.930908203125,
      "epoch": 0.978866402807856,
      "grad_norm": 14.33617115020752,
      "kl": 5.1875,
      "learning_rate": 6.660440755272833e-10,
      "loss": 0.2658,
      "reward": 0.5329241380095482,
      "reward_std": 0.17029426246881485,
      "rewards/accuracy_reward": 0.1250000074505806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4079241305589676,
      "step": 3277
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1792.7188110351562,
      "epoch": 0.9791651108953775,
      "grad_norm": 8.5479154586792,
      "kl": 4.796875,
      "learning_rate": 6.471583280099613e-10,
      "loss": 0.2599,
      "reward": 0.5055803880095482,
      "reward_std": 0.16871783696115017,
      "rewards/accuracy_reward": 0.0892857201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.416294664144516,
      "step": 3278
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1733.5558776855469,
      "epoch": 0.9794638189828989,
      "grad_norm": 7.3055644035339355,
      "kl": 3.4765625,
      "learning_rate": 6.285438519451902e-10,
      "loss": 0.2024,
      "reward": 0.4966518133878708,
      "reward_std": 0.15309485793113708,
      "rewards/accuracy_reward": 0.06919643143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4274553805589676,
      "step": 3279
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1765.7701721191406,
      "epoch": 0.9797625270704204,
      "grad_norm": 7.271821022033691,
      "kl": 4.45703125,
      "learning_rate": 6.102006675836602e-10,
      "loss": 0.2843,
      "reward": 0.5000000298023224,
      "reward_std": 0.13319792784750462,
      "rewards/accuracy_reward": 0.08258928847499192,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4174107313156128,
      "step": 3280
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1744.1920776367188,
      "epoch": 0.9800612351579419,
      "grad_norm": 8.650383949279785,
      "kl": 3.265625,
      "learning_rate": 5.921287948809917e-10,
      "loss": 0.194,
      "reward": 0.4536830633878708,
      "reward_std": 0.14453030563890934,
      "rewards/accuracy_reward": 0.02008928661234677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937723517418,
      "step": 3281
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1774.321533203125,
      "epoch": 0.9803599432454634,
      "grad_norm": 13.32384204864502,
      "kl": 3.7890625,
      "learning_rate": 5.743282534975968e-10,
      "loss": 0.22,
      "reward": 0.5106026902794838,
      "reward_std": 0.17397739738225937,
      "rewards/accuracy_reward": 0.0937500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4168526977300644,
      "step": 3282
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1688.8438110351562,
      "epoch": 0.9806586513329848,
      "grad_norm": 5.329275608062744,
      "kl": 3.921875,
      "learning_rate": 5.567990627987073e-10,
      "loss": 0.2316,
      "reward": 0.549107164144516,
      "reward_std": 0.1544457357376814,
      "rewards/accuracy_reward": 0.12276786309666932,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4263393059372902,
      "step": 3283
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1686.9777526855469,
      "epoch": 0.9809573594205063,
      "grad_norm": 7.508930683135986,
      "kl": 3.4296875,
      "learning_rate": 5.395412418544021e-10,
      "loss": 0.2039,
      "reward": 0.4559151977300644,
      "reward_std": 0.13626448065042496,
      "rewards/accuracy_reward": 0.022321429569274187,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937723517418,
      "step": 3284
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1733.7277221679688,
      "epoch": 0.9812560675080277,
      "grad_norm": 4.931296348571777,
      "kl": 3.90625,
      "learning_rate": 5.225548094394405e-10,
      "loss": 0.2465,
      "reward": 0.5122768059372902,
      "reward_std": 0.13777787052094936,
      "rewards/accuracy_reward": 0.082589291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875298023224,
      "step": 3285
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1775.9554138183594,
      "epoch": 0.9815547755955493,
      "grad_norm": 58.704505920410156,
      "kl": 5.3203125,
      "learning_rate": 5.058397840334572e-10,
      "loss": 0.3105,
      "reward": 0.4386160969734192,
      "reward_std": 0.16509902477264404,
      "rewards/accuracy_reward": 0.03125000116415322,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4073660969734192,
      "step": 3286
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1801.3147888183594,
      "epoch": 0.9818534836830707,
      "grad_norm": 13.329440116882324,
      "kl": 3.25,
      "learning_rate": 4.893961838207117e-10,
      "loss": 0.1886,
      "reward": 0.4693080633878708,
      "reward_std": 0.1451030671596527,
      "rewards/accuracy_reward": 0.040178573690354824,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294887661934,
      "step": 3287
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1753.1340026855469,
      "epoch": 0.9821521917705922,
      "grad_norm": 13.64964771270752,
      "kl": 4.89453125,
      "learning_rate": 4.732240266902554e-10,
      "loss": 0.2835,
      "reward": 0.4988839402794838,
      "reward_std": 0.16150518134236336,
      "rewards/accuracy_reward": 0.08035714668221772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185267984867096,
      "step": 3288
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1773.3906860351562,
      "epoch": 0.9824508998581136,
      "grad_norm": 7.367542266845703,
      "kl": 4.0234375,
      "learning_rate": 4.573233302358204e-10,
      "loss": 0.229,
      "reward": 0.4771205559372902,
      "reward_std": 0.11112837120890617,
      "rewards/accuracy_reward": 0.04464285937137902,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4324776977300644,
      "step": 3289
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1728.6005249023438,
      "epoch": 0.9827496079456352,
      "grad_norm": 6.219101428985596,
      "kl": 4.890625,
      "learning_rate": 4.4169411175576375e-10,
      "loss": 0.3024,
      "reward": 0.5050223469734192,
      "reward_std": 0.18626913614571095,
      "rewards/accuracy_reward": 0.0870535746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4179687649011612,
      "step": 3290
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1831.977783203125,
      "epoch": 0.9830483160331566,
      "grad_norm": 8.061516761779785,
      "kl": 3.2265625,
      "learning_rate": 4.2633638825317895e-10,
      "loss": 0.1925,
      "reward": 0.5301339402794838,
      "reward_std": 0.14275600761175156,
      "rewards/accuracy_reward": 0.10491071944124997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232387661934,
      "step": 3291
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1737.0670471191406,
      "epoch": 0.9833470241206781,
      "grad_norm": 8.11727523803711,
      "kl": 4.359375,
      "learning_rate": 4.1125017643578454e-10,
      "loss": 0.2643,
      "reward": 0.5189732238650322,
      "reward_std": 0.1531633771955967,
      "rewards/accuracy_reward": 0.10044643143191934,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4185268059372902,
      "step": 3292
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1755.993408203125,
      "epoch": 0.9836457322081995,
      "grad_norm": 9.283466339111328,
      "kl": 5.078125,
      "learning_rate": 3.9643549271584085e-10,
      "loss": 0.2901,
      "reward": 0.5279018133878708,
      "reward_std": 0.18255575746297836,
      "rewards/accuracy_reward": 0.11160714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.416294664144516,
      "step": 3293
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1793.6250915527344,
      "epoch": 0.983944440295721,
      "grad_norm": 12.233380317687988,
      "kl": 5.1484375,
      "learning_rate": 3.818923532103169e-10,
      "loss": 0.2952,
      "reward": 0.4570312723517418,
      "reward_std": 0.16420544311404228,
      "rewards/accuracy_reward": 0.044642860535532236,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4123884215950966,
      "step": 3294
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1703.9107666015625,
      "epoch": 0.9842431483832424,
      "grad_norm": 15.064461708068848,
      "kl": 4.37890625,
      "learning_rate": 3.6762077374069554e-10,
      "loss": 0.2533,
      "reward": 0.4771205633878708,
      "reward_std": 0.1393560878932476,
      "rewards/accuracy_reward": 0.05580357322469354,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4213169813156128,
      "step": 3295
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1794.0045166015625,
      "epoch": 0.984541856470764,
      "grad_norm": 9.326604843139648,
      "kl": 3.88671875,
      "learning_rate": 3.5362076983308507e-10,
      "loss": 0.207,
      "reward": 0.4776785895228386,
      "reward_std": 0.12650376372039318,
      "rewards/accuracy_reward": 0.05580357392318547,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750149011612,
      "step": 3296
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1744.7701721191406,
      "epoch": 0.9848405645582854,
      "grad_norm": 6.801052570343018,
      "kl": 3.234375,
      "learning_rate": 3.3989235671808003e-10,
      "loss": 0.1779,
      "reward": 0.553571455180645,
      "reward_std": 0.151376748457551,
      "rewards/accuracy_reward": 0.12276786169968545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035895228386,
      "step": 3297
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1767.1875915527344,
      "epoch": 0.9851392726458069,
      "grad_norm": 19.659704208374023,
      "kl": 4.140625,
      "learning_rate": 3.264355493309001e-10,
      "loss": 0.2255,
      "reward": 0.5039062798023224,
      "reward_std": 0.14806809090077877,
      "rewards/accuracy_reward": 0.0803571455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.423549123108387,
      "step": 3298
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1774.8259582519531,
      "epoch": 0.9854379807333283,
      "grad_norm": 4.726123332977295,
      "kl": 3.734375,
      "learning_rate": 3.132503623111682e-10,
      "loss": 0.2153,
      "reward": 0.4693080559372902,
      "reward_std": 0.1282098963856697,
      "rewards/accuracy_reward": 0.04910714481957257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.420200914144516,
      "step": 3299
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1733.2947387695312,
      "epoch": 0.9857366888208499,
      "grad_norm": 7.275485038757324,
      "kl": 3.72265625,
      "learning_rate": 3.0033681000313225e-10,
      "loss": 0.2229,
      "reward": 0.487165205180645,
      "reward_std": 0.1699267439544201,
      "rewards/accuracy_reward": 0.053571431897580624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4335937798023224,
      "step": 3300
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1714.2277526855469,
      "epoch": 0.9860353969083713,
      "grad_norm": 8.473512649536133,
      "kl": 3.55859375,
      "learning_rate": 2.8769490645547123e-10,
      "loss": 0.2159,
      "reward": 0.5770089626312256,
      "reward_std": 0.14893153123557568,
      "rewards/accuracy_reward": 0.1428571492433548,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4341518059372902,
      "step": 3301
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1810.8192749023438,
      "epoch": 0.9863341049958928,
      "grad_norm": 7.660743713378906,
      "kl": 4.27734375,
      "learning_rate": 2.753246654212671e-10,
      "loss": 0.238,
      "reward": 0.4481026902794838,
      "reward_std": 0.16958937607705593,
      "rewards/accuracy_reward": 0.026785716181620955,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4213169813156128,
      "step": 3302
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1778.71435546875,
      "epoch": 0.9866328130834142,
      "grad_norm": 16.448728561401367,
      "kl": 6.02734375,
      "learning_rate": 2.6322610035822703e-10,
      "loss": 0.3524,
      "reward": 0.4492187723517418,
      "reward_std": 0.13968057744204998,
      "rewards/accuracy_reward": 0.05357143026776612,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.395647332072258,
      "step": 3303
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1746.3929138183594,
      "epoch": 0.9869315211709357,
      "grad_norm": 13.308065414428711,
      "kl": 2.57421875,
      "learning_rate": 2.513992244283225e-10,
      "loss": 0.1552,
      "reward": 0.550223246216774,
      "reward_std": 0.1146874176338315,
      "rewards/accuracy_reward": 0.10491071827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125149011612,
      "step": 3304
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1791.4577026367188,
      "epoch": 0.9872302292584572,
      "grad_norm": 8.713001251220703,
      "kl": 4.21484375,
      "learning_rate": 2.3984405049806697e-10,
      "loss": 0.2302,
      "reward": 0.5016741156578064,
      "reward_std": 0.1467285230755806,
      "rewards/accuracy_reward": 0.08258929196745157,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4190848469734192,
      "step": 3305
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1801.8348999023438,
      "epoch": 0.9875289373459787,
      "grad_norm": 11.928108215332031,
      "kl": 3.708984375,
      "learning_rate": 2.28560591138377e-10,
      "loss": 0.2027,
      "reward": 0.4765625149011612,
      "reward_std": 0.13067300617694855,
      "rewards/accuracy_reward": 0.051339288242161274,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232313156128,
      "step": 3306
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1772.0982971191406,
      "epoch": 0.9878276454335001,
      "grad_norm": 4.269120693206787,
      "kl": 4.5546875,
      "learning_rate": 2.1754885862451667e-10,
      "loss": 0.2662,
      "reward": 0.4743303880095482,
      "reward_std": 0.14944340288639069,
      "rewards/accuracy_reward": 0.060267859837040305,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625223517418,
      "step": 3307
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1707.4554138183594,
      "epoch": 0.9881263535210216,
      "grad_norm": 10.0681734085083,
      "kl": 3.6796875,
      "learning_rate": 2.0680886493620875e-10,
      "loss": 0.2325,
      "reward": 0.5295759290456772,
      "reward_std": 0.16445429995656013,
      "rewards/accuracy_reward": 0.098214291036129,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.431361623108387,
      "step": 3308
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1698.4219665527344,
      "epoch": 0.988425061608543,
      "grad_norm": 9.034782409667969,
      "kl": 3.931640625,
      "learning_rate": 1.9634062175746814e-10,
      "loss": 0.2395,
      "reward": 0.5770089626312256,
      "reward_std": 0.1477005798369646,
      "rewards/accuracy_reward": 0.14732143841683865,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875223517418,
      "step": 3309
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1788.0826721191406,
      "epoch": 0.9887237696960646,
      "grad_norm": 13.084781646728516,
      "kl": 4.1953125,
      "learning_rate": 1.861441404767683e-10,
      "loss": 0.2236,
      "reward": 0.5156250223517418,
      "reward_std": 0.18328437954187393,
      "rewards/accuracy_reward": 0.09598214854486287,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4196428805589676,
      "step": 3310
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1739.72998046875,
      "epoch": 0.989022477783586,
      "grad_norm": 17.625808715820312,
      "kl": 4.48046875,
      "learning_rate": 1.7621943218681933e-10,
      "loss": 0.2524,
      "reward": 0.4614955559372902,
      "reward_std": 0.1440868340432644,
      "rewards/accuracy_reward": 0.042410714784637094,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4190848395228386,
      "step": 3311
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1820.7478637695312,
      "epoch": 0.9893211858711075,
      "grad_norm": 17.064064025878906,
      "kl": 5.22265625,
      "learning_rate": 1.6656650768478998e-10,
      "loss": 0.2759,
      "reward": 0.412388414144516,
      "reward_std": 0.11719540879130363,
      "rewards/accuracy_reward": 0.004464285913854837,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.407924123108387,
      "step": 3312
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1775.1473999023438,
      "epoch": 0.9896198939586289,
      "grad_norm": 7.499997138977051,
      "kl": 3.52734375,
      "learning_rate": 1.5718537747208548e-10,
      "loss": 0.1934,
      "reward": 0.5089285969734192,
      "reward_std": 0.1658635027706623,
      "rewards/accuracy_reward": 0.08482143329456449,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.424107164144516,
      "step": 3313
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1788.7701721191406,
      "epoch": 0.9899186020461505,
      "grad_norm": 7.739022254943848,
      "kl": 4.26953125,
      "learning_rate": 1.4807605175445882e-10,
      "loss": 0.2513,
      "reward": 0.491071455180645,
      "reward_std": 0.17251384630799294,
      "rewards/accuracy_reward": 0.0714285729918629,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4196428805589676,
      "step": 3314
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1814.1875915527344,
      "epoch": 0.9902173101336719,
      "grad_norm": 7.714453220367432,
      "kl": 4.052734375,
      "learning_rate": 1.392385404419827e-10,
      "loss": 0.2182,
      "reward": 0.4654018133878708,
      "reward_std": 0.1209937110543251,
      "rewards/accuracy_reward": 0.044642860535532236,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4207589402794838,
      "step": 3315
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1833.8505249023438,
      "epoch": 0.9905160182211933,
      "grad_norm": 9.077740669250488,
      "kl": 4.37109375,
      "learning_rate": 1.3067285314896648e-10,
      "loss": 0.2411,
      "reward": 0.4849330559372902,
      "reward_std": 0.16769181191921234,
      "rewards/accuracy_reward": 0.07366071688011289,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4112723395228386,
      "step": 3316
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1734.7701721191406,
      "epoch": 0.9908147263087148,
      "grad_norm": 4.371685981750488,
      "kl": 3.955078125,
      "learning_rate": 1.2237899919406714e-10,
      "loss": 0.2241,
      "reward": 0.4815848544239998,
      "reward_std": 0.13078913651406765,
      "rewards/accuracy_reward": 0.06473214412108064,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4168526977300644,
      "step": 3317
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1858.1429443359375,
      "epoch": 0.9911134343962362,
      "grad_norm": 31.608108520507812,
      "kl": 5.953125,
      "learning_rate": 1.143569876001782e-10,
      "loss": 0.3017,
      "reward": 0.4921875298023224,
      "reward_std": 0.15660439990460873,
      "rewards/accuracy_reward": 0.1004464328289032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3917410895228386,
      "step": 3318
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1727.3728637695312,
      "epoch": 0.9914121424837578,
      "grad_norm": 11.799995422363281,
      "kl": 3.189453125,
      "learning_rate": 1.0660682709445757e-10,
      "loss": 0.204,
      "reward": 0.4944196715950966,
      "reward_std": 0.13499074801802635,
      "rewards/accuracy_reward": 0.0580357164144516,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4363839477300644,
      "step": 3319
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1747.4130249023438,
      "epoch": 0.9917108505712792,
      "grad_norm": 3.338597536087036,
      "kl": 3.98046875,
      "learning_rate": 9.912852610832744e-11,
      "loss": 0.2451,
      "reward": 0.576450914144516,
      "reward_std": 0.14545562490820885,
      "rewards/accuracy_reward": 0.14732143469154835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294813156128,
      "step": 3320
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1754.5670471191406,
      "epoch": 0.9920095586588007,
      "grad_norm": 7.347782611846924,
      "kl": 3.78125,
      "learning_rate": 9.192209277744667e-11,
      "loss": 0.2276,
      "reward": 0.5401785895228386,
      "reward_std": 0.18300871923565865,
      "rewards/accuracy_reward": 0.1116071455180645,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 3321
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1747.68310546875,
      "epoch": 0.9923082667463221,
      "grad_norm": 8.304889678955078,
      "kl": 3.779296875,
      "learning_rate": 8.498753494171063e-11,
      "loss": 0.2293,
      "reward": 0.512276828289032,
      "reward_std": 0.18688325583934784,
      "rewards/accuracy_reward": 0.0892857201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4229910969734192,
      "step": 3322
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1716.9688415527344,
      "epoch": 0.9926069748338436,
      "grad_norm": 6.801143169403076,
      "kl": 3.220703125,
      "learning_rate": 7.832486014522355e-11,
      "loss": 0.1794,
      "reward": 0.573660746216774,
      "reward_std": 0.21418777480721474,
      "rewards/accuracy_reward": 0.13839286006987095,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435267873108387,
      "step": 3323
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1798.6965026855469,
      "epoch": 0.992905682921365,
      "grad_norm": 9.317425727844238,
      "kl": 4.25,
      "learning_rate": 7.193407563635401e-11,
      "loss": 0.2238,
      "reward": 0.4698661044239998,
      "reward_std": 0.16084278095513582,
      "rewards/accuracy_reward": 0.04241071757860482,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4274553880095482,
      "step": 3324
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1687.2745971679688,
      "epoch": 0.9932043910088866,
      "grad_norm": 13.472347259521484,
      "kl": 3.583984375,
      "learning_rate": 6.581518836759614e-11,
      "loss": 0.2217,
      "reward": 0.4553571566939354,
      "reward_std": 0.12054107896983624,
      "rewards/accuracy_reward": 0.020089286379516125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4352678805589676,
      "step": 3325
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1762.6607971191406,
      "epoch": 0.993503099096408,
      "grad_norm": 8.46340560913086,
      "kl": 3.80859375,
      "learning_rate": 5.996820499573619e-11,
      "loss": 0.2379,
      "reward": 0.5357143059372902,
      "reward_std": 0.1387111134827137,
      "rewards/accuracy_reward": 0.1049107201397419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4308035895228386,
      "step": 3326
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1825.2032165527344,
      "epoch": 0.9938018071839295,
      "grad_norm": 14.501029968261719,
      "kl": 4.38671875,
      "learning_rate": 5.439313188171368e-11,
      "loss": 0.2311,
      "reward": 0.479910746216774,
      "reward_std": 0.1539971437305212,
      "rewards/accuracy_reward": 0.06473214598372579,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4151785895228386,
      "step": 3327
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1724.6920471191406,
      "epoch": 0.9941005152714509,
      "grad_norm": 8.068860054016113,
      "kl": 4.01953125,
      "learning_rate": 4.9089975090677025e-11,
      "loss": 0.2372,
      "reward": 0.4988839402794838,
      "reward_std": 0.19038104265928268,
      "rewards/accuracy_reward": 0.07812500488944352,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4207589477300644,
      "step": 3328
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.8862609863281,
      "epoch": 0.9943992233589725,
      "grad_norm": 5.389657497406006,
      "kl": 3.8828125,
      "learning_rate": 4.4058740391927917e-11,
      "loss": 0.2158,
      "reward": 0.5496651977300644,
      "reward_std": 0.1852264553308487,
      "rewards/accuracy_reward": 0.12946429522708058,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.420200914144516,
      "step": 3329
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1758.8215026855469,
      "epoch": 0.9946979314464939,
      "grad_norm": 16.15484046936035,
      "kl": 3.390625,
      "learning_rate": 3.929943325892138e-11,
      "loss": 0.225,
      "reward": 0.4648437723517418,
      "reward_std": 0.14931855909526348,
      "rewards/accuracy_reward": 0.042410716181620955,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4224330633878708,
      "step": 3330
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1794.384033203125,
      "epoch": 0.9949966395340154,
      "grad_norm": 12.177657127380371,
      "kl": 3.935546875,
      "learning_rate": 3.481205886937677e-11,
      "loss": 0.2182,
      "reward": 0.4687500298023224,
      "reward_std": 0.1693696491420269,
      "rewards/accuracy_reward": 0.05357142956927419,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4151785895228386,
      "step": 3331
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1795.6005249023438,
      "epoch": 0.9952953476215368,
      "grad_norm": 15.174638748168945,
      "kl": 4.70703125,
      "learning_rate": 3.059662210508351e-11,
      "loss": 0.2515,
      "reward": 0.4838169887661934,
      "reward_std": 0.1500650830566883,
      "rewards/accuracy_reward": 0.0714285746216774,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4123884066939354,
      "step": 3332
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1783.1764526367188,
      "epoch": 0.9955940557090583,
      "grad_norm": 26.510358810424805,
      "kl": 4.046875,
      "learning_rate": 2.6653127552039857e-11,
      "loss": 0.2588,
      "reward": 0.5329241305589676,
      "reward_std": 0.14724184572696686,
      "rewards/accuracy_reward": 0.1071428582072258,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4257812723517418,
      "step": 3333
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1705.9889526367188,
      "epoch": 0.9958927637965798,
      "grad_norm": 19.114683151245117,
      "kl": 3.1484375,
      "learning_rate": 2.298157950036961e-11,
      "loss": 0.1983,
      "reward": 0.5290178805589676,
      "reward_std": 0.12315829284489155,
      "rewards/accuracy_reward": 0.08705357578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4419642984867096,
      "step": 3334
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1832.0670166015625,
      "epoch": 0.9961914718841013,
      "grad_norm": 18.255840301513672,
      "kl": 4.87890625,
      "learning_rate": 1.95819819443499e-11,
      "loss": 0.2573,
      "reward": 0.4793526902794838,
      "reward_std": 0.151786956936121,
      "rewards/accuracy_reward": 0.06250000232830644,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4168526977300644,
      "step": 3335
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1745.6697387695312,
      "epoch": 0.9964901799716227,
      "grad_norm": 16.174894332885742,
      "kl": 3.26953125,
      "learning_rate": 1.6454338582411186e-11,
      "loss": 0.2027,
      "reward": 0.5518973469734192,
      "reward_std": 0.16958784870803356,
      "rewards/accuracy_reward": 0.11607143399305642,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.435825914144516,
      "step": 3336
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1775.1875915527344,
      "epoch": 0.9967888880591442,
      "grad_norm": 13.705008506774902,
      "kl": 4.1171875,
      "learning_rate": 1.3598652817137235e-11,
      "loss": 0.2314,
      "reward": 0.5279018059372902,
      "reward_std": 0.13746609166264534,
      "rewards/accuracy_reward": 0.10267857508733869,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4252232387661934,
      "step": 3337
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1734.1540832519531,
      "epoch": 0.9970875961466656,
      "grad_norm": 9.357471466064453,
      "kl": 3.31640625,
      "learning_rate": 1.101492775523738e-11,
      "loss": 0.2,
      "reward": 0.5206473469734192,
      "reward_std": 0.15125216729938984,
      "rewards/accuracy_reward": 0.08258929080329835,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4380580484867096,
      "step": 3338
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1756.18310546875,
      "epoch": 0.9973863042341872,
      "grad_norm": 6.2276153564453125,
      "kl": 3.11328125,
      "learning_rate": 8.703166207546519e-12,
      "loss": 0.164,
      "reward": 0.5379464626312256,
      "reward_std": 0.1417186800390482,
      "rewards/accuracy_reward": 0.1093750074505806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4285714477300644,
      "step": 3339
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1666.4844360351562,
      "epoch": 0.9976850123217086,
      "grad_norm": 3.647441864013672,
      "kl": 3.408203125,
      "learning_rate": 6.663370689025116e-12,
      "loss": 0.2087,
      "reward": 0.5295759215950966,
      "reward_std": 0.1530846431851387,
      "rewards/accuracy_reward": 0.08928571827709675,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4402901902794838,
      "step": 3340
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1797.1630554199219,
      "epoch": 0.9979837204092301,
      "grad_norm": 7.0721611976623535,
      "kl": 4.078125,
      "learning_rate": 4.895543418786952e-12,
      "loss": 0.2267,
      "reward": 0.5005580633878708,
      "reward_std": 0.14737574383616447,
      "rewards/accuracy_reward": 0.07366071944124997,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4268973395228386,
      "step": 3341
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1782.3706359863281,
      "epoch": 0.9982824284967515,
      "grad_norm": 8.780208587646484,
      "kl": 3.83203125,
      "learning_rate": 3.39968632001586e-12,
      "loss": 0.2099,
      "reward": 0.5429687649011612,
      "reward_std": 0.1996011659502983,
      "rewards/accuracy_reward": 0.1250000107102096,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4179687723517418,
      "step": 3342
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1744.7188415527344,
      "epoch": 0.998581136584273,
      "grad_norm": 10.397480010986328,
      "kl": 3.7578125,
      "learning_rate": 2.1758010201045063e-12,
      "loss": 0.2329,
      "reward": 0.4983259215950966,
      "reward_std": 0.16362856328487396,
      "rewards/accuracy_reward": 0.06919643329456449,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294887661934,
      "step": 3343
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1725.8215026855469,
      "epoch": 0.9988798446717945,
      "grad_norm": 4.162888050079346,
      "kl": 3.62890625,
      "learning_rate": 1.2238888505156087e-12,
      "loss": 0.2243,
      "reward": 0.5161830559372902,
      "reward_std": 0.16230366006493568,
      "rewards/accuracy_reward": 0.08705357206054032,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4291294813156128,
      "step": 3344
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1858.4331359863281,
      "epoch": 0.999178552759316,
      "grad_norm": 9.17155647277832,
      "kl": 4.8828125,
      "learning_rate": 5.439508467819375e-13,
      "loss": 0.2617,
      "reward": 0.462611623108387,
      "reward_std": 0.16037741117179394,
      "rewards/accuracy_reward": 0.05580357415601611,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4068080484867096,
      "step": 3345
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1686.8817749023438,
      "epoch": 0.9994772608468374,
      "grad_norm": 23.49078941345215,
      "kl": 2.82421875,
      "learning_rate": 1.3598774867285002e-13,
      "loss": 0.1918,
      "reward": 0.611049123108387,
      "reward_std": 0.17189978808164597,
      "rewards/accuracy_reward": 0.16071429592557251,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4503348395228386,
      "step": 3346
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1807.5380249023438,
      "epoch": 0.9997759689343589,
      "grad_norm": 17.386194229125977,
      "kl": 4.5703125,
      "learning_rate": 0.0,
      "loss": 0.241,
      "reward": 0.4966518133878708,
      "reward_std": 0.16918558441102505,
      "rewards/accuracy_reward": 0.08705357578583062,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4095982387661934,
      "step": 3347
    },
    {
      "epoch": 0.9997759689343589,
      "step": 3347,
      "total_flos": 0.0,
      "train_loss": 0.08076191042202922,
      "train_runtime": 127083.2818,
      "train_samples_per_second": 0.738,
      "train_steps_per_second": 0.026
    }
  ],
  "logging_steps": 1,
  "max_steps": 3347,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 400,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 16,
  "trial_name": null,
  "trial_params": null
}