File size: 5,661 Bytes
ded719e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 |
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.9936305732484076,
"eval_steps": 500,
"global_step": 78,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.012738853503184714,
"grad_norm": 13.343631461650636,
"learning_rate": 6.25e-08,
"logits/chosen": -2.923123598098755,
"logits/rejected": -2.883568048477173,
"logps/chosen": -275.7285461425781,
"logps/pi_response": -74.39646911621094,
"logps/ref_response": -74.39646911621094,
"logps/rejected": -99.70209503173828,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1
},
{
"epoch": 0.12738853503184713,
"grad_norm": 11.253513207180946,
"learning_rate": 4.989935734988097e-07,
"logits/chosen": -2.8164381980895996,
"logits/rejected": -2.781937837600708,
"logps/chosen": -227.93551635742188,
"logps/pi_response": -64.4197998046875,
"logps/ref_response": -63.84031295776367,
"logps/rejected": -105.435302734375,
"loss": 0.6891,
"rewards/accuracies": 0.5208333134651184,
"rewards/chosen": 0.005453579593449831,
"rewards/margins": 0.006863526534289122,
"rewards/rejected": -0.0014099470572546124,
"step": 10
},
{
"epoch": 0.25477707006369427,
"grad_norm": 7.603005231304034,
"learning_rate": 4.646121984004665e-07,
"logits/chosen": -2.661804676055908,
"logits/rejected": -2.629075050354004,
"logps/chosen": -276.488037109375,
"logps/pi_response": -90.92379760742188,
"logps/ref_response": -75.4796142578125,
"logps/rejected": -123.3162612915039,
"loss": 0.6422,
"rewards/accuracies": 0.8062499761581421,
"rewards/chosen": 0.035294655710458755,
"rewards/margins": 0.16240255534648895,
"rewards/rejected": -0.1271079182624817,
"step": 20
},
{
"epoch": 0.3821656050955414,
"grad_norm": 6.985251610579986,
"learning_rate": 3.877242453630256e-07,
"logits/chosen": -2.6086363792419434,
"logits/rejected": -2.5879595279693604,
"logps/chosen": -249.6334686279297,
"logps/pi_response": -124.77403259277344,
"logps/ref_response": -73.44969177246094,
"logps/rejected": -150.97244262695312,
"loss": 0.6136,
"rewards/accuracies": 0.768750011920929,
"rewards/chosen": -0.165732279419899,
"rewards/margins": 0.25328412652015686,
"rewards/rejected": -0.41901636123657227,
"step": 30
},
{
"epoch": 0.5095541401273885,
"grad_norm": 10.763929928294964,
"learning_rate": 2.8355831645441387e-07,
"logits/chosen": -2.5407841205596924,
"logits/rejected": -2.5103654861450195,
"logps/chosen": -229.60879516601562,
"logps/pi_response": -129.8967742919922,
"logps/ref_response": -65.92216491699219,
"logps/rejected": -167.57904052734375,
"loss": 0.5969,
"rewards/accuracies": 0.737500011920929,
"rewards/chosen": -0.27106621861457825,
"rewards/margins": 0.2314259111881256,
"rewards/rejected": -0.5024920701980591,
"step": 40
},
{
"epoch": 0.6369426751592356,
"grad_norm": 10.451471450915024,
"learning_rate": 1.7274575140626315e-07,
"logits/chosen": -2.5744354724884033,
"logits/rejected": -2.5362110137939453,
"logps/chosen": -273.7886047363281,
"logps/pi_response": -153.47801208496094,
"logps/ref_response": -70.1203842163086,
"logps/rejected": -174.98007202148438,
"loss": 0.5556,
"rewards/accuracies": 0.762499988079071,
"rewards/chosen": -0.27356070280075073,
"rewards/margins": 0.4625588357448578,
"rewards/rejected": -0.7361195683479309,
"step": 50
},
{
"epoch": 0.7643312101910829,
"grad_norm": 11.045205015520507,
"learning_rate": 7.723433775328384e-08,
"logits/chosen": -2.589782476425171,
"logits/rejected": -2.533975124359131,
"logps/chosen": -272.664306640625,
"logps/pi_response": -169.4038543701172,
"logps/ref_response": -68.78282165527344,
"logps/rejected": -199.9068145751953,
"loss": 0.5591,
"rewards/accuracies": 0.737500011920929,
"rewards/chosen": -0.3962182402610779,
"rewards/margins": 0.48118796944618225,
"rewards/rejected": -0.877406120300293,
"step": 60
},
{
"epoch": 0.89171974522293,
"grad_norm": 9.344569317622456,
"learning_rate": 1.5941282340065697e-08,
"logits/chosen": -2.5518054962158203,
"logits/rejected": -2.5584278106689453,
"logps/chosen": -255.5514678955078,
"logps/pi_response": -165.05784606933594,
"logps/ref_response": -72.47773742675781,
"logps/rejected": -196.47874450683594,
"loss": 0.5526,
"rewards/accuracies": 0.768750011920929,
"rewards/chosen": -0.327322781085968,
"rewards/margins": 0.45833224058151245,
"rewards/rejected": -0.7856550812721252,
"step": 70
},
{
"epoch": 0.9936305732484076,
"step": 78,
"total_flos": 0.0,
"train_loss": 0.5938225159278283,
"train_runtime": 1654.7204,
"train_samples_per_second": 6.043,
"train_steps_per_second": 0.047
}
],
"logging_steps": 10,
"max_steps": 78,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 100,
"total_flos": 0.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
|