diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,18200 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.999297541394882, + "eval_steps": 400, + "global_step": 5604, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.002676032781401572, + "grad_norm": 9.17970088825437, + "learning_rate": 8.9126559714795e-09, + "logits/chosen": -0.07006202638149261, + "logits/rejected": 0.1360432207584381, + "logps/chosen": -1.7161109447479248, + "logps/rejected": -1.8897171020507812, + "loss": 2.289, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.7161109447479248, + "rewards/margins": 0.17360590398311615, + "rewards/rejected": -1.8897171020507812, + "semantic_entropy": 0.6584368348121643, + "step": 5 + }, + { + "epoch": 0.005352065562803144, + "grad_norm": 22.707738390768604, + "learning_rate": 1.7825311942959e-08, + "logits/chosen": 0.006351391319185495, + "logits/rejected": 0.12548018991947174, + "logps/chosen": -1.801119089126587, + "logps/rejected": -1.8452117443084717, + "loss": 2.3709, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.801119089126587, + "rewards/margins": 0.04409261420369148, + "rewards/rejected": -1.8452117443084717, + "semantic_entropy": 0.6396836042404175, + "step": 10 + }, + { + "epoch": 0.008028098344204716, + "grad_norm": 20.020177676042728, + "learning_rate": 2.67379679144385e-08, + "logits/chosen": -0.024356648325920105, + "logits/rejected": 0.07323823869228363, + "logps/chosen": -1.6344894170761108, + "logps/rejected": -1.7645975351333618, + "loss": 2.2682, + "rewards/accuracies": 0.4937500059604645, + "rewards/chosen": -1.6344894170761108, + "rewards/margins": 0.13010787963867188, + "rewards/rejected": -1.7645975351333618, + "semantic_entropy": 0.693292498588562, + "step": 15 + }, + { + "epoch": 0.010704131125606288, + "grad_norm": 10.276327712753156, + "learning_rate": 3.5650623885918e-08, + "logits/chosen": -0.02814987301826477, + "logits/rejected": 0.056642692536115646, + "logps/chosen": -1.725663185119629, + "logps/rejected": -1.8059051036834717, + "loss": 2.3382, + "rewards/accuracies": 0.48750001192092896, + "rewards/chosen": -1.725663185119629, + "rewards/margins": 0.08024205267429352, + "rewards/rejected": -1.8059051036834717, + "semantic_entropy": 0.6685150861740112, + "step": 20 + }, + { + "epoch": 0.013380163907007862, + "grad_norm": 27.180149093707602, + "learning_rate": 4.45632798573975e-08, + "logits/chosen": -0.043658602982759476, + "logits/rejected": 0.040141694247722626, + "logps/chosen": -1.8687822818756104, + "logps/rejected": -1.7786893844604492, + "loss": 2.4973, + "rewards/accuracies": 0.3812499940395355, + "rewards/chosen": -1.8687822818756104, + "rewards/margins": -0.09009285271167755, + "rewards/rejected": -1.7786893844604492, + "semantic_entropy": 0.6433964967727661, + "step": 25 + }, + { + "epoch": 0.016056196688409432, + "grad_norm": 24.409252182598028, + "learning_rate": 5.3475935828877e-08, + "logits/chosen": -0.08515056222677231, + "logits/rejected": 0.0056139142252504826, + "logps/chosen": -1.90885329246521, + "logps/rejected": -1.8324095010757446, + "loss": 2.4654, + "rewards/accuracies": 0.4437499940395355, + "rewards/chosen": -1.90885329246521, + "rewards/margins": -0.07644428312778473, + "rewards/rejected": -1.8324095010757446, + "semantic_entropy": 0.6178733706474304, + "step": 30 + }, + { + "epoch": 0.018732229469811006, + "grad_norm": 16.574736077868895, + "learning_rate": 6.23885918003565e-08, + "logits/chosen": -0.04204554110765457, + "logits/rejected": 0.11867649853229523, + "logps/chosen": -1.845669150352478, + "logps/rejected": -1.9964662790298462, + "loss": 2.4259, + "rewards/accuracies": 0.48750001192092896, + "rewards/chosen": -1.845669150352478, + "rewards/margins": 0.15079709887504578, + "rewards/rejected": -1.9964662790298462, + "semantic_entropy": 0.6348816752433777, + "step": 35 + }, + { + "epoch": 0.021408262251212576, + "grad_norm": 23.292130959882876, + "learning_rate": 7.1301247771836e-08, + "logits/chosen": 0.038313932716846466, + "logits/rejected": 0.2096748799085617, + "logps/chosen": -1.8787275552749634, + "logps/rejected": -1.7414252758026123, + "loss": 2.4792, + "rewards/accuracies": 0.45625001192092896, + "rewards/chosen": -1.8787275552749634, + "rewards/margins": -0.13730214536190033, + "rewards/rejected": -1.7414252758026123, + "semantic_entropy": 0.6435689926147461, + "step": 40 + }, + { + "epoch": 0.02408429503261415, + "grad_norm": 20.477762582760594, + "learning_rate": 8.021390374331551e-08, + "logits/chosen": -0.003790763672441244, + "logits/rejected": 0.18562594056129456, + "logps/chosen": -1.8347675800323486, + "logps/rejected": -1.8684990406036377, + "loss": 2.4304, + "rewards/accuracies": 0.48750001192092896, + "rewards/chosen": -1.8347675800323486, + "rewards/margins": 0.033731609582901, + "rewards/rejected": -1.8684990406036377, + "semantic_entropy": 0.6498380899429321, + "step": 45 + }, + { + "epoch": 0.026760327814015723, + "grad_norm": 24.218166746857367, + "learning_rate": 8.9126559714795e-08, + "logits/chosen": -0.044481705874204636, + "logits/rejected": 0.10558140277862549, + "logps/chosen": -1.8938385248184204, + "logps/rejected": -1.774763822555542, + "loss": 2.4776, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -1.8938385248184204, + "rewards/margins": -0.11907454580068588, + "rewards/rejected": -1.774763822555542, + "semantic_entropy": 0.6348689794540405, + "step": 50 + }, + { + "epoch": 0.029436360595417294, + "grad_norm": 19.346768220162602, + "learning_rate": 9.80392156862745e-08, + "logits/chosen": -0.095049187541008, + "logits/rejected": 0.12877969443798065, + "logps/chosen": -1.8224636316299438, + "logps/rejected": -1.8576208353042603, + "loss": 2.4, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.8224636316299438, + "rewards/margins": 0.035157203674316406, + "rewards/rejected": -1.8576208353042603, + "semantic_entropy": 0.6459155082702637, + "step": 55 + }, + { + "epoch": 0.032112393376818864, + "grad_norm": 22.14339393235795, + "learning_rate": 1.06951871657754e-07, + "logits/chosen": -0.06976853311061859, + "logits/rejected": 0.12213323265314102, + "logps/chosen": -1.7787249088287354, + "logps/rejected": -1.8833869695663452, + "loss": 2.3455, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -1.7787249088287354, + "rewards/margins": 0.1046619862318039, + "rewards/rejected": -1.8833869695663452, + "semantic_entropy": 0.6385133862495422, + "step": 60 + }, + { + "epoch": 0.03478842615822044, + "grad_norm": 21.514956821961622, + "learning_rate": 1.158645276292335e-07, + "logits/chosen": -0.02376721426844597, + "logits/rejected": 0.12463350594043732, + "logps/chosen": -1.6288169622421265, + "logps/rejected": -1.7579704523086548, + "loss": 2.2499, + "rewards/accuracies": 0.53125, + "rewards/chosen": -1.6288169622421265, + "rewards/margins": 0.12915340065956116, + "rewards/rejected": -1.7579704523086548, + "semantic_entropy": 0.6990243196487427, + "step": 65 + }, + { + "epoch": 0.03746445893962201, + "grad_norm": 21.041922188660656, + "learning_rate": 1.24777183600713e-07, + "logits/chosen": -0.07015866041183472, + "logits/rejected": 0.08012659847736359, + "logps/chosen": -1.754617691040039, + "logps/rejected": -1.7989566326141357, + "loss": 2.3606, + "rewards/accuracies": 0.42500001192092896, + "rewards/chosen": -1.754617691040039, + "rewards/margins": 0.04433891549706459, + "rewards/rejected": -1.7989566326141357, + "semantic_entropy": 0.657590925693512, + "step": 70 + }, + { + "epoch": 0.04014049172102358, + "grad_norm": 17.482112329039467, + "learning_rate": 1.3368983957219251e-07, + "logits/chosen": -0.04342065379023552, + "logits/rejected": 0.13985566794872284, + "logps/chosen": -1.7492595911026, + "logps/rejected": -2.0036559104919434, + "loss": 2.318, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -1.7492595911026, + "rewards/margins": 0.2543964982032776, + "rewards/rejected": -2.0036559104919434, + "semantic_entropy": 0.6419968605041504, + "step": 75 + }, + { + "epoch": 0.04281652450242515, + "grad_norm": 15.988514228088759, + "learning_rate": 1.42602495543672e-07, + "logits/chosen": -0.007324705831706524, + "logits/rejected": 0.09563705325126648, + "logps/chosen": -1.6806221008300781, + "logps/rejected": -1.7140804529190063, + "loss": 2.2999, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.6806221008300781, + "rewards/margins": 0.03345843032002449, + "rewards/rejected": -1.7140804529190063, + "semantic_entropy": 0.6795748472213745, + "step": 80 + }, + { + "epoch": 0.04549255728382673, + "grad_norm": 11.611154920796695, + "learning_rate": 1.5151515151515152e-07, + "logits/chosen": -0.17916560173034668, + "logits/rejected": 0.05491837114095688, + "logps/chosen": -1.7472093105316162, + "logps/rejected": -1.9139823913574219, + "loss": 2.3546, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.7472093105316162, + "rewards/margins": 0.1667732149362564, + "rewards/rejected": -1.9139823913574219, + "semantic_entropy": 0.6593233346939087, + "step": 85 + }, + { + "epoch": 0.0481685900652283, + "grad_norm": 22.467095092939147, + "learning_rate": 1.6042780748663102e-07, + "logits/chosen": 0.07985838502645493, + "logits/rejected": 0.043650977313518524, + "logps/chosen": -1.6980218887329102, + "logps/rejected": -1.7370532751083374, + "loss": 2.3257, + "rewards/accuracies": 0.46875, + "rewards/chosen": -1.6980218887329102, + "rewards/margins": 0.03903146833181381, + "rewards/rejected": -1.7370532751083374, + "semantic_entropy": 0.6805222630500793, + "step": 90 + }, + { + "epoch": 0.05084462284662987, + "grad_norm": 26.392041621319432, + "learning_rate": 1.693404634581105e-07, + "logits/chosen": -0.09472827613353729, + "logits/rejected": 0.051044244319200516, + "logps/chosen": -1.7366969585418701, + "logps/rejected": -1.8614689111709595, + "loss": 2.3268, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.7366969585418701, + "rewards/margins": 0.12477195262908936, + "rewards/rejected": -1.8614689111709595, + "semantic_entropy": 0.6559633612632751, + "step": 95 + }, + { + "epoch": 0.05352065562803145, + "grad_norm": 11.596621418184593, + "learning_rate": 1.7825311942959e-07, + "logits/chosen": -0.04228469356894493, + "logits/rejected": 0.019458714872598648, + "logps/chosen": -1.6304876804351807, + "logps/rejected": -1.7368942499160767, + "loss": 2.2454, + "rewards/accuracies": 0.5062500238418579, + "rewards/chosen": -1.6304876804351807, + "rewards/margins": 0.10640676319599152, + "rewards/rejected": -1.7368942499160767, + "semantic_entropy": 0.6917005777359009, + "step": 100 + }, + { + "epoch": 0.05619668840943302, + "grad_norm": 16.464623708925348, + "learning_rate": 1.8716577540106952e-07, + "logits/chosen": 0.03008785843849182, + "logits/rejected": 0.056171614676713943, + "logps/chosen": -1.5428593158721924, + "logps/rejected": -1.7105385065078735, + "loss": 2.1781, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.5428593158721924, + "rewards/margins": 0.16767899692058563, + "rewards/rejected": -1.7105385065078735, + "semantic_entropy": 0.7140295505523682, + "step": 105 + }, + { + "epoch": 0.05887272119083459, + "grad_norm": 15.32604700768848, + "learning_rate": 1.96078431372549e-07, + "logits/chosen": -0.02091386541724205, + "logits/rejected": 0.07242826372385025, + "logps/chosen": -1.532142996788025, + "logps/rejected": -1.5921493768692017, + "loss": 2.2011, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -1.532142996788025, + "rewards/margins": 0.06000634282827377, + "rewards/rejected": -1.5921493768692017, + "semantic_entropy": 0.7271608710289001, + "step": 110 + }, + { + "epoch": 0.06154875397223616, + "grad_norm": 17.01697563425044, + "learning_rate": 2.049910873440285e-07, + "logits/chosen": -0.007334655616432428, + "logits/rejected": 0.19188645482063293, + "logps/chosen": -1.5300090312957764, + "logps/rejected": -1.7706422805786133, + "loss": 2.1436, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.5300090312957764, + "rewards/margins": 0.24063313007354736, + "rewards/rejected": -1.7706422805786133, + "semantic_entropy": 0.7088441848754883, + "step": 115 + }, + { + "epoch": 0.06422478675363773, + "grad_norm": 17.820280607333967, + "learning_rate": 2.13903743315508e-07, + "logits/chosen": -0.10667898505926132, + "logits/rejected": 0.058054011315107346, + "logps/chosen": -1.5706758499145508, + "logps/rejected": -1.6764158010482788, + "loss": 2.2182, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -1.5706758499145508, + "rewards/margins": 0.10573999583721161, + "rewards/rejected": -1.6764158010482788, + "semantic_entropy": 0.707051694393158, + "step": 120 + }, + { + "epoch": 0.0669008195350393, + "grad_norm": 5.961991097766711, + "learning_rate": 2.2281639928698751e-07, + "logits/chosen": -0.08696900308132172, + "logits/rejected": 0.043225500732660294, + "logps/chosen": -1.5204510688781738, + "logps/rejected": -1.4943760633468628, + "loss": 2.2164, + "rewards/accuracies": 0.4625000059604645, + "rewards/chosen": -1.5204510688781738, + "rewards/margins": -0.026074940338730812, + "rewards/rejected": -1.4943760633468628, + "semantic_entropy": 0.7435601353645325, + "step": 125 + }, + { + "epoch": 0.06957685231644088, + "grad_norm": 21.999826366025292, + "learning_rate": 2.31729055258467e-07, + "logits/chosen": 0.011612406000494957, + "logits/rejected": 0.1401824653148651, + "logps/chosen": -1.5498534440994263, + "logps/rejected": -1.6641845703125, + "loss": 2.1912, + "rewards/accuracies": 0.48750001192092896, + "rewards/chosen": -1.5498534440994263, + "rewards/margins": 0.11433116346597672, + "rewards/rejected": -1.6641845703125, + "semantic_entropy": 0.7085353136062622, + "step": 130 + }, + { + "epoch": 0.07225288509784245, + "grad_norm": 21.874086544749737, + "learning_rate": 2.406417112299465e-07, + "logits/chosen": -0.07757744938135147, + "logits/rejected": 0.03269508481025696, + "logps/chosen": -1.5848090648651123, + "logps/rejected": -1.628740668296814, + "loss": 2.2373, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -1.5848090648651123, + "rewards/margins": 0.04393132030963898, + "rewards/rejected": -1.628740668296814, + "semantic_entropy": 0.7085167765617371, + "step": 135 + }, + { + "epoch": 0.07492891787924402, + "grad_norm": 11.387359947808873, + "learning_rate": 2.49554367201426e-07, + "logits/chosen": -0.058408986777067184, + "logits/rejected": 0.1023082286119461, + "logps/chosen": -1.5260465145111084, + "logps/rejected": -1.6099777221679688, + "loss": 2.1991, + "rewards/accuracies": 0.48750001192092896, + "rewards/chosen": -1.5260465145111084, + "rewards/margins": 0.08393123000860214, + "rewards/rejected": -1.6099777221679688, + "semantic_entropy": 0.7268117070198059, + "step": 140 + }, + { + "epoch": 0.0776049506606456, + "grad_norm": 14.134217333680969, + "learning_rate": 2.5846702317290554e-07, + "logits/chosen": -0.04021311178803444, + "logits/rejected": 0.10950696468353271, + "logps/chosen": -1.394166111946106, + "logps/rejected": -1.4935468435287476, + "loss": 2.1092, + "rewards/accuracies": 0.48750001192092896, + "rewards/chosen": -1.394166111946106, + "rewards/margins": 0.09938089549541473, + "rewards/rejected": -1.4935468435287476, + "semantic_entropy": 0.7643908262252808, + "step": 145 + }, + { + "epoch": 0.08028098344204716, + "grad_norm": 14.660549090593136, + "learning_rate": 2.6737967914438503e-07, + "logits/chosen": -0.11129583418369293, + "logits/rejected": 0.038066692650318146, + "logps/chosen": -1.321720838546753, + "logps/rejected": -1.3290258646011353, + "loss": 2.1164, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -1.321720838546753, + "rewards/margins": 0.0073050023056566715, + "rewards/rejected": -1.3290258646011353, + "semantic_entropy": 0.8203862905502319, + "step": 150 + }, + { + "epoch": 0.08295701622344874, + "grad_norm": 9.337305048028103, + "learning_rate": 2.762923351158645e-07, + "logits/chosen": -0.13234972953796387, + "logits/rejected": -0.0875357836484909, + "logps/chosen": -1.3184583187103271, + "logps/rejected": -1.4332258701324463, + "loss": 2.0793, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -1.3184583187103271, + "rewards/margins": 0.11476727575063705, + "rewards/rejected": -1.4332258701324463, + "semantic_entropy": 0.7989853024482727, + "step": 155 + }, + { + "epoch": 0.0856330490048503, + "grad_norm": 9.617308878215045, + "learning_rate": 2.85204991087344e-07, + "logits/chosen": -0.21252942085266113, + "logits/rejected": -0.08288715779781342, + "logps/chosen": -1.4070971012115479, + "logps/rejected": -1.3882343769073486, + "loss": 2.182, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -1.4070971012115479, + "rewards/margins": -0.018862640485167503, + "rewards/rejected": -1.3882343769073486, + "semantic_entropy": 0.7836942076683044, + "step": 160 + }, + { + "epoch": 0.08830908178625188, + "grad_norm": 12.078012222248885, + "learning_rate": 2.941176470588235e-07, + "logits/chosen": -0.11357660591602325, + "logits/rejected": 0.04716426134109497, + "logps/chosen": -1.3212950229644775, + "logps/rejected": -1.3990843296051025, + "loss": 2.1161, + "rewards/accuracies": 0.48750001192092896, + "rewards/chosen": -1.3212950229644775, + "rewards/margins": 0.07778936624526978, + "rewards/rejected": -1.3990843296051025, + "semantic_entropy": 0.7998541593551636, + "step": 165 + }, + { + "epoch": 0.09098511456765346, + "grad_norm": 13.254483655244782, + "learning_rate": 3.0303030303030305e-07, + "logits/chosen": -0.13945594429969788, + "logits/rejected": -0.0916006937623024, + "logps/chosen": -1.4381730556488037, + "logps/rejected": -1.5070384740829468, + "loss": 2.1607, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.4381730556488037, + "rewards/margins": 0.06886538118124008, + "rewards/rejected": -1.5070384740829468, + "semantic_entropy": 0.763955295085907, + "step": 170 + }, + { + "epoch": 0.09366114734905502, + "grad_norm": 10.22284067289044, + "learning_rate": 3.1194295900178254e-07, + "logits/chosen": -0.009927010163664818, + "logits/rejected": -0.01633218303322792, + "logps/chosen": -1.3267039060592651, + "logps/rejected": -1.4154396057128906, + "loss": 2.1055, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.3267039060592651, + "rewards/margins": 0.08873560279607773, + "rewards/rejected": -1.4154396057128906, + "semantic_entropy": 0.7996432781219482, + "step": 175 + }, + { + "epoch": 0.0963371801304566, + "grad_norm": 7.5910930778984005, + "learning_rate": 3.2085561497326203e-07, + "logits/chosen": -0.060885727405548096, + "logits/rejected": -0.06494145840406418, + "logps/chosen": -1.3440454006195068, + "logps/rejected": -1.5446110963821411, + "loss": 2.0722, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.3440454006195068, + "rewards/margins": 0.20056560635566711, + "rewards/rejected": -1.5446110963821411, + "semantic_entropy": 0.7831510901451111, + "step": 180 + }, + { + "epoch": 0.09901321291185818, + "grad_norm": 7.822879105238539, + "learning_rate": 3.297682709447415e-07, + "logits/chosen": -0.20560979843139648, + "logits/rejected": -0.12279047816991806, + "logps/chosen": -1.3355904817581177, + "logps/rejected": -1.3831441402435303, + "loss": 2.1299, + "rewards/accuracies": 0.4937500059604645, + "rewards/chosen": -1.3355904817581177, + "rewards/margins": 0.04755370691418648, + "rewards/rejected": -1.3831441402435303, + "semantic_entropy": 0.7968094944953918, + "step": 185 + }, + { + "epoch": 0.10168924569325974, + "grad_norm": 9.9835519249466, + "learning_rate": 3.38680926916221e-07, + "logits/chosen": -0.10206165164709091, + "logits/rejected": 0.010062957182526588, + "logps/chosen": -1.2628173828125, + "logps/rejected": -1.3929227590560913, + "loss": 2.0604, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -1.2628173828125, + "rewards/margins": 0.13010530173778534, + "rewards/rejected": -1.3929227590560913, + "semantic_entropy": 0.8037274479866028, + "step": 190 + }, + { + "epoch": 0.10436527847466132, + "grad_norm": 7.658920533300685, + "learning_rate": 3.475935828877005e-07, + "logits/chosen": -0.031039467081427574, + "logits/rejected": 0.11408261954784393, + "logps/chosen": -1.2417497634887695, + "logps/rejected": -1.4050052165985107, + "loss": 2.0318, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.2417497634887695, + "rewards/margins": 0.16325534880161285, + "rewards/rejected": -1.4050052165985107, + "semantic_entropy": 0.8272998929023743, + "step": 195 + }, + { + "epoch": 0.1070413112560629, + "grad_norm": 18.816124807543744, + "learning_rate": 3.5650623885918e-07, + "logits/chosen": -0.12287290394306183, + "logits/rejected": 0.005303362850099802, + "logps/chosen": -1.3640367984771729, + "logps/rejected": -1.4029943943023682, + "loss": 2.1351, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.3640367984771729, + "rewards/margins": 0.03895748034119606, + "rewards/rejected": -1.4029943943023682, + "semantic_entropy": 0.791954755783081, + "step": 200 + }, + { + "epoch": 0.10971734403746446, + "grad_norm": 13.834762563320641, + "learning_rate": 3.654188948306595e-07, + "logits/chosen": -0.08338096737861633, + "logits/rejected": 0.053165972232818604, + "logps/chosen": -1.2766786813735962, + "logps/rejected": -1.346866488456726, + "loss": 2.0827, + "rewards/accuracies": 0.53125, + "rewards/chosen": -1.2766786813735962, + "rewards/margins": 0.07018790394067764, + "rewards/rejected": -1.346866488456726, + "semantic_entropy": 0.8064363598823547, + "step": 205 + }, + { + "epoch": 0.11239337681886603, + "grad_norm": 12.941469291971545, + "learning_rate": 3.7433155080213904e-07, + "logits/chosen": -0.19147761166095734, + "logits/rejected": -0.01435632724314928, + "logps/chosen": -1.3535292148590088, + "logps/rejected": -1.4615424871444702, + "loss": 2.1148, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.3535292148590088, + "rewards/margins": 0.10801327228546143, + "rewards/rejected": -1.4615424871444702, + "semantic_entropy": 0.7920994162559509, + "step": 210 + }, + { + "epoch": 0.1150694096002676, + "grad_norm": 10.314040391266644, + "learning_rate": 3.8324420677361853e-07, + "logits/chosen": -0.2077387273311615, + "logits/rejected": 0.02725372649729252, + "logps/chosen": -1.375613808631897, + "logps/rejected": -1.4290225505828857, + "loss": 2.1308, + "rewards/accuracies": 0.53125, + "rewards/chosen": -1.375613808631897, + "rewards/margins": 0.053408555686473846, + "rewards/rejected": -1.4290225505828857, + "semantic_entropy": 0.7987397313117981, + "step": 215 + }, + { + "epoch": 0.11774544238166917, + "grad_norm": 14.32317789711802, + "learning_rate": 3.92156862745098e-07, + "logits/chosen": -0.005137534346431494, + "logits/rejected": 0.0829891785979271, + "logps/chosen": -1.297531247138977, + "logps/rejected": -1.4413492679595947, + "loss": 2.062, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.297531247138977, + "rewards/margins": 0.14381805062294006, + "rewards/rejected": -1.4413492679595947, + "semantic_entropy": 0.8080397844314575, + "step": 220 + }, + { + "epoch": 0.12042147516307075, + "grad_norm": 7.505593107015226, + "learning_rate": 4.010695187165775e-07, + "logits/chosen": -0.13666629791259766, + "logits/rejected": 0.020686468109488487, + "logps/chosen": -1.3048591613769531, + "logps/rejected": -1.4310519695281982, + "loss": 2.0589, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.3048591613769531, + "rewards/margins": 0.12619265913963318, + "rewards/rejected": -1.4310519695281982, + "semantic_entropy": 0.8000715970993042, + "step": 225 + }, + { + "epoch": 0.12309750794447231, + "grad_norm": 7.547965569715929, + "learning_rate": 4.09982174688057e-07, + "logits/chosen": -0.05341342091560364, + "logits/rejected": 0.014276454225182533, + "logps/chosen": -1.3072868585586548, + "logps/rejected": -1.463921308517456, + "loss": 2.0822, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.3072868585586548, + "rewards/margins": 0.15663442015647888, + "rewards/rejected": -1.463921308517456, + "semantic_entropy": 0.7943694591522217, + "step": 230 + }, + { + "epoch": 0.1257735407258739, + "grad_norm": 12.836607023053217, + "learning_rate": 4.188948306595365e-07, + "logits/chosen": -0.03887845203280449, + "logits/rejected": 0.08260010182857513, + "logps/chosen": -1.2770813703536987, + "logps/rejected": -1.4421340227127075, + "loss": 2.0463, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2770813703536987, + "rewards/margins": 0.1650528609752655, + "rewards/rejected": -1.4421340227127075, + "semantic_entropy": 0.8039000630378723, + "step": 235 + }, + { + "epoch": 0.12844957350727546, + "grad_norm": 5.810148561096096, + "learning_rate": 4.27807486631016e-07, + "logits/chosen": -0.049771975725889206, + "logits/rejected": 0.0695604532957077, + "logps/chosen": -1.293505072593689, + "logps/rejected": -1.4756234884262085, + "loss": 2.0815, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.293505072593689, + "rewards/margins": 0.1821182668209076, + "rewards/rejected": -1.4756234884262085, + "semantic_entropy": 0.7817009091377258, + "step": 240 + }, + { + "epoch": 0.13112560628867703, + "grad_norm": 8.487258861235883, + "learning_rate": 4.3672014260249554e-07, + "logits/chosen": 0.0011432438623160124, + "logits/rejected": 0.11024985462427139, + "logps/chosen": -1.4130035638809204, + "logps/rejected": -1.4374676942825317, + "loss": 2.1648, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.4130035638809204, + "rewards/margins": 0.024464275687932968, + "rewards/rejected": -1.4374676942825317, + "semantic_entropy": 0.7734571099281311, + "step": 245 + }, + { + "epoch": 0.1338016390700786, + "grad_norm": 10.014551196171814, + "learning_rate": 4.4563279857397503e-07, + "logits/chosen": -0.06559150665998459, + "logits/rejected": 0.09288983047008514, + "logps/chosen": -1.2952531576156616, + "logps/rejected": -1.3528729677200317, + "loss": 2.1092, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -1.2952531576156616, + "rewards/margins": 0.05761975795030594, + "rewards/rejected": -1.3528729677200317, + "semantic_entropy": 0.811192512512207, + "step": 250 + }, + { + "epoch": 0.1364776718514802, + "grad_norm": 8.957601045394023, + "learning_rate": 4.545454545454545e-07, + "logits/chosen": -0.04644311964511871, + "logits/rejected": 0.08571015298366547, + "logps/chosen": -1.2608213424682617, + "logps/rejected": -1.3652595281600952, + "loss": 2.0626, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -1.2608213424682617, + "rewards/margins": 0.10443822294473648, + "rewards/rejected": -1.3652595281600952, + "semantic_entropy": 0.821729302406311, + "step": 255 + }, + { + "epoch": 0.13915370463288176, + "grad_norm": 7.633348470343695, + "learning_rate": 4.63458110516934e-07, + "logits/chosen": -0.25433415174484253, + "logits/rejected": -0.15580113232135773, + "logps/chosen": -1.347470760345459, + "logps/rejected": -1.5038115978240967, + "loss": 2.0622, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.347470760345459, + "rewards/margins": 0.15634091198444366, + "rewards/rejected": -1.5038115978240967, + "semantic_entropy": 0.7835784554481506, + "step": 260 + }, + { + "epoch": 0.1418297374142833, + "grad_norm": 9.896790172976969, + "learning_rate": 4.723707664884135e-07, + "logits/chosen": -0.10201990604400635, + "logits/rejected": -0.018684420734643936, + "logps/chosen": -1.3355344533920288, + "logps/rejected": -1.501531720161438, + "loss": 2.067, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.3355344533920288, + "rewards/margins": 0.1659972369670868, + "rewards/rejected": -1.501531720161438, + "semantic_entropy": 0.7691968083381653, + "step": 265 + }, + { + "epoch": 0.1445057701956849, + "grad_norm": 7.608482620241505, + "learning_rate": 4.81283422459893e-07, + "logits/chosen": -0.10369201004505157, + "logits/rejected": 0.01996661350131035, + "logps/chosen": -1.3129630088806152, + "logps/rejected": -1.409122109413147, + "loss": 2.0988, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.3129630088806152, + "rewards/margins": 0.09615901112556458, + "rewards/rejected": -1.409122109413147, + "semantic_entropy": 0.7943571209907532, + "step": 270 + }, + { + "epoch": 0.14718180297708647, + "grad_norm": 9.589428959834835, + "learning_rate": 4.901960784313725e-07, + "logits/chosen": -0.04439730569720268, + "logits/rejected": 0.05072161555290222, + "logps/chosen": -1.263169765472412, + "logps/rejected": -1.4157510995864868, + "loss": 2.0775, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.263169765472412, + "rewards/margins": 0.1525813192129135, + "rewards/rejected": -1.4157510995864868, + "semantic_entropy": 0.8186517953872681, + "step": 275 + }, + { + "epoch": 0.14985783575848804, + "grad_norm": 10.223827861187196, + "learning_rate": 4.99108734402852e-07, + "logits/chosen": -0.12224537134170532, + "logits/rejected": 0.020121756941080093, + "logps/chosen": -1.3141443729400635, + "logps/rejected": -1.4087907075881958, + "loss": 2.0781, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.3141443729400635, + "rewards/margins": 0.09464634954929352, + "rewards/rejected": -1.4087907075881958, + "semantic_entropy": 0.8114911317825317, + "step": 280 + }, + { + "epoch": 0.15253386853988962, + "grad_norm": 9.044891488981245, + "learning_rate": 5.080213903743315e-07, + "logits/chosen": -0.09391235560178757, + "logits/rejected": 0.03823528066277504, + "logps/chosen": -1.341046929359436, + "logps/rejected": -1.4223486185073853, + "loss": 2.1028, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.341046929359436, + "rewards/margins": 0.08130180835723877, + "rewards/rejected": -1.4223486185073853, + "semantic_entropy": 0.7922171950340271, + "step": 285 + }, + { + "epoch": 0.1552099013212912, + "grad_norm": 8.115808422311316, + "learning_rate": 5.169340463458111e-07, + "logits/chosen": -0.12526479363441467, + "logits/rejected": 0.16373701393604279, + "logps/chosen": -1.3602113723754883, + "logps/rejected": -1.4761052131652832, + "loss": 2.0772, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.3602113723754883, + "rewards/margins": 0.11589386314153671, + "rewards/rejected": -1.4761052131652832, + "semantic_entropy": 0.7861930131912231, + "step": 290 + }, + { + "epoch": 0.15788593410269275, + "grad_norm": 11.35002284350681, + "learning_rate": 5.258467023172905e-07, + "logits/chosen": -0.08692610263824463, + "logits/rejected": -0.03394615650177002, + "logps/chosen": -1.262650728225708, + "logps/rejected": -1.3980472087860107, + "loss": 2.0494, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.262650728225708, + "rewards/margins": 0.13539646565914154, + "rewards/rejected": -1.3980472087860107, + "semantic_entropy": 0.8054231405258179, + "step": 295 + }, + { + "epoch": 0.16056196688409433, + "grad_norm": 7.6555481324022905, + "learning_rate": 5.347593582887701e-07, + "logits/chosen": -0.09255888313055038, + "logits/rejected": 0.06662900745868683, + "logps/chosen": -1.3005911111831665, + "logps/rejected": -1.3753496408462524, + "loss": 2.107, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.3005911111831665, + "rewards/margins": 0.0747586116194725, + "rewards/rejected": -1.3753496408462524, + "semantic_entropy": 0.8094614744186401, + "step": 300 + }, + { + "epoch": 0.1632379996654959, + "grad_norm": 6.716780191320411, + "learning_rate": 5.436720142602496e-07, + "logits/chosen": -0.05763741210103035, + "logits/rejected": 0.008676018565893173, + "logps/chosen": -1.4021122455596924, + "logps/rejected": -1.4093527793884277, + "loss": 2.1533, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.4021122455596924, + "rewards/margins": 0.007240760140120983, + "rewards/rejected": -1.4093527793884277, + "semantic_entropy": 0.7869191765785217, + "step": 305 + }, + { + "epoch": 0.16591403244689748, + "grad_norm": 8.57327455330772, + "learning_rate": 5.52584670231729e-07, + "logits/chosen": -0.23644232749938965, + "logits/rejected": -0.1527770459651947, + "logps/chosen": -1.365864872932434, + "logps/rejected": -1.450217366218567, + "loss": 2.1311, + "rewards/accuracies": 0.5062500238418579, + "rewards/chosen": -1.365864872932434, + "rewards/margins": 0.0843525156378746, + "rewards/rejected": -1.450217366218567, + "semantic_entropy": 0.7928934097290039, + "step": 310 + }, + { + "epoch": 0.16859006522829906, + "grad_norm": 9.533260225990018, + "learning_rate": 5.614973262032086e-07, + "logits/chosen": -0.04432467371225357, + "logits/rejected": 0.09949810057878494, + "logps/chosen": -1.3562901020050049, + "logps/rejected": -1.5015990734100342, + "loss": 2.0876, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.3562901020050049, + "rewards/margins": 0.14530906081199646, + "rewards/rejected": -1.5015990734100342, + "semantic_entropy": 0.7747354507446289, + "step": 315 + }, + { + "epoch": 0.1712660980097006, + "grad_norm": 6.5495269716021705, + "learning_rate": 5.70409982174688e-07, + "logits/chosen": -0.09863855689764023, + "logits/rejected": 0.025040656328201294, + "logps/chosen": -1.3150360584259033, + "logps/rejected": -1.365338683128357, + "loss": 2.1136, + "rewards/accuracies": 0.5062500238418579, + "rewards/chosen": -1.3150360584259033, + "rewards/margins": 0.050302695482969284, + "rewards/rejected": -1.365338683128357, + "semantic_entropy": 0.8078792691230774, + "step": 320 + }, + { + "epoch": 0.17394213079110218, + "grad_norm": 9.359341559750296, + "learning_rate": 5.793226381461676e-07, + "logits/chosen": -0.1586337685585022, + "logits/rejected": -0.054130129516124725, + "logps/chosen": -1.3173141479492188, + "logps/rejected": -1.5577419996261597, + "loss": 2.0609, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.3173141479492188, + "rewards/margins": 0.24042773246765137, + "rewards/rejected": -1.5577419996261597, + "semantic_entropy": 0.785629391670227, + "step": 325 + }, + { + "epoch": 0.17661816357250376, + "grad_norm": 9.90455638090829, + "learning_rate": 5.88235294117647e-07, + "logits/chosen": -0.03396131470799446, + "logits/rejected": 0.106153704226017, + "logps/chosen": -1.3295948505401611, + "logps/rejected": -1.5008153915405273, + "loss": 2.0891, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.3295948505401611, + "rewards/margins": 0.1712205708026886, + "rewards/rejected": -1.5008153915405273, + "semantic_entropy": 0.7894498705863953, + "step": 330 + }, + { + "epoch": 0.17929419635390534, + "grad_norm": 9.675886750939776, + "learning_rate": 5.971479500891266e-07, + "logits/chosen": 0.004796019289642572, + "logits/rejected": 0.10043302923440933, + "logps/chosen": -1.3419394493103027, + "logps/rejected": -1.375423789024353, + "loss": 2.1345, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -1.3419394493103027, + "rewards/margins": 0.03348435088992119, + "rewards/rejected": -1.375423789024353, + "semantic_entropy": 0.8081732988357544, + "step": 335 + }, + { + "epoch": 0.18197022913530692, + "grad_norm": 10.253171249347128, + "learning_rate": 6.060606060606061e-07, + "logits/chosen": -0.06982637941837311, + "logits/rejected": 0.06200215965509415, + "logps/chosen": -1.400742769241333, + "logps/rejected": -1.4839637279510498, + "loss": 2.1445, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.400742769241333, + "rewards/margins": 0.0832209438085556, + "rewards/rejected": -1.4839637279510498, + "semantic_entropy": 0.7617381811141968, + "step": 340 + }, + { + "epoch": 0.1846462619167085, + "grad_norm": 13.82680468025849, + "learning_rate": 6.149732620320855e-07, + "logits/chosen": 0.005692244973033667, + "logits/rejected": 0.03132449835538864, + "logps/chosen": -1.3126928806304932, + "logps/rejected": -1.452416181564331, + "loss": 2.0592, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.3126928806304932, + "rewards/margins": 0.13972336053848267, + "rewards/rejected": -1.452416181564331, + "semantic_entropy": 0.802249550819397, + "step": 345 + }, + { + "epoch": 0.18732229469811004, + "grad_norm": 9.737973493965074, + "learning_rate": 6.238859180035651e-07, + "logits/chosen": -0.025805365294218063, + "logits/rejected": 0.05890653654932976, + "logps/chosen": -1.2869572639465332, + "logps/rejected": -1.4053207635879517, + "loss": 2.0759, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.2869572639465332, + "rewards/margins": 0.11836358159780502, + "rewards/rejected": -1.4053207635879517, + "semantic_entropy": 0.8062549829483032, + "step": 350 + }, + { + "epoch": 0.18999832747951162, + "grad_norm": 11.011017446465962, + "learning_rate": 6.327985739750445e-07, + "logits/chosen": -0.08332131057977676, + "logits/rejected": 0.1384621560573578, + "logps/chosen": -1.3908402919769287, + "logps/rejected": -1.417327880859375, + "loss": 2.1468, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -1.3908402919769287, + "rewards/margins": 0.02648766338825226, + "rewards/rejected": -1.417327880859375, + "semantic_entropy": 0.7860895395278931, + "step": 355 + }, + { + "epoch": 0.1926743602609132, + "grad_norm": 9.3029388405961, + "learning_rate": 6.417112299465241e-07, + "logits/chosen": -0.0865727886557579, + "logits/rejected": -0.012034505605697632, + "logps/chosen": -1.3202612400054932, + "logps/rejected": -1.440613031387329, + "loss": 2.1384, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.3202612400054932, + "rewards/margins": 0.12035173177719116, + "rewards/rejected": -1.440613031387329, + "semantic_entropy": 0.8006309270858765, + "step": 360 + }, + { + "epoch": 0.19535039304231477, + "grad_norm": 11.887096782670497, + "learning_rate": 6.506238859180035e-07, + "logits/chosen": -0.022515257820487022, + "logits/rejected": 0.05280578136444092, + "logps/chosen": -1.2991836071014404, + "logps/rejected": -1.399091124534607, + "loss": 2.0964, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2991836071014404, + "rewards/margins": 0.09990767389535904, + "rewards/rejected": -1.399091124534607, + "semantic_entropy": 0.8175728917121887, + "step": 365 + }, + { + "epoch": 0.19802642582371635, + "grad_norm": 7.41980630149422, + "learning_rate": 6.59536541889483e-07, + "logits/chosen": -0.03407667204737663, + "logits/rejected": 0.056151580065488815, + "logps/chosen": -1.294122338294983, + "logps/rejected": -1.3374214172363281, + "loss": 2.1084, + "rewards/accuracies": 0.5062500238418579, + "rewards/chosen": -1.294122338294983, + "rewards/margins": 0.04329918324947357, + "rewards/rejected": -1.3374214172363281, + "semantic_entropy": 0.8323699235916138, + "step": 370 + }, + { + "epoch": 0.2007024586051179, + "grad_norm": 15.730415357023626, + "learning_rate": 6.684491978609626e-07, + "logits/chosen": -0.08421232551336288, + "logits/rejected": 0.06763456016778946, + "logps/chosen": -1.2762327194213867, + "logps/rejected": -1.4177181720733643, + "loss": 2.08, + "rewards/accuracies": 0.53125, + "rewards/chosen": -1.2762327194213867, + "rewards/margins": 0.14148563146591187, + "rewards/rejected": -1.4177181720733643, + "semantic_entropy": 0.8105593919754028, + "step": 375 + }, + { + "epoch": 0.20337849138651948, + "grad_norm": 9.693161959090537, + "learning_rate": 6.77361853832442e-07, + "logits/chosen": -0.05102665349841118, + "logits/rejected": 0.027332540601491928, + "logps/chosen": -1.2896788120269775, + "logps/rejected": -1.4495028257369995, + "loss": 2.0546, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2896788120269775, + "rewards/margins": 0.1598241627216339, + "rewards/rejected": -1.4495028257369995, + "semantic_entropy": 0.8106985092163086, + "step": 380 + }, + { + "epoch": 0.20605452416792105, + "grad_norm": 5.182351383221804, + "learning_rate": 6.862745098039216e-07, + "logits/chosen": -0.036210425198078156, + "logits/rejected": 0.03844881430268288, + "logps/chosen": -1.3870257139205933, + "logps/rejected": -1.3795428276062012, + "loss": 2.1431, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.3870257139205933, + "rewards/margins": -0.007482861168682575, + "rewards/rejected": -1.3795428276062012, + "semantic_entropy": 0.792412519454956, + "step": 385 + }, + { + "epoch": 0.20873055694932263, + "grad_norm": 9.966131209763185, + "learning_rate": 6.95187165775401e-07, + "logits/chosen": 0.016994182020425797, + "logits/rejected": 0.1718176305294037, + "logps/chosen": -1.3804776668548584, + "logps/rejected": -1.4428892135620117, + "loss": 2.1521, + "rewards/accuracies": 0.46875, + "rewards/chosen": -1.3804776668548584, + "rewards/margins": 0.0624115951359272, + "rewards/rejected": -1.4428892135620117, + "semantic_entropy": 0.7761930823326111, + "step": 390 + }, + { + "epoch": 0.2114065897307242, + "grad_norm": 7.097965730394271, + "learning_rate": 7.040998217468806e-07, + "logits/chosen": -0.080187126994133, + "logits/rejected": 0.07204331457614899, + "logps/chosen": -1.3347288370132446, + "logps/rejected": -1.3476696014404297, + "loss": 2.1163, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.3347288370132446, + "rewards/margins": 0.01294087152928114, + "rewards/rejected": -1.3476696014404297, + "semantic_entropy": 0.8120431900024414, + "step": 395 + }, + { + "epoch": 0.2140826225121258, + "grad_norm": 8.470660073637783, + "learning_rate": 7.1301247771836e-07, + "logits/chosen": 0.05173783749341965, + "logits/rejected": 0.14466162025928497, + "logps/chosen": -1.3133375644683838, + "logps/rejected": -1.4095127582550049, + "loss": 2.0607, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.3133375644683838, + "rewards/margins": 0.09617514908313751, + "rewards/rejected": -1.4095127582550049, + "semantic_entropy": 0.804615318775177, + "step": 400 + }, + { + "epoch": 0.2140826225121258, + "eval_logits/chosen": 0.24520687758922577, + "eval_logits/rejected": 0.32969510555267334, + "eval_logps/chosen": -1.345367193222046, + "eval_logps/rejected": -1.475685954093933, + "eval_loss": 2.0924832820892334, + "eval_rewards/accuracies": 0.5563797950744629, + "eval_rewards/chosen": -1.345367193222046, + "eval_rewards/margins": 0.1303185522556305, + "eval_rewards/rejected": -1.475685954093933, + "eval_runtime": 35.1527, + "eval_samples_per_second": 38.262, + "eval_semantic_entropy": 0.7909919619560242, + "eval_steps_per_second": 9.587, + "step": 400 + }, + { + "epoch": 0.21675865529352734, + "grad_norm": 9.553550584241732, + "learning_rate": 7.219251336898395e-07, + "logits/chosen": -0.039488695561885834, + "logits/rejected": 0.051676005125045776, + "logps/chosen": -1.319966197013855, + "logps/rejected": -1.3851897716522217, + "loss": 2.1136, + "rewards/accuracies": 0.4937500059604645, + "rewards/chosen": -1.319966197013855, + "rewards/margins": 0.06522355228662491, + "rewards/rejected": -1.3851897716522217, + "semantic_entropy": 0.8008989095687866, + "step": 405 + }, + { + "epoch": 0.2194346880749289, + "grad_norm": 12.81546563496579, + "learning_rate": 7.30837789661319e-07, + "logits/chosen": -0.004357346799224615, + "logits/rejected": 0.12372720241546631, + "logps/chosen": -1.292946457862854, + "logps/rejected": -1.3807947635650635, + "loss": 2.0843, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -1.292946457862854, + "rewards/margins": 0.08784836530685425, + "rewards/rejected": -1.3807947635650635, + "semantic_entropy": 0.8132365942001343, + "step": 410 + }, + { + "epoch": 0.2221107208563305, + "grad_norm": 5.543288881603541, + "learning_rate": 7.397504456327985e-07, + "logits/chosen": -0.02848835289478302, + "logits/rejected": 0.003672828432172537, + "logps/chosen": -1.2885775566101074, + "logps/rejected": -1.446232557296753, + "loss": 2.0468, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.2885775566101074, + "rewards/margins": 0.15765497088432312, + "rewards/rejected": -1.446232557296753, + "semantic_entropy": 0.8051525354385376, + "step": 415 + }, + { + "epoch": 0.22478675363773207, + "grad_norm": 7.998047741738567, + "learning_rate": 7.486631016042781e-07, + "logits/chosen": -0.023648854345083237, + "logits/rejected": 0.16269460320472717, + "logps/chosen": -1.2729085683822632, + "logps/rejected": -1.3676693439483643, + "loss": 2.0836, + "rewards/accuracies": 0.4937500059604645, + "rewards/chosen": -1.2729085683822632, + "rewards/margins": 0.09476063400506973, + "rewards/rejected": -1.3676693439483643, + "semantic_entropy": 0.8268201947212219, + "step": 420 + }, + { + "epoch": 0.22746278641913364, + "grad_norm": 6.633391520285902, + "learning_rate": 7.575757575757575e-07, + "logits/chosen": -0.042364347726106644, + "logits/rejected": 0.1575302630662918, + "logps/chosen": -1.3138656616210938, + "logps/rejected": -1.4691503047943115, + "loss": 2.0663, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.3138656616210938, + "rewards/margins": 0.15528468787670135, + "rewards/rejected": -1.4691503047943115, + "semantic_entropy": 0.8047497868537903, + "step": 425 + }, + { + "epoch": 0.2301388192005352, + "grad_norm": 7.516999022167481, + "learning_rate": 7.664884135472371e-07, + "logits/chosen": -0.08585725724697113, + "logits/rejected": 0.1032024621963501, + "logps/chosen": -1.3429131507873535, + "logps/rejected": -1.4755594730377197, + "loss": 2.0833, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.3429131507873535, + "rewards/margins": 0.13264614343643188, + "rewards/rejected": -1.4755594730377197, + "semantic_entropy": 0.7835851907730103, + "step": 430 + }, + { + "epoch": 0.23281485198193677, + "grad_norm": 10.217305267561882, + "learning_rate": 7.754010695187165e-07, + "logits/chosen": -0.03247809037566185, + "logits/rejected": 0.05013108253479004, + "logps/chosen": -1.215358018875122, + "logps/rejected": -1.347867727279663, + "loss": 2.0371, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.215358018875122, + "rewards/margins": 0.1325097382068634, + "rewards/rejected": -1.347867727279663, + "semantic_entropy": 0.8345514535903931, + "step": 435 + }, + { + "epoch": 0.23549088476333835, + "grad_norm": 7.388443140413564, + "learning_rate": 7.84313725490196e-07, + "logits/chosen": -0.022949252277612686, + "logits/rejected": 0.061340272426605225, + "logps/chosen": -1.2916299104690552, + "logps/rejected": -1.3803313970565796, + "loss": 2.087, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -1.2916299104690552, + "rewards/margins": 0.08870140463113785, + "rewards/rejected": -1.3803313970565796, + "semantic_entropy": 0.8170446157455444, + "step": 440 + }, + { + "epoch": 0.23816691754473993, + "grad_norm": 8.493171472969088, + "learning_rate": 7.932263814616755e-07, + "logits/chosen": -0.045423250645399094, + "logits/rejected": 0.062299858778715134, + "logps/chosen": -1.318433403968811, + "logps/rejected": -1.4452886581420898, + "loss": 2.1018, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -1.318433403968811, + "rewards/margins": 0.12685513496398926, + "rewards/rejected": -1.4452886581420898, + "semantic_entropy": 0.7928746938705444, + "step": 445 + }, + { + "epoch": 0.2408429503261415, + "grad_norm": 7.634251232185677, + "learning_rate": 8.02139037433155e-07, + "logits/chosen": -0.004715797491371632, + "logits/rejected": 0.11216776072978973, + "logps/chosen": -1.3025964498519897, + "logps/rejected": -1.4417173862457275, + "loss": 2.064, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.3025964498519897, + "rewards/margins": 0.13912081718444824, + "rewards/rejected": -1.4417173862457275, + "semantic_entropy": 0.8090659379959106, + "step": 450 + }, + { + "epoch": 0.24351898310754308, + "grad_norm": 9.123904294480766, + "learning_rate": 8.110516934046346e-07, + "logits/chosen": -0.02629506029188633, + "logits/rejected": 0.058896519243717194, + "logps/chosen": -1.2463464736938477, + "logps/rejected": -1.4314829111099243, + "loss": 2.0277, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2463464736938477, + "rewards/margins": 0.18513646721839905, + "rewards/rejected": -1.4314829111099243, + "semantic_entropy": 0.819383978843689, + "step": 455 + }, + { + "epoch": 0.24619501588894463, + "grad_norm": 7.988610941036874, + "learning_rate": 8.19964349376114e-07, + "logits/chosen": -0.13996991515159607, + "logits/rejected": -0.02202531509101391, + "logps/chosen": -1.373464822769165, + "logps/rejected": -1.4234976768493652, + "loss": 2.134, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.373464822769165, + "rewards/margins": 0.05003304407000542, + "rewards/rejected": -1.4234976768493652, + "semantic_entropy": 0.7923992872238159, + "step": 460 + }, + { + "epoch": 0.2488710486703462, + "grad_norm": 9.581479715947285, + "learning_rate": 8.288770053475936e-07, + "logits/chosen": 0.11567505449056625, + "logits/rejected": 0.1285204142332077, + "logps/chosen": -1.2783681154251099, + "logps/rejected": -1.4448531866073608, + "loss": 2.0768, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.2783681154251099, + "rewards/margins": 0.1664850413799286, + "rewards/rejected": -1.4448531866073608, + "semantic_entropy": 0.801103949546814, + "step": 465 + }, + { + "epoch": 0.2515470814517478, + "grad_norm": 9.216483509650814, + "learning_rate": 8.37789661319073e-07, + "logits/chosen": 0.15024113655090332, + "logits/rejected": 0.09940201789140701, + "logps/chosen": -1.239457130432129, + "logps/rejected": -1.4274377822875977, + "loss": 2.014, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.239457130432129, + "rewards/margins": 0.18798059225082397, + "rewards/rejected": -1.4274377822875977, + "semantic_entropy": 0.8202102780342102, + "step": 470 + }, + { + "epoch": 0.25422311423314936, + "grad_norm": 7.406586737551856, + "learning_rate": 8.467023172905525e-07, + "logits/chosen": -0.0853961706161499, + "logits/rejected": 0.046978384256362915, + "logps/chosen": -1.304258108139038, + "logps/rejected": -1.5052478313446045, + "loss": 2.0343, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.304258108139038, + "rewards/margins": 0.20098969340324402, + "rewards/rejected": -1.5052478313446045, + "semantic_entropy": 0.7981175184249878, + "step": 475 + }, + { + "epoch": 0.2568991470145509, + "grad_norm": 12.42290439017345, + "learning_rate": 8.55614973262032e-07, + "logits/chosen": -0.06797462701797485, + "logits/rejected": 0.12427964061498642, + "logps/chosen": -1.2787773609161377, + "logps/rejected": -1.3450984954833984, + "loss": 2.1266, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.2787773609161377, + "rewards/margins": 0.06632117927074432, + "rewards/rejected": -1.3450984954833984, + "semantic_entropy": 0.8253456950187683, + "step": 480 + }, + { + "epoch": 0.2595751797959525, + "grad_norm": 12.319073676190952, + "learning_rate": 8.645276292335115e-07, + "logits/chosen": -0.02665388025343418, + "logits/rejected": 0.007270003668963909, + "logps/chosen": -1.372032880783081, + "logps/rejected": -1.44978928565979, + "loss": 2.1258, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -1.372032880783081, + "rewards/margins": 0.07775656878948212, + "rewards/rejected": -1.44978928565979, + "semantic_entropy": 0.7805415391921997, + "step": 485 + }, + { + "epoch": 0.26225121257735406, + "grad_norm": 8.267686997372833, + "learning_rate": 8.734402852049911e-07, + "logits/chosen": -0.0180144514888525, + "logits/rejected": 0.04987755045294762, + "logps/chosen": -1.3115794658660889, + "logps/rejected": -1.3869727849960327, + "loss": 2.1172, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.3115794658660889, + "rewards/margins": 0.07539341598749161, + "rewards/rejected": -1.3869727849960327, + "semantic_entropy": 0.81377112865448, + "step": 490 + }, + { + "epoch": 0.26492724535875567, + "grad_norm": 9.738357137083968, + "learning_rate": 8.823529411764705e-07, + "logits/chosen": -0.058453939855098724, + "logits/rejected": -0.04326649382710457, + "logps/chosen": -1.3211174011230469, + "logps/rejected": -1.4324402809143066, + "loss": 2.095, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -1.3211174011230469, + "rewards/margins": 0.1113228052854538, + "rewards/rejected": -1.4324402809143066, + "semantic_entropy": 0.8082423210144043, + "step": 495 + }, + { + "epoch": 0.2676032781401572, + "grad_norm": 8.372403169674499, + "learning_rate": 8.912655971479501e-07, + "logits/chosen": -0.06073238328099251, + "logits/rejected": 0.03806382417678833, + "logps/chosen": -1.234489917755127, + "logps/rejected": -1.3935258388519287, + "loss": 2.0403, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.234489917755127, + "rewards/margins": 0.15903595089912415, + "rewards/rejected": -1.3935258388519287, + "semantic_entropy": 0.8348814249038696, + "step": 500 + }, + { + "epoch": 0.27027931092155877, + "grad_norm": 9.329346348003517, + "learning_rate": 9.001782531194295e-07, + "logits/chosen": -0.0702035129070282, + "logits/rejected": 0.06675246357917786, + "logps/chosen": -1.3525861501693726, + "logps/rejected": -1.3896262645721436, + "loss": 2.0984, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -1.3525861501693726, + "rewards/margins": 0.03703995421528816, + "rewards/rejected": -1.3896262645721436, + "semantic_entropy": 0.7991023063659668, + "step": 505 + }, + { + "epoch": 0.2729553437029604, + "grad_norm": 7.793041739139555, + "learning_rate": 9.09090909090909e-07, + "logits/chosen": 0.0727655366063118, + "logits/rejected": 0.13345302641391754, + "logps/chosen": -1.3091163635253906, + "logps/rejected": -1.4701882600784302, + "loss": 2.0514, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.3091163635253906, + "rewards/margins": 0.1610718071460724, + "rewards/rejected": -1.4701882600784302, + "semantic_entropy": 0.7971314787864685, + "step": 510 + }, + { + "epoch": 0.2756313764843619, + "grad_norm": 8.366167229049527, + "learning_rate": 9.180035650623885e-07, + "logits/chosen": 0.020826030522584915, + "logits/rejected": 0.11258859932422638, + "logps/chosen": -1.251171350479126, + "logps/rejected": -1.3965065479278564, + "loss": 2.0153, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.251171350479126, + "rewards/margins": 0.1453351080417633, + "rewards/rejected": -1.3965065479278564, + "semantic_entropy": 0.8115334510803223, + "step": 515 + }, + { + "epoch": 0.27830740926576353, + "grad_norm": 6.708105392677698, + "learning_rate": 9.26916221033868e-07, + "logits/chosen": -0.10605648905038834, + "logits/rejected": 0.02459835447371006, + "logps/chosen": -1.2906177043914795, + "logps/rejected": -1.3887439966201782, + "loss": 2.0989, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.2906177043914795, + "rewards/margins": 0.09812645614147186, + "rewards/rejected": -1.3887439966201782, + "semantic_entropy": 0.8078464269638062, + "step": 520 + }, + { + "epoch": 0.2809834420471651, + "grad_norm": 18.59523614539976, + "learning_rate": 9.358288770053476e-07, + "logits/chosen": 0.09513764083385468, + "logits/rejected": 0.16091565787792206, + "logps/chosen": -1.2693884372711182, + "logps/rejected": -1.4406367540359497, + "loss": 2.049, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.2693884372711182, + "rewards/margins": 0.1712484359741211, + "rewards/rejected": -1.4406367540359497, + "semantic_entropy": 0.8163102269172668, + "step": 525 + }, + { + "epoch": 0.2836594748285666, + "grad_norm": 6.400397273187949, + "learning_rate": 9.44741532976827e-07, + "logits/chosen": 0.05827958509325981, + "logits/rejected": 0.14098814129829407, + "logps/chosen": -1.2453418970108032, + "logps/rejected": -1.3420130014419556, + "loss": 2.0805, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.2453418970108032, + "rewards/margins": 0.09667106717824936, + "rewards/rejected": -1.3420130014419556, + "semantic_entropy": 0.8379393815994263, + "step": 530 + }, + { + "epoch": 0.28633550760996823, + "grad_norm": 6.655980461093334, + "learning_rate": 9.536541889483066e-07, + "logits/chosen": -0.10052146762609482, + "logits/rejected": 0.15529409050941467, + "logps/chosen": -1.2592411041259766, + "logps/rejected": -1.3248337507247925, + "loss": 2.0783, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.2592411041259766, + "rewards/margins": 0.06559250503778458, + "rewards/rejected": -1.3248337507247925, + "semantic_entropy": 0.8288901448249817, + "step": 535 + }, + { + "epoch": 0.2890115403913698, + "grad_norm": 8.028377167353824, + "learning_rate": 9.62566844919786e-07, + "logits/chosen": 0.03284211829304695, + "logits/rejected": 0.10378841310739517, + "logps/chosen": -1.3858778476715088, + "logps/rejected": -1.4422986507415771, + "loss": 2.1258, + "rewards/accuracies": 0.53125, + "rewards/chosen": -1.3858778476715088, + "rewards/margins": 0.056420810520648956, + "rewards/rejected": -1.4422986507415771, + "semantic_entropy": 0.7777124643325806, + "step": 540 + }, + { + "epoch": 0.2916875731727714, + "grad_norm": 7.5790293844104895, + "learning_rate": 9.714795008912655e-07, + "logits/chosen": -0.08762449026107788, + "logits/rejected": 0.1115502342581749, + "logps/chosen": -1.2886834144592285, + "logps/rejected": -1.4010090827941895, + "loss": 2.0677, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.2886834144592285, + "rewards/margins": 0.11232553422451019, + "rewards/rejected": -1.4010090827941895, + "semantic_entropy": 0.8187274932861328, + "step": 545 + }, + { + "epoch": 0.29436360595417294, + "grad_norm": 8.962981954490106, + "learning_rate": 9.80392156862745e-07, + "logits/chosen": 0.045073509216308594, + "logits/rejected": 0.11081943660974503, + "logps/chosen": -1.2883306741714478, + "logps/rejected": -1.4169273376464844, + "loss": 2.0432, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.2883306741714478, + "rewards/margins": 0.12859681248664856, + "rewards/rejected": -1.4169273376464844, + "semantic_entropy": 0.8060620427131653, + "step": 550 + }, + { + "epoch": 0.2970396387355745, + "grad_norm": 11.47050872584094, + "learning_rate": 9.893048128342244e-07, + "logits/chosen": -0.07750057429075241, + "logits/rejected": 0.04340776801109314, + "logps/chosen": -1.3622779846191406, + "logps/rejected": -1.4258826971054077, + "loss": 2.1151, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.3622779846191406, + "rewards/margins": 0.06360460817813873, + "rewards/rejected": -1.4258826971054077, + "semantic_entropy": 0.7788089513778687, + "step": 555 + }, + { + "epoch": 0.2997156715169761, + "grad_norm": 10.029435787937103, + "learning_rate": 9.98217468805704e-07, + "logits/chosen": 0.05368703603744507, + "logits/rejected": 0.0629192367196083, + "logps/chosen": -1.2183544635772705, + "logps/rejected": -1.3473310470581055, + "loss": 2.0256, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.2183544635772705, + "rewards/margins": 0.12897667288780212, + "rewards/rejected": -1.3473310470581055, + "semantic_entropy": 0.8247224688529968, + "step": 560 + }, + { + "epoch": 0.30239170429837764, + "grad_norm": 5.518574576054694, + "learning_rate": 9.999984476788462e-07, + "logits/chosen": 0.01707286201417446, + "logits/rejected": 0.06581038981676102, + "logps/chosen": -1.332328200340271, + "logps/rejected": -1.4508365392684937, + "loss": 2.1007, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.332328200340271, + "rewards/margins": 0.11850825697183609, + "rewards/rejected": -1.4508365392684937, + "semantic_entropy": 0.7935990691184998, + "step": 565 + }, + { + "epoch": 0.30506773707977924, + "grad_norm": 8.87344715447719, + "learning_rate": 9.999921413906797e-07, + "logits/chosen": -0.043978843837976456, + "logits/rejected": 0.16928060352802277, + "logps/chosen": -1.3080699443817139, + "logps/rejected": -1.3878307342529297, + "loss": 2.0981, + "rewards/accuracies": 0.4937500059604645, + "rewards/chosen": -1.3080699443817139, + "rewards/margins": 0.07976074516773224, + "rewards/rejected": -1.3878307342529297, + "semantic_entropy": 0.7911024689674377, + "step": 570 + }, + { + "epoch": 0.3077437698611808, + "grad_norm": 6.60439384217086, + "learning_rate": 9.999809841765644e-07, + "logits/chosen": -0.004473379347473383, + "logits/rejected": 0.052548039704561234, + "logps/chosen": -1.2383952140808105, + "logps/rejected": -1.35186767578125, + "loss": 2.0731, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.2383952140808105, + "rewards/margins": 0.11347250640392303, + "rewards/rejected": -1.35186767578125, + "semantic_entropy": 0.8186966180801392, + "step": 575 + }, + { + "epoch": 0.3104198026425824, + "grad_norm": 7.041622382443657, + "learning_rate": 9.999649761447477e-07, + "logits/chosen": -0.039054978638887405, + "logits/rejected": 0.11595281213521957, + "logps/chosen": -1.2413501739501953, + "logps/rejected": -1.4057656526565552, + "loss": 2.0201, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.2413501739501953, + "rewards/margins": 0.16441559791564941, + "rewards/rejected": -1.4057656526565552, + "semantic_entropy": 0.829367995262146, + "step": 580 + }, + { + "epoch": 0.31309583542398395, + "grad_norm": 7.080204952027214, + "learning_rate": 9.999441174505398e-07, + "logits/chosen": -0.07520152628421783, + "logits/rejected": 0.022036874666810036, + "logps/chosen": -1.378169298171997, + "logps/rejected": -1.4294803142547607, + "loss": 2.1316, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.378169298171997, + "rewards/margins": 0.05131111294031143, + "rewards/rejected": -1.4294803142547607, + "semantic_entropy": 0.7770919799804688, + "step": 585 + }, + { + "epoch": 0.3157718682053855, + "grad_norm": 17.319598964810336, + "learning_rate": 9.999184082963116e-07, + "logits/chosen": -0.0469600148499012, + "logits/rejected": 0.07453016191720963, + "logps/chosen": -1.3526276350021362, + "logps/rejected": -1.3841309547424316, + "loss": 2.1237, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.3526276350021362, + "rewards/margins": 0.031503356993198395, + "rewards/rejected": -1.3841309547424316, + "semantic_entropy": 0.7946040034294128, + "step": 590 + }, + { + "epoch": 0.3184479009867871, + "grad_norm": 9.00235073415317, + "learning_rate": 9.998878489314937e-07, + "logits/chosen": 0.003964888397604227, + "logits/rejected": 0.12411659955978394, + "logps/chosen": -1.2833412885665894, + "logps/rejected": -1.350610613822937, + "loss": 2.0957, + "rewards/accuracies": 0.46875, + "rewards/chosen": -1.2833412885665894, + "rewards/margins": 0.06726942956447601, + "rewards/rejected": -1.350610613822937, + "semantic_entropy": 0.8191946148872375, + "step": 595 + }, + { + "epoch": 0.32112393376818865, + "grad_norm": 7.922144554064656, + "learning_rate": 9.99852439652573e-07, + "logits/chosen": -0.051727332174777985, + "logits/rejected": 0.08876247704029083, + "logps/chosen": -1.2790896892547607, + "logps/rejected": -1.3022220134735107, + "loss": 2.1024, + "rewards/accuracies": 0.4625000059604645, + "rewards/chosen": -1.2790896892547607, + "rewards/margins": 0.023132145404815674, + "rewards/rejected": -1.3022220134735107, + "semantic_entropy": 0.8291428685188293, + "step": 600 + }, + { + "epoch": 0.32379996654959026, + "grad_norm": 9.965995704685826, + "learning_rate": 9.998121808030904e-07, + "logits/chosen": -0.09909897297620773, + "logits/rejected": -0.017181608825922012, + "logps/chosen": -1.3383922576904297, + "logps/rejected": -1.487484335899353, + "loss": 2.0822, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.3383922576904297, + "rewards/margins": 0.1490919589996338, + "rewards/rejected": -1.487484335899353, + "semantic_entropy": 0.7997158765792847, + "step": 605 + }, + { + "epoch": 0.3264759993309918, + "grad_norm": 12.670537832160065, + "learning_rate": 9.997670727736379e-07, + "logits/chosen": 0.044879984110593796, + "logits/rejected": 0.19142109155654907, + "logps/chosen": -1.3152719736099243, + "logps/rejected": -1.384782314300537, + "loss": 2.1013, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.3152719736099243, + "rewards/margins": 0.06951036304235458, + "rewards/rejected": -1.384782314300537, + "semantic_entropy": 0.7921839952468872, + "step": 610 + }, + { + "epoch": 0.32915203211239336, + "grad_norm": 5.2822164816811235, + "learning_rate": 9.99717116001853e-07, + "logits/chosen": -0.03817535936832428, + "logits/rejected": 0.06552287191152573, + "logps/chosen": -1.313217282295227, + "logps/rejected": -1.4523591995239258, + "loss": 2.0817, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.313217282295227, + "rewards/margins": 0.13914184272289276, + "rewards/rejected": -1.4523591995239258, + "semantic_entropy": 0.8057335615158081, + "step": 615 + }, + { + "epoch": 0.33182806489379496, + "grad_norm": 7.158263688307781, + "learning_rate": 9.996623109724173e-07, + "logits/chosen": 0.05255531147122383, + "logits/rejected": 0.1168251484632492, + "logps/chosen": -1.3770633935928345, + "logps/rejected": -1.4801380634307861, + "loss": 2.0993, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.3770633935928345, + "rewards/margins": 0.10307463258504868, + "rewards/rejected": -1.4801380634307861, + "semantic_entropy": 0.7853333353996277, + "step": 620 + }, + { + "epoch": 0.3345040976751965, + "grad_norm": 9.250643921051761, + "learning_rate": 9.996026582170488e-07, + "logits/chosen": 0.060080431401729584, + "logits/rejected": 0.17307524383068085, + "logps/chosen": -1.2922842502593994, + "logps/rejected": -1.4193775653839111, + "loss": 2.0662, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.2922842502593994, + "rewards/margins": 0.12709322571754456, + "rewards/rejected": -1.4193775653839111, + "semantic_entropy": 0.809944748878479, + "step": 625 + }, + { + "epoch": 0.3371801304565981, + "grad_norm": 7.144016418620915, + "learning_rate": 9.995381583144996e-07, + "logits/chosen": -0.03415603190660477, + "logits/rejected": 0.06720946729183197, + "logps/chosen": -1.3283525705337524, + "logps/rejected": -1.4664812088012695, + "loss": 2.0554, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.3283525705337524, + "rewards/margins": 0.1381286084651947, + "rewards/rejected": -1.4664812088012695, + "semantic_entropy": 0.7949910759925842, + "step": 630 + }, + { + "epoch": 0.33985616323799966, + "grad_norm": 6.276400394476788, + "learning_rate": 9.994688118905471e-07, + "logits/chosen": -0.024661913514137268, + "logits/rejected": 0.20788328349590302, + "logps/chosen": -1.3947147130966187, + "logps/rejected": -1.4336068630218506, + "loss": 2.1615, + "rewards/accuracies": 0.45625001192092896, + "rewards/chosen": -1.3947147130966187, + "rewards/margins": 0.03889207914471626, + "rewards/rejected": -1.4336068630218506, + "semantic_entropy": 0.7786458730697632, + "step": 635 + }, + { + "epoch": 0.3425321960194012, + "grad_norm": 9.572705692674822, + "learning_rate": 9.993946196179912e-07, + "logits/chosen": -0.08139273524284363, + "logits/rejected": 0.1119336485862732, + "logps/chosen": -1.3305104970932007, + "logps/rejected": -1.4301774501800537, + "loss": 2.129, + "rewards/accuracies": 0.46875, + "rewards/chosen": -1.3305104970932007, + "rewards/margins": 0.09966699033975601, + "rewards/rejected": -1.4301774501800537, + "semantic_entropy": 0.7754771709442139, + "step": 640 + }, + { + "epoch": 0.3452082288008028, + "grad_norm": 6.77104258950228, + "learning_rate": 9.993155822166455e-07, + "logits/chosen": -0.08386670053005219, + "logits/rejected": 0.001519903540611267, + "logps/chosen": -1.245597004890442, + "logps/rejected": -1.409886360168457, + "loss": 2.0635, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.245597004890442, + "rewards/margins": 0.16428932547569275, + "rewards/rejected": -1.409886360168457, + "semantic_entropy": 0.8197149038314819, + "step": 645 + }, + { + "epoch": 0.34788426158220437, + "grad_norm": 10.81121162736983, + "learning_rate": 9.992317004533313e-07, + "logits/chosen": -0.026128700003027916, + "logits/rejected": 0.11188068240880966, + "logps/chosen": -1.3841702938079834, + "logps/rejected": -1.5400774478912354, + "loss": 2.1066, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.3841702938079834, + "rewards/margins": 0.15590718388557434, + "rewards/rejected": -1.5400774478912354, + "semantic_entropy": 0.7806428670883179, + "step": 650 + }, + { + "epoch": 0.350560294363606, + "grad_norm": 7.821570622584799, + "learning_rate": 9.991429751418696e-07, + "logits/chosen": 0.04313744977116585, + "logits/rejected": 0.04878374561667442, + "logps/chosen": -1.3229328393936157, + "logps/rejected": -1.5045139789581299, + "loss": 2.0712, + "rewards/accuracies": 0.53125, + "rewards/chosen": -1.3229328393936157, + "rewards/margins": 0.18158110976219177, + "rewards/rejected": -1.5045139789581299, + "semantic_entropy": 0.7957868576049805, + "step": 655 + }, + { + "epoch": 0.3532363271450075, + "grad_norm": 7.47285160939844, + "learning_rate": 9.99049407143074e-07, + "logits/chosen": -0.0009026184561662376, + "logits/rejected": 0.12640218436717987, + "logps/chosen": -1.2817261219024658, + "logps/rejected": -1.3218984603881836, + "loss": 2.0897, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -1.2817261219024658, + "rewards/margins": 0.04017230123281479, + "rewards/rejected": -1.3218984603881836, + "semantic_entropy": 0.8074017763137817, + "step": 660 + }, + { + "epoch": 0.35591235992640907, + "grad_norm": 8.036948339099359, + "learning_rate": 9.989509973647416e-07, + "logits/chosen": 0.004626098088920116, + "logits/rejected": 0.14154979586601257, + "logps/chosen": -1.245429277420044, + "logps/rejected": -1.3789770603179932, + "loss": 2.0455, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.245429277420044, + "rewards/margins": 0.13354769349098206, + "rewards/rejected": -1.3789770603179932, + "semantic_entropy": 0.8267159461975098, + "step": 665 + }, + { + "epoch": 0.3585883927078107, + "grad_norm": 8.52772335856538, + "learning_rate": 9.988477467616445e-07, + "logits/chosen": -0.016855059191584587, + "logits/rejected": 0.1922694444656372, + "logps/chosen": -1.2839257717132568, + "logps/rejected": -1.335697889328003, + "loss": 2.0814, + "rewards/accuracies": 0.4625000059604645, + "rewards/chosen": -1.2839257717132568, + "rewards/margins": 0.05177216976881027, + "rewards/rejected": -1.335697889328003, + "semantic_entropy": 0.8241464495658875, + "step": 670 + }, + { + "epoch": 0.3612644254892122, + "grad_norm": 9.71077016531089, + "learning_rate": 9.987396563355205e-07, + "logits/chosen": -0.027623683214187622, + "logits/rejected": 0.05166678503155708, + "logps/chosen": -1.275935411453247, + "logps/rejected": -1.4907596111297607, + "loss": 2.0427, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.275935411453247, + "rewards/margins": 0.21482422947883606, + "rewards/rejected": -1.4907596111297607, + "semantic_entropy": 0.8094412088394165, + "step": 675 + }, + { + "epoch": 0.36394045827061383, + "grad_norm": 9.612038280767669, + "learning_rate": 9.986267271350631e-07, + "logits/chosen": 0.05635018274188042, + "logits/rejected": 0.21157081425189972, + "logps/chosen": -1.3226648569107056, + "logps/rejected": -1.3817113637924194, + "loss": 2.1285, + "rewards/accuracies": 0.4937500059604645, + "rewards/chosen": -1.3226648569107056, + "rewards/margins": 0.05904661491513252, + "rewards/rejected": -1.3817113637924194, + "semantic_entropy": 0.8033465147018433, + "step": 680 + }, + { + "epoch": 0.3666164910520154, + "grad_norm": 8.664165560897748, + "learning_rate": 9.985089602559123e-07, + "logits/chosen": 0.022344741970300674, + "logits/rejected": 0.1773933470249176, + "logps/chosen": -1.300033450126648, + "logps/rejected": -1.3612678050994873, + "loss": 2.112, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.300033450126648, + "rewards/margins": 0.06123412400484085, + "rewards/rejected": -1.3612678050994873, + "semantic_entropy": 0.8237413167953491, + "step": 685 + }, + { + "epoch": 0.369292523833417, + "grad_norm": 9.941705389220667, + "learning_rate": 9.983863568406428e-07, + "logits/chosen": 0.05462765693664551, + "logits/rejected": 0.08798164129257202, + "logps/chosen": -1.3029800653457642, + "logps/rejected": -1.4281790256500244, + "loss": 2.0599, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.3029800653457642, + "rewards/margins": 0.12519893050193787, + "rewards/rejected": -1.4281790256500244, + "semantic_entropy": 0.7990589141845703, + "step": 690 + }, + { + "epoch": 0.37196855661481854, + "grad_norm": 7.327915372973922, + "learning_rate": 9.982589180787532e-07, + "logits/chosen": 0.003890749765560031, + "logits/rejected": 0.0950247123837471, + "logps/chosen": -1.1939213275909424, + "logps/rejected": -1.3700560331344604, + "loss": 1.9896, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.1939213275909424, + "rewards/margins": 0.1761348396539688, + "rewards/rejected": -1.3700560331344604, + "semantic_entropy": 0.8320623636245728, + "step": 695 + }, + { + "epoch": 0.3746445893962201, + "grad_norm": 6.387595285459338, + "learning_rate": 9.981266452066553e-07, + "logits/chosen": -0.1137634664773941, + "logits/rejected": 0.020167222246527672, + "logps/chosen": -1.3625977039337158, + "logps/rejected": -1.4447782039642334, + "loss": 2.1061, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -1.3625977039337158, + "rewards/margins": 0.082180455327034, + "rewards/rejected": -1.4447782039642334, + "semantic_entropy": 0.7831142544746399, + "step": 700 + }, + { + "epoch": 0.3773206221776217, + "grad_norm": 6.265710436146083, + "learning_rate": 9.979895395076608e-07, + "logits/chosen": -0.07541519403457642, + "logits/rejected": 0.09938128292560577, + "logps/chosen": -1.316994071006775, + "logps/rejected": -1.4729902744293213, + "loss": 2.0786, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.316994071006775, + "rewards/margins": 0.15599612891674042, + "rewards/rejected": -1.4729902744293213, + "semantic_entropy": 0.7971211671829224, + "step": 705 + }, + { + "epoch": 0.37999665495902324, + "grad_norm": 7.845912676948312, + "learning_rate": 9.9784760231197e-07, + "logits/chosen": 0.02958468720316887, + "logits/rejected": 0.12384209781885147, + "logps/chosen": -1.2575891017913818, + "logps/rejected": -1.391495704650879, + "loss": 2.0457, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2575891017913818, + "rewards/margins": 0.13390661776065826, + "rewards/rejected": -1.391495704650879, + "semantic_entropy": 0.8254655003547668, + "step": 710 + }, + { + "epoch": 0.38267268774042484, + "grad_norm": 8.126446651554827, + "learning_rate": 9.97700834996658e-07, + "logits/chosen": -0.015873288735747337, + "logits/rejected": 0.1502937376499176, + "logps/chosen": -1.3387916088104248, + "logps/rejected": -1.4656779766082764, + "loss": 2.1058, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.3387916088104248, + "rewards/margins": 0.12688623368740082, + "rewards/rejected": -1.4656779766082764, + "semantic_entropy": 0.790082573890686, + "step": 715 + }, + { + "epoch": 0.3853487205218264, + "grad_norm": 7.634151426689379, + "learning_rate": 9.97549238985662e-07, + "logits/chosen": 0.032521337270736694, + "logits/rejected": 0.21769611537456512, + "logps/chosen": -1.3835394382476807, + "logps/rejected": -1.4489504098892212, + "loss": 2.1219, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.3835394382476807, + "rewards/margins": 0.0654108077287674, + "rewards/rejected": -1.4489504098892212, + "semantic_entropy": 0.7764595150947571, + "step": 720 + }, + { + "epoch": 0.38802475330322794, + "grad_norm": 9.067806855520102, + "learning_rate": 9.973928157497674e-07, + "logits/chosen": -0.07280922681093216, + "logits/rejected": 0.06471310555934906, + "logps/chosen": -1.2224928140640259, + "logps/rejected": -1.479270339012146, + "loss": 2.0026, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.2224928140640259, + "rewards/margins": 0.2567773759365082, + "rewards/rejected": -1.479270339012146, + "semantic_entropy": 0.8193053007125854, + "step": 725 + }, + { + "epoch": 0.39070078608462955, + "grad_norm": 9.424853255347328, + "learning_rate": 9.972315668065927e-07, + "logits/chosen": -0.11625852435827255, + "logits/rejected": 0.04367467388510704, + "logps/chosen": -1.3176000118255615, + "logps/rejected": -1.418208122253418, + "loss": 2.0845, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.3176000118255615, + "rewards/margins": 0.10060807317495346, + "rewards/rejected": -1.418208122253418, + "semantic_entropy": 0.8038949966430664, + "step": 730 + }, + { + "epoch": 0.3933768188660311, + "grad_norm": 6.511587191195001, + "learning_rate": 9.97065493720576e-07, + "logits/chosen": -0.06525146961212158, + "logits/rejected": 0.03639759123325348, + "logps/chosen": -1.338934063911438, + "logps/rejected": -1.4079347848892212, + "loss": 2.0846, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.338934063911438, + "rewards/margins": 0.06900075078010559, + "rewards/rejected": -1.4079347848892212, + "semantic_entropy": 0.7866891622543335, + "step": 735 + }, + { + "epoch": 0.3960528516474327, + "grad_norm": 10.655773370348717, + "learning_rate": 9.968945981029594e-07, + "logits/chosen": -0.06840531527996063, + "logits/rejected": 0.09909432381391525, + "logps/chosen": -1.3917211294174194, + "logps/rejected": -1.4396966695785522, + "loss": 2.1392, + "rewards/accuracies": 0.48124998807907104, + "rewards/chosen": -1.3917211294174194, + "rewards/margins": 0.04797549173235893, + "rewards/rejected": -1.4396966695785522, + "semantic_entropy": 0.7690067887306213, + "step": 740 + }, + { + "epoch": 0.39872888442883425, + "grad_norm": 7.2486898664496655, + "learning_rate": 9.967188816117726e-07, + "logits/chosen": 0.03993947058916092, + "logits/rejected": 0.1131952553987503, + "logps/chosen": -1.3727612495422363, + "logps/rejected": -1.5764000415802002, + "loss": 2.0991, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.3727612495422363, + "rewards/margins": 0.20363883674144745, + "rewards/rejected": -1.5764000415802002, + "semantic_entropy": 0.7843629717826843, + "step": 745 + }, + { + "epoch": 0.4014049172102358, + "grad_norm": 6.977794397353747, + "learning_rate": 9.965383459518179e-07, + "logits/chosen": -0.020700041204690933, + "logits/rejected": 0.14366576075553894, + "logps/chosen": -1.3073142766952515, + "logps/rejected": -1.472053050994873, + "loss": 2.0539, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.3073142766952515, + "rewards/margins": 0.16473881900310516, + "rewards/rejected": -1.472053050994873, + "semantic_entropy": 0.8046748042106628, + "step": 750 + }, + { + "epoch": 0.4040809499916374, + "grad_norm": 5.541992821131799, + "learning_rate": 9.963529928746533e-07, + "logits/chosen": 0.03654784709215164, + "logits/rejected": 0.1720220446586609, + "logps/chosen": -1.3389575481414795, + "logps/rejected": -1.428675889968872, + "loss": 2.0955, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -1.3389575481414795, + "rewards/margins": 0.08971838653087616, + "rewards/rejected": -1.428675889968872, + "semantic_entropy": 0.8020346760749817, + "step": 755 + }, + { + "epoch": 0.40675698277303896, + "grad_norm": 5.428036718467446, + "learning_rate": 9.961628241785746e-07, + "logits/chosen": -0.07116580754518509, + "logits/rejected": -0.0020129233598709106, + "logps/chosen": -1.3516285419464111, + "logps/rejected": -1.5027011632919312, + "loss": 2.0824, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.3516285419464111, + "rewards/margins": 0.15107260644435883, + "rewards/rejected": -1.5027011632919312, + "semantic_entropy": 0.7827638387680054, + "step": 760 + }, + { + "epoch": 0.40943301555444056, + "grad_norm": 14.644131619246702, + "learning_rate": 9.959678417085998e-07, + "logits/chosen": -0.04214087873697281, + "logits/rejected": 0.047335632145404816, + "logps/chosen": -1.315185308456421, + "logps/rejected": -1.4251818656921387, + "loss": 2.0729, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.315185308456421, + "rewards/margins": 0.10999642312526703, + "rewards/rejected": -1.4251818656921387, + "semantic_entropy": 0.8101986050605774, + "step": 765 + }, + { + "epoch": 0.4121090483358421, + "grad_norm": 7.817281275347967, + "learning_rate": 9.957680473564493e-07, + "logits/chosen": 0.05925404280424118, + "logits/rejected": 0.18113121390342712, + "logps/chosen": -1.2680829763412476, + "logps/rejected": -1.492113471031189, + "loss": 2.018, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.2680829763412476, + "rewards/margins": 0.22403042018413544, + "rewards/rejected": -1.492113471031189, + "semantic_entropy": 0.8082300424575806, + "step": 770 + }, + { + "epoch": 0.41478508111724366, + "grad_norm": 8.006770491529315, + "learning_rate": 9.95563443060529e-07, + "logits/chosen": -0.08509613573551178, + "logits/rejected": 0.08377712965011597, + "logps/chosen": -1.3459033966064453, + "logps/rejected": -1.5283472537994385, + "loss": 2.0884, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.3459033966064453, + "rewards/margins": 0.18244390189647675, + "rewards/rejected": -1.5283472537994385, + "semantic_entropy": 0.7789994478225708, + "step": 775 + }, + { + "epoch": 0.41746111389864526, + "grad_norm": 6.993465817399214, + "learning_rate": 9.95354030805911e-07, + "logits/chosen": -0.13927313685417175, + "logits/rejected": 0.0057820407673716545, + "logps/chosen": -1.2701890468597412, + "logps/rejected": -1.435626745223999, + "loss": 2.0319, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.2701890468597412, + "rewards/margins": 0.16543766856193542, + "rewards/rejected": -1.435626745223999, + "semantic_entropy": 0.8075081706047058, + "step": 780 + }, + { + "epoch": 0.4201371466800468, + "grad_norm": 8.559119399764109, + "learning_rate": 9.951398126243133e-07, + "logits/chosen": 0.013558772392570972, + "logits/rejected": 0.13729970157146454, + "logps/chosen": -1.2590985298156738, + "logps/rejected": -1.4598546028137207, + "loss": 2.0322, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.2590985298156738, + "rewards/margins": 0.2007559984922409, + "rewards/rejected": -1.4598546028137207, + "semantic_entropy": 0.8115525245666504, + "step": 785 + }, + { + "epoch": 0.4228131794614484, + "grad_norm": 7.447630204541813, + "learning_rate": 9.94920790594082e-07, + "logits/chosen": -0.057849399745464325, + "logits/rejected": 0.06551474332809448, + "logps/chosen": -1.3101880550384521, + "logps/rejected": -1.3954278230667114, + "loss": 2.0839, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -1.3101880550384521, + "rewards/margins": 0.08523982018232346, + "rewards/rejected": -1.3954278230667114, + "semantic_entropy": 0.8039585947990417, + "step": 790 + }, + { + "epoch": 0.42548921224284997, + "grad_norm": 6.021313817313798, + "learning_rate": 9.946969668401696e-07, + "logits/chosen": -0.07067342847585678, + "logits/rejected": 0.11503966152667999, + "logps/chosen": -1.2911334037780762, + "logps/rejected": -1.4451611042022705, + "loss": 2.0554, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2911334037780762, + "rewards/margins": 0.15402746200561523, + "rewards/rejected": -1.4451611042022705, + "semantic_entropy": 0.7996021509170532, + "step": 795 + }, + { + "epoch": 0.4281652450242516, + "grad_norm": 7.6340391184091345, + "learning_rate": 9.944683435341155e-07, + "logits/chosen": -0.03889727592468262, + "logits/rejected": 0.03886578604578972, + "logps/chosen": -1.2934906482696533, + "logps/rejected": -1.3591432571411133, + "loss": 2.0818, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.2934906482696533, + "rewards/margins": 0.06565256416797638, + "rewards/rejected": -1.3591432571411133, + "semantic_entropy": 0.8230938911437988, + "step": 800 + }, + { + "epoch": 0.4281652450242516, + "eval_logits/chosen": 0.28142249584198, + "eval_logits/rejected": 0.3657718598842621, + "eval_logps/chosen": -1.3264150619506836, + "eval_logps/rejected": -1.4818079471588135, + "eval_loss": 2.0713980197906494, + "eval_rewards/accuracies": 0.5586053133010864, + "eval_rewards/chosen": -1.3264150619506836, + "eval_rewards/margins": 0.1553928405046463, + "eval_rewards/rejected": -1.4818079471588135, + "eval_runtime": 34.574, + "eval_samples_per_second": 38.902, + "eval_semantic_entropy": 0.7937625646591187, + "eval_steps_per_second": 9.747, + "step": 800 + }, + { + "epoch": 0.4308412778056531, + "grad_norm": 6.792990665970197, + "learning_rate": 9.942349228940236e-07, + "logits/chosen": -0.06414131075143814, + "logits/rejected": 0.09192129969596863, + "logps/chosen": -1.3388340473175049, + "logps/rejected": -1.5097267627716064, + "loss": 2.0773, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.3388340473175049, + "rewards/margins": 0.17089280486106873, + "rewards/rejected": -1.5097267627716064, + "semantic_entropy": 0.7890151143074036, + "step": 805 + }, + { + "epoch": 0.43351731058705467, + "grad_norm": 8.231160151703877, + "learning_rate": 9.939967071845424e-07, + "logits/chosen": 0.04229467362165451, + "logits/rejected": 0.11417990922927856, + "logps/chosen": -1.2498180866241455, + "logps/rejected": -1.394517183303833, + "loss": 2.0305, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.2498180866241455, + "rewards/margins": 0.14469924569129944, + "rewards/rejected": -1.394517183303833, + "semantic_entropy": 0.8242220878601074, + "step": 810 + }, + { + "epoch": 0.4361933433684563, + "grad_norm": 8.264111773098524, + "learning_rate": 9.937536987168413e-07, + "logits/chosen": 0.01688297465443611, + "logits/rejected": 0.1425141841173172, + "logps/chosen": -1.2386586666107178, + "logps/rejected": -1.4612213373184204, + "loss": 1.9811, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2386586666107178, + "rewards/margins": 0.22256258130073547, + "rewards/rejected": -1.4612213373184204, + "semantic_entropy": 0.8069499731063843, + "step": 815 + }, + { + "epoch": 0.4388693761498578, + "grad_norm": 7.653356935759378, + "learning_rate": 9.935058998485896e-07, + "logits/chosen": 0.041509293019771576, + "logits/rejected": 0.08839239925146103, + "logps/chosen": -1.2882188558578491, + "logps/rejected": -1.4596892595291138, + "loss": 2.0518, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.2882188558578491, + "rewards/margins": 0.17147038877010345, + "rewards/rejected": -1.4596892595291138, + "semantic_entropy": 0.8028401136398315, + "step": 820 + }, + { + "epoch": 0.44154540893125943, + "grad_norm": 8.096203198152919, + "learning_rate": 9.932533129839333e-07, + "logits/chosen": -0.03431471437215805, + "logits/rejected": 0.0898442417383194, + "logps/chosen": -1.2290394306182861, + "logps/rejected": -1.338889241218567, + "loss": 2.0372, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -1.2290394306182861, + "rewards/margins": 0.10984987020492554, + "rewards/rejected": -1.338889241218567, + "semantic_entropy": 0.8230849504470825, + "step": 825 + }, + { + "epoch": 0.444221441712661, + "grad_norm": 6.980870065250988, + "learning_rate": 9.929959405734711e-07, + "logits/chosen": 0.06439422070980072, + "logits/rejected": 0.22203806042671204, + "logps/chosen": -1.3431024551391602, + "logps/rejected": -1.4144035577774048, + "loss": 2.0952, + "rewards/accuracies": 0.5062500238418579, + "rewards/chosen": -1.3431024551391602, + "rewards/margins": 0.07130113244056702, + "rewards/rejected": -1.4144035577774048, + "semantic_entropy": 0.7883853912353516, + "step": 830 + }, + { + "epoch": 0.44689747449406253, + "grad_norm": 10.31936880093806, + "learning_rate": 9.927337851142314e-07, + "logits/chosen": 0.0007377028232440352, + "logits/rejected": 0.12605439126491547, + "logps/chosen": -1.258170247077942, + "logps/rejected": -1.3931313753128052, + "loss": 2.0665, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.258170247077942, + "rewards/margins": 0.13496126234531403, + "rewards/rejected": -1.3931313753128052, + "semantic_entropy": 0.8210614323616028, + "step": 835 + }, + { + "epoch": 0.44957350727546413, + "grad_norm": 8.318944088325118, + "learning_rate": 9.924668491496474e-07, + "logits/chosen": -0.0008976459503173828, + "logits/rejected": 0.1518554389476776, + "logps/chosen": -1.2993545532226562, + "logps/rejected": -1.4758427143096924, + "loss": 2.0728, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.2993545532226562, + "rewards/margins": 0.17648813128471375, + "rewards/rejected": -1.4758427143096924, + "semantic_entropy": 0.8037001490592957, + "step": 840 + }, + { + "epoch": 0.4522495400568657, + "grad_norm": 4.966451864110367, + "learning_rate": 9.92195135269533e-07, + "logits/chosen": 0.04422593489289284, + "logits/rejected": 0.10922133922576904, + "logps/chosen": -1.303062081336975, + "logps/rejected": -1.3727529048919678, + "loss": 2.0961, + "rewards/accuracies": 0.53125, + "rewards/chosen": -1.303062081336975, + "rewards/margins": 0.06969080865383148, + "rewards/rejected": -1.3727529048919678, + "semantic_entropy": 0.8001149892807007, + "step": 845 + }, + { + "epoch": 0.4549255728382673, + "grad_norm": 9.661166270747207, + "learning_rate": 9.919186461100574e-07, + "logits/chosen": 0.014062756672501564, + "logits/rejected": 0.07492586970329285, + "logps/chosen": -1.265263557434082, + "logps/rejected": -1.409030795097351, + "loss": 2.0278, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.265263557434082, + "rewards/margins": 0.1437673270702362, + "rewards/rejected": -1.409030795097351, + "semantic_entropy": 0.8205739259719849, + "step": 850 + }, + { + "epoch": 0.45760160561966884, + "grad_norm": 6.229750804352298, + "learning_rate": 9.9163738435372e-07, + "logits/chosen": -0.012746746651828289, + "logits/rejected": 0.127157062292099, + "logps/chosen": -1.320353388786316, + "logps/rejected": -1.5081716775894165, + "loss": 2.0957, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.320353388786316, + "rewards/margins": 0.18781821429729462, + "rewards/rejected": -1.5081716775894165, + "semantic_entropy": 0.7857638597488403, + "step": 855 + }, + { + "epoch": 0.4602776384010704, + "grad_norm": 5.359816930546816, + "learning_rate": 9.913513527293234e-07, + "logits/chosen": -0.05882059782743454, + "logits/rejected": 0.0931854099035263, + "logps/chosen": -1.3550149202346802, + "logps/rejected": -1.555934190750122, + "loss": 2.0702, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.3550149202346802, + "rewards/margins": 0.20091931521892548, + "rewards/rejected": -1.555934190750122, + "semantic_entropy": 0.7748786807060242, + "step": 860 + }, + { + "epoch": 0.462953671182472, + "grad_norm": 11.204908787425222, + "learning_rate": 9.910605540119474e-07, + "logits/chosen": -0.01107565127313137, + "logits/rejected": 0.07848058640956879, + "logps/chosen": -1.257359266281128, + "logps/rejected": -1.4614964723587036, + "loss": 2.0473, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.257359266281128, + "rewards/margins": 0.2041374146938324, + "rewards/rejected": -1.4614964723587036, + "semantic_entropy": 0.8138553500175476, + "step": 865 + }, + { + "epoch": 0.46562970396387354, + "grad_norm": 7.420345286910088, + "learning_rate": 9.907649910229227e-07, + "logits/chosen": -0.10609734058380127, + "logits/rejected": 0.13778510689735413, + "logps/chosen": -1.3095638751983643, + "logps/rejected": -1.429024338722229, + "loss": 2.0781, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.3095638751983643, + "rewards/margins": 0.11946046352386475, + "rewards/rejected": -1.429024338722229, + "semantic_entropy": 0.8033844232559204, + "step": 870 + }, + { + "epoch": 0.46830573674527515, + "grad_norm": 6.920647449331957, + "learning_rate": 9.90464666629803e-07, + "logits/chosen": 0.015931006520986557, + "logits/rejected": 0.08639371395111084, + "logps/chosen": -1.3419044017791748, + "logps/rejected": -1.471523642539978, + "loss": 2.1269, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.3419044017791748, + "rewards/margins": 0.1296190768480301, + "rewards/rejected": -1.471523642539978, + "semantic_entropy": 0.7931000590324402, + "step": 875 + }, + { + "epoch": 0.4709817695266767, + "grad_norm": 6.174422720488045, + "learning_rate": 9.901595837463363e-07, + "logits/chosen": 0.028076793998479843, + "logits/rejected": 0.18086400628089905, + "logps/chosen": -1.390434980392456, + "logps/rejected": -1.5196599960327148, + "loss": 2.1118, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.390434980392456, + "rewards/margins": 0.12922483682632446, + "rewards/rejected": -1.5196599960327148, + "semantic_entropy": 0.7693473696708679, + "step": 880 + }, + { + "epoch": 0.47365780230807825, + "grad_norm": 8.745523774867733, + "learning_rate": 9.898497453324384e-07, + "logits/chosen": -0.04087982326745987, + "logits/rejected": 0.0349409282207489, + "logps/chosen": -1.280450463294983, + "logps/rejected": -1.4879987239837646, + "loss": 2.0342, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.280450463294983, + "rewards/margins": 0.20754830539226532, + "rewards/rejected": -1.4879987239837646, + "semantic_entropy": 0.8025510907173157, + "step": 885 + }, + { + "epoch": 0.47633383508947985, + "grad_norm": 7.976852885294416, + "learning_rate": 9.895351543941628e-07, + "logits/chosen": -0.15843790769577026, + "logits/rejected": -0.04290686175227165, + "logps/chosen": -1.3285924196243286, + "logps/rejected": -1.4766861200332642, + "loss": 2.0804, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.3285924196243286, + "rewards/margins": 0.14809374511241913, + "rewards/rejected": -1.4766861200332642, + "semantic_entropy": 0.7809569239616394, + "step": 890 + }, + { + "epoch": 0.4790098678708814, + "grad_norm": 8.131211934482984, + "learning_rate": 9.892158139836724e-07, + "logits/chosen": 0.05946514755487442, + "logits/rejected": 0.17120105028152466, + "logps/chosen": -1.2260630130767822, + "logps/rejected": -1.3467620611190796, + "loss": 2.0329, + "rewards/accuracies": 0.53125, + "rewards/chosen": -1.2260630130767822, + "rewards/margins": 0.12069927155971527, + "rewards/rejected": -1.3467620611190796, + "semantic_entropy": 0.8377145528793335, + "step": 895 + }, + { + "epoch": 0.481685900652283, + "grad_norm": 7.01576110260866, + "learning_rate": 9.88891727199209e-07, + "logits/chosen": -0.07405869662761688, + "logits/rejected": -0.011778157204389572, + "logps/chosen": -1.2250185012817383, + "logps/rejected": -1.4775097370147705, + "loss": 1.9937, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2250185012817383, + "rewards/margins": 0.25249117612838745, + "rewards/rejected": -1.4775097370147705, + "semantic_entropy": 0.8225153684616089, + "step": 900 + }, + { + "epoch": 0.48436193343368455, + "grad_norm": 7.600017516249421, + "learning_rate": 9.885628971850641e-07, + "logits/chosen": 0.029972026124596596, + "logits/rejected": 0.2067229300737381, + "logps/chosen": -1.2975659370422363, + "logps/rejected": -1.4992420673370361, + "loss": 2.0551, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.2975659370422363, + "rewards/margins": 0.20167620480060577, + "rewards/rejected": -1.4992420673370361, + "semantic_entropy": 0.7884010076522827, + "step": 905 + }, + { + "epoch": 0.48703796621508616, + "grad_norm": 5.169626708622108, + "learning_rate": 9.882293271315481e-07, + "logits/chosen": -0.04081651568412781, + "logits/rejected": 0.06294569373130798, + "logps/chosen": -1.3483736515045166, + "logps/rejected": -1.4402248859405518, + "loss": 2.1274, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.3483736515045166, + "rewards/margins": 0.09185134619474411, + "rewards/rejected": -1.4402248859405518, + "semantic_entropy": 0.7705464959144592, + "step": 910 + }, + { + "epoch": 0.4897139989964877, + "grad_norm": 6.381739732007515, + "learning_rate": 9.878910202749589e-07, + "logits/chosen": -0.0046323067508637905, + "logits/rejected": 0.16577397286891937, + "logps/chosen": -1.270925521850586, + "logps/rejected": -1.4246925115585327, + "loss": 2.038, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.270925521850586, + "rewards/margins": 0.15376701951026917, + "rewards/rejected": -1.4246925115585327, + "semantic_entropy": 0.815610408782959, + "step": 915 + }, + { + "epoch": 0.49239003177788926, + "grad_norm": 10.446352300340633, + "learning_rate": 9.875479798975512e-07, + "logits/chosen": 0.08526929467916489, + "logits/rejected": 0.20687708258628845, + "logps/chosen": -1.2325282096862793, + "logps/rejected": -1.399568796157837, + "loss": 2.0354, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.2325282096862793, + "rewards/margins": 0.16704055666923523, + "rewards/rejected": -1.399568796157837, + "semantic_entropy": 0.8271034955978394, + "step": 920 + }, + { + "epoch": 0.49506606455929086, + "grad_norm": 7.057665238411657, + "learning_rate": 9.87200209327504e-07, + "logits/chosen": -0.04000019282102585, + "logits/rejected": 0.11704482138156891, + "logps/chosen": -1.3189841508865356, + "logps/rejected": -1.3870525360107422, + "loss": 2.076, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.3189841508865356, + "rewards/margins": 0.06806856393814087, + "rewards/rejected": -1.3870525360107422, + "semantic_entropy": 0.802047848701477, + "step": 925 + }, + { + "epoch": 0.4977420973406924, + "grad_norm": 11.790706049371428, + "learning_rate": 9.868477119388894e-07, + "logits/chosen": -0.05107710883021355, + "logits/rejected": 0.05921541526913643, + "logps/chosen": -1.2975822687149048, + "logps/rejected": -1.510721206665039, + "loss": 2.0754, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.2975822687149048, + "rewards/margins": 0.21313898265361786, + "rewards/rejected": -1.510721206665039, + "semantic_entropy": 0.7891716361045837, + "step": 930 + }, + { + "epoch": 0.500418130122094, + "grad_norm": 9.266305959833732, + "learning_rate": 9.864904911516383e-07, + "logits/chosen": 0.03296453505754471, + "logits/rejected": 0.07001104205846786, + "logps/chosen": -1.225716233253479, + "logps/rejected": -1.433119773864746, + "loss": 2.0381, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.225716233253479, + "rewards/margins": 0.20740346610546112, + "rewards/rejected": -1.433119773864746, + "semantic_entropy": 0.8156035542488098, + "step": 935 + }, + { + "epoch": 0.5030941629034956, + "grad_norm": 8.080697489828468, + "learning_rate": 9.861285504315084e-07, + "logits/chosen": 0.0014633402461186051, + "logits/rejected": 0.10632189363241196, + "logps/chosen": -1.2933341264724731, + "logps/rejected": -1.3772052526474, + "loss": 2.0787, + "rewards/accuracies": 0.53125, + "rewards/chosen": -1.2933341264724731, + "rewards/margins": 0.08387112617492676, + "rewards/rejected": -1.3772052526474, + "semantic_entropy": 0.8132776021957397, + "step": 940 + }, + { + "epoch": 0.5057701956848971, + "grad_norm": 7.401792535316144, + "learning_rate": 9.857618932900502e-07, + "logits/chosen": -0.028624827042222023, + "logits/rejected": 0.09492968767881393, + "logps/chosen": -1.2746442556381226, + "logps/rejected": -1.4527875185012817, + "loss": 2.0347, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.2746442556381226, + "rewards/margins": 0.1781432330608368, + "rewards/rejected": -1.4527875185012817, + "semantic_entropy": 0.8073099255561829, + "step": 945 + }, + { + "epoch": 0.5084462284662987, + "grad_norm": 6.120494404082912, + "learning_rate": 9.853905232845727e-07, + "logits/chosen": -0.014512471854686737, + "logits/rejected": 0.14616458117961884, + "logps/chosen": -1.3691775798797607, + "logps/rejected": -1.420702338218689, + "loss": 2.1453, + "rewards/accuracies": 0.48124998807907104, + "rewards/chosen": -1.3691775798797607, + "rewards/margins": 0.051524870097637177, + "rewards/rejected": -1.420702338218689, + "semantic_entropy": 0.784176230430603, + "step": 950 + }, + { + "epoch": 0.5111222612477003, + "grad_norm": 5.814071967296598, + "learning_rate": 9.850144440181095e-07, + "logits/chosen": -0.013354201801121235, + "logits/rejected": 0.19118373095989227, + "logps/chosen": -1.353991985321045, + "logps/rejected": -1.4249238967895508, + "loss": 2.1352, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -1.353991985321045, + "rewards/margins": 0.07093212008476257, + "rewards/rejected": -1.4249238967895508, + "semantic_entropy": 0.792451024055481, + "step": 955 + }, + { + "epoch": 0.5137982940291018, + "grad_norm": 6.959793634999091, + "learning_rate": 9.846336591393832e-07, + "logits/chosen": -0.039055611938238144, + "logits/rejected": 0.09675654023885727, + "logps/chosen": -1.3108570575714111, + "logps/rejected": -1.4037373065948486, + "loss": 2.0793, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.3108570575714111, + "rewards/margins": 0.09288022667169571, + "rewards/rejected": -1.4037373065948486, + "semantic_entropy": 0.8116083145141602, + "step": 960 + }, + { + "epoch": 0.5164743268105034, + "grad_norm": 6.933208594532436, + "learning_rate": 9.842481723427704e-07, + "logits/chosen": 0.037173621356487274, + "logits/rejected": 0.02412385307252407, + "logps/chosen": -1.3575502634048462, + "logps/rejected": -1.5369902849197388, + "loss": 2.0761, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.3575502634048462, + "rewards/margins": 0.1794399619102478, + "rewards/rejected": -1.5369902849197388, + "semantic_entropy": 0.7766879796981812, + "step": 965 + }, + { + "epoch": 0.519150359591905, + "grad_norm": 14.792974650803629, + "learning_rate": 9.838579873682658e-07, + "logits/chosen": 0.051386892795562744, + "logits/rejected": 0.05269744247198105, + "logps/chosen": -1.238379716873169, + "logps/rejected": -1.3489514589309692, + "loss": 2.0761, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.238379716873169, + "rewards/margins": 0.1105717197060585, + "rewards/rejected": -1.3489514589309692, + "semantic_entropy": 0.8146070241928101, + "step": 970 + }, + { + "epoch": 0.5218263923733065, + "grad_norm": 5.9146769092326315, + "learning_rate": 9.834631080014457e-07, + "logits/chosen": -0.11935378611087799, + "logits/rejected": 0.04193400591611862, + "logps/chosen": -1.3102973699569702, + "logps/rejected": -1.422062873840332, + "loss": 2.0691, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.3102973699569702, + "rewards/margins": 0.11176545917987823, + "rewards/rejected": -1.422062873840332, + "semantic_entropy": 0.7963775396347046, + "step": 975 + }, + { + "epoch": 0.5245024251547081, + "grad_norm": 12.142848553242148, + "learning_rate": 9.830635380734312e-07, + "logits/chosen": -0.10059794038534164, + "logits/rejected": 0.08547468483448029, + "logps/chosen": -1.3487987518310547, + "logps/rejected": -1.466112732887268, + "loss": 2.1156, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.3487987518310547, + "rewards/margins": 0.11731388419866562, + "rewards/rejected": -1.466112732887268, + "semantic_entropy": 0.7900714874267578, + "step": 980 + }, + { + "epoch": 0.5271784579361097, + "grad_norm": 10.252170399799747, + "learning_rate": 9.826592814608517e-07, + "logits/chosen": 0.006975511554628611, + "logits/rejected": 0.18399174511432648, + "logps/chosen": -1.3139753341674805, + "logps/rejected": -1.4533307552337646, + "loss": 2.0657, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.3139753341674805, + "rewards/margins": 0.13935549557209015, + "rewards/rejected": -1.4533307552337646, + "semantic_entropy": 0.791069507598877, + "step": 985 + }, + { + "epoch": 0.5298544907175113, + "grad_norm": 7.295258985513694, + "learning_rate": 9.822503420858067e-07, + "logits/chosen": 0.051158357411623, + "logits/rejected": 0.08825485408306122, + "logps/chosen": -1.186547040939331, + "logps/rejected": -1.4159901142120361, + "loss": 1.9773, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.186547040939331, + "rewards/margins": 0.2294432371854782, + "rewards/rejected": -1.4159901142120361, + "semantic_entropy": 0.8391534090042114, + "step": 990 + }, + { + "epoch": 0.5325305234989128, + "grad_norm": 7.682194736532205, + "learning_rate": 9.818367239158277e-07, + "logits/chosen": 0.06565249711275101, + "logits/rejected": 0.13260416686534882, + "logps/chosen": -1.3053300380706787, + "logps/rejected": -1.3484420776367188, + "loss": 2.1178, + "rewards/accuracies": 0.46875, + "rewards/chosen": -1.3053300380706787, + "rewards/margins": 0.04311198741197586, + "rewards/rejected": -1.3484420776367188, + "semantic_entropy": 0.8133613467216492, + "step": 995 + }, + { + "epoch": 0.5352065562803144, + "grad_norm": 7.541848489315755, + "learning_rate": 9.8141843096384e-07, + "logits/chosen": 0.07169102132320404, + "logits/rejected": 0.17855951189994812, + "logps/chosen": -1.3297080993652344, + "logps/rejected": -1.4753717184066772, + "loss": 2.0654, + "rewards/accuracies": 0.5062500238418579, + "rewards/chosen": -1.3297080993652344, + "rewards/margins": 0.14566387236118317, + "rewards/rejected": -1.4753717184066772, + "semantic_entropy": 0.7911792993545532, + "step": 1000 + }, + { + "epoch": 0.537882589061716, + "grad_norm": 9.710505906410269, + "learning_rate": 9.809954672881237e-07, + "logits/chosen": 0.025641867890954018, + "logits/rejected": 0.18236692249774933, + "logps/chosen": -1.336159586906433, + "logps/rejected": -1.445340871810913, + "loss": 2.1067, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.336159586906433, + "rewards/margins": 0.10918121039867401, + "rewards/rejected": -1.445340871810913, + "semantic_entropy": 0.7793572545051575, + "step": 1005 + }, + { + "epoch": 0.5405586218431175, + "grad_norm": 6.04701898316921, + "learning_rate": 9.80567836992274e-07, + "logits/chosen": -0.006621755659580231, + "logits/rejected": 0.16392755508422852, + "logps/chosen": -1.1972862482070923, + "logps/rejected": -1.4301495552062988, + "loss": 1.9837, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.1972862482070923, + "rewards/margins": 0.23286327719688416, + "rewards/rejected": -1.4301495552062988, + "semantic_entropy": 0.8263761401176453, + "step": 1010 + }, + { + "epoch": 0.5432346546245191, + "grad_norm": 9.21549679839962, + "learning_rate": 9.801355442251625e-07, + "logits/chosen": -0.013613523915410042, + "logits/rejected": 0.1519920378923416, + "logps/chosen": -1.263755202293396, + "logps/rejected": -1.4387528896331787, + "loss": 2.0395, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.263755202293396, + "rewards/margins": 0.17499780654907227, + "rewards/rejected": -1.4387528896331787, + "semantic_entropy": 0.8105853796005249, + "step": 1015 + }, + { + "epoch": 0.5459106874059207, + "grad_norm": 9.448199550904986, + "learning_rate": 9.796985931808949e-07, + "logits/chosen": -0.008507566526532173, + "logits/rejected": 0.12182090431451797, + "logps/chosen": -1.316265344619751, + "logps/rejected": -1.4976341724395752, + "loss": 2.0655, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.316265344619751, + "rewards/margins": 0.1813688725233078, + "rewards/rejected": -1.4976341724395752, + "semantic_entropy": 0.7896450757980347, + "step": 1020 + }, + { + "epoch": 0.5485867201873222, + "grad_norm": 8.559242488471353, + "learning_rate": 9.792569880987724e-07, + "logits/chosen": -0.05689584091305733, + "logits/rejected": 0.050126176327466965, + "logps/chosen": -1.2220516204833984, + "logps/rejected": -1.4843356609344482, + "loss": 1.9726, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2220516204833984, + "rewards/margins": 0.26228418946266174, + "rewards/rejected": -1.4843356609344482, + "semantic_entropy": 0.8148363828659058, + "step": 1025 + }, + { + "epoch": 0.5512627529687238, + "grad_norm": 7.995435761621511, + "learning_rate": 9.788107332632493e-07, + "logits/chosen": -0.04692408815026283, + "logits/rejected": 0.03308724984526634, + "logps/chosen": -1.2967350482940674, + "logps/rejected": -1.3965247869491577, + "loss": 2.0667, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.2967350482940674, + "rewards/margins": 0.09978990256786346, + "rewards/rejected": -1.3965247869491577, + "semantic_entropy": 0.8044675588607788, + "step": 1030 + }, + { + "epoch": 0.5539387857501255, + "grad_norm": 6.868541007975739, + "learning_rate": 9.783598330038924e-07, + "logits/chosen": -0.04235472530126572, + "logits/rejected": 0.0641079992055893, + "logps/chosen": -1.3835198879241943, + "logps/rejected": -1.4320094585418701, + "loss": 2.1453, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -1.3835198879241943, + "rewards/margins": 0.048489637672901154, + "rewards/rejected": -1.4320094585418701, + "semantic_entropy": 0.7668356895446777, + "step": 1035 + }, + { + "epoch": 0.5566148185315271, + "grad_norm": 7.118306635424307, + "learning_rate": 9.779042916953376e-07, + "logits/chosen": -0.023677151650190353, + "logits/rejected": 0.11396167427301407, + "logps/chosen": -1.3007886409759521, + "logps/rejected": -1.4533780813217163, + "loss": 2.063, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.3007886409759521, + "rewards/margins": 0.15258948504924774, + "rewards/rejected": -1.4533780813217163, + "semantic_entropy": 0.8035913705825806, + "step": 1040 + }, + { + "epoch": 0.5592908513129285, + "grad_norm": 6.137340844968411, + "learning_rate": 9.774441137572487e-07, + "logits/chosen": -0.07179170101881027, + "logits/rejected": 0.056475620716810226, + "logps/chosen": -1.2769570350646973, + "logps/rejected": -1.4823251962661743, + "loss": 2.0323, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.2769570350646973, + "rewards/margins": 0.20536811649799347, + "rewards/rejected": -1.4823251962661743, + "semantic_entropy": 0.7965216636657715, + "step": 1045 + }, + { + "epoch": 0.5619668840943302, + "grad_norm": 13.048109925378954, + "learning_rate": 9.76979303654274e-07, + "logits/chosen": -0.09428082406520844, + "logits/rejected": -0.0019208311568945646, + "logps/chosen": -1.3292375802993774, + "logps/rejected": -1.5085084438323975, + "loss": 2.0742, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.3292375802993774, + "rewards/margins": 0.1792708784341812, + "rewards/rejected": -1.5085084438323975, + "semantic_entropy": 0.7879042029380798, + "step": 1050 + }, + { + "epoch": 0.5646429168757318, + "grad_norm": 11.948351218965916, + "learning_rate": 9.765098658960035e-07, + "logits/chosen": -0.023944038897752762, + "logits/rejected": 0.04848332703113556, + "logps/chosen": -1.3154340982437134, + "logps/rejected": -1.4888745546340942, + "loss": 2.0566, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.3154340982437134, + "rewards/margins": 0.17344054579734802, + "rewards/rejected": -1.4888745546340942, + "semantic_entropy": 0.7741323709487915, + "step": 1055 + }, + { + "epoch": 0.5673189496571333, + "grad_norm": 8.531396445978677, + "learning_rate": 9.76035805036924e-07, + "logits/chosen": 0.01059248112142086, + "logits/rejected": 0.175526961684227, + "logps/chosen": -1.3848090171813965, + "logps/rejected": -1.5161800384521484, + "loss": 2.1052, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.3848090171813965, + "rewards/margins": 0.13137102127075195, + "rewards/rejected": -1.5161800384521484, + "semantic_entropy": 0.7662585973739624, + "step": 1060 + }, + { + "epoch": 0.5699949824385349, + "grad_norm": 6.393750476767566, + "learning_rate": 9.755571256763764e-07, + "logits/chosen": 0.028286850079894066, + "logits/rejected": 0.14960213005542755, + "logps/chosen": -1.2523295879364014, + "logps/rejected": -1.4868011474609375, + "loss": 1.9998, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2523295879364014, + "rewards/margins": 0.23447155952453613, + "rewards/rejected": -1.4868011474609375, + "semantic_entropy": 0.7917420864105225, + "step": 1065 + }, + { + "epoch": 0.5726710152199365, + "grad_norm": 7.772674162303587, + "learning_rate": 9.750738324585097e-07, + "logits/chosen": -0.0999111533164978, + "logits/rejected": 0.1293635070323944, + "logps/chosen": -1.3048207759857178, + "logps/rejected": -1.5056029558181763, + "loss": 2.0207, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.3048207759857178, + "rewards/margins": 0.20078222453594208, + "rewards/rejected": -1.5056029558181763, + "semantic_entropy": 0.7988306879997253, + "step": 1070 + }, + { + "epoch": 0.5753470480013381, + "grad_norm": 7.3703453263060315, + "learning_rate": 9.74585930072237e-07, + "logits/chosen": -0.029549255967140198, + "logits/rejected": 0.0868106484413147, + "logps/chosen": -1.2544111013412476, + "logps/rejected": -1.4883978366851807, + "loss": 2.0379, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2544111013412476, + "rewards/margins": 0.23398666083812714, + "rewards/rejected": -1.4883978366851807, + "semantic_entropy": 0.8126281499862671, + "step": 1075 + }, + { + "epoch": 0.5780230807827396, + "grad_norm": 9.372783611133503, + "learning_rate": 9.740934232511892e-07, + "logits/chosen": -0.0778825581073761, + "logits/rejected": 0.02420179545879364, + "logps/chosen": -1.3782198429107666, + "logps/rejected": -1.4401485919952393, + "loss": 2.1265, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -1.3782198429107666, + "rewards/margins": 0.06192876026034355, + "rewards/rejected": -1.4401485919952393, + "semantic_entropy": 0.7773522138595581, + "step": 1080 + }, + { + "epoch": 0.5806991135641412, + "grad_norm": 6.910970222784744, + "learning_rate": 9.735963167736698e-07, + "logits/chosen": 0.008498705923557281, + "logits/rejected": 0.16839997470378876, + "logps/chosen": -1.335448980331421, + "logps/rejected": -1.3817907571792603, + "loss": 2.1352, + "rewards/accuracies": 0.48750001192092896, + "rewards/chosen": -1.335448980331421, + "rewards/margins": 0.04634168744087219, + "rewards/rejected": -1.3817907571792603, + "semantic_entropy": 0.8023829460144043, + "step": 1085 + }, + { + "epoch": 0.5833751463455428, + "grad_norm": 6.445622120943113, + "learning_rate": 9.730946154626078e-07, + "logits/chosen": -0.009309718385338783, + "logits/rejected": 0.08693398535251617, + "logps/chosen": -1.3074315786361694, + "logps/rejected": -1.3497371673583984, + "loss": 2.1139, + "rewards/accuracies": 0.4937500059604645, + "rewards/chosen": -1.3074315786361694, + "rewards/margins": 0.04230565205216408, + "rewards/rejected": -1.3497371673583984, + "semantic_entropy": 0.8121057748794556, + "step": 1090 + }, + { + "epoch": 0.5860511791269443, + "grad_norm": 10.138338076160649, + "learning_rate": 9.725883241855117e-07, + "logits/chosen": -0.11470967531204224, + "logits/rejected": 0.01609751023352146, + "logps/chosen": -1.2618416547775269, + "logps/rejected": -1.4212886095046997, + "loss": 2.0482, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.2618416547775269, + "rewards/margins": 0.15944695472717285, + "rewards/rejected": -1.4212886095046997, + "semantic_entropy": 0.811863899230957, + "step": 1095 + }, + { + "epoch": 0.5887272119083459, + "grad_norm": 8.91871654543862, + "learning_rate": 9.720774478544218e-07, + "logits/chosen": -0.00520699005573988, + "logits/rejected": 0.09051401168107986, + "logps/chosen": -1.1748051643371582, + "logps/rejected": -1.500089406967163, + "loss": 1.954, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.1748051643371582, + "rewards/margins": 0.3252841532230377, + "rewards/rejected": -1.500089406967163, + "semantic_entropy": 0.8329869508743286, + "step": 1100 + }, + { + "epoch": 0.5914032446897475, + "grad_norm": 7.263517263520201, + "learning_rate": 9.715619914258624e-07, + "logits/chosen": -0.043882619589567184, + "logits/rejected": 0.029377352446317673, + "logps/chosen": -1.297758936882019, + "logps/rejected": -1.424983263015747, + "loss": 2.0699, + "rewards/accuracies": 0.53125, + "rewards/chosen": -1.297758936882019, + "rewards/margins": 0.1272241324186325, + "rewards/rejected": -1.424983263015747, + "semantic_entropy": 0.799700140953064, + "step": 1105 + }, + { + "epoch": 0.594079277471149, + "grad_norm": 9.534682687394518, + "learning_rate": 9.710419599007937e-07, + "logits/chosen": -0.00919708888977766, + "logits/rejected": 0.11027137190103531, + "logps/chosen": -1.2790029048919678, + "logps/rejected": -1.341855764389038, + "loss": 2.076, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -1.2790029048919678, + "rewards/margins": 0.06285303086042404, + "rewards/rejected": -1.341855764389038, + "semantic_entropy": 0.8141870498657227, + "step": 1110 + }, + { + "epoch": 0.5967553102525506, + "grad_norm": 11.77533862230331, + "learning_rate": 9.705173583245643e-07, + "logits/chosen": 0.05369449406862259, + "logits/rejected": 0.16988304257392883, + "logps/chosen": -1.2029147148132324, + "logps/rejected": -1.4237674474716187, + "loss": 1.9884, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.2029147148132324, + "rewards/margins": 0.2208527773618698, + "rewards/rejected": -1.4237674474716187, + "semantic_entropy": 0.8146249651908875, + "step": 1115 + }, + { + "epoch": 0.5994313430339522, + "grad_norm": 8.183547599673236, + "learning_rate": 9.699881917868609e-07, + "logits/chosen": -0.13708457350730896, + "logits/rejected": -0.037772614508867264, + "logps/chosen": -1.2671456336975098, + "logps/rejected": -1.4331334829330444, + "loss": 2.0164, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.2671456336975098, + "rewards/margins": 0.16598792374134064, + "rewards/rejected": -1.4331334829330444, + "semantic_entropy": 0.8121305704116821, + "step": 1120 + }, + { + "epoch": 0.6021073758153538, + "grad_norm": 9.638683526837815, + "learning_rate": 9.694544654216594e-07, + "logits/chosen": -0.11906708776950836, + "logits/rejected": 0.0555792935192585, + "logps/chosen": -1.2739698886871338, + "logps/rejected": -1.4571704864501953, + "loss": 2.0407, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.2739698886871338, + "rewards/margins": 0.18320061266422272, + "rewards/rejected": -1.4571704864501953, + "semantic_entropy": 0.8070036172866821, + "step": 1125 + }, + { + "epoch": 0.6047834085967553, + "grad_norm": 7.966285318311775, + "learning_rate": 9.689161844071755e-07, + "logits/chosen": 0.04917113855481148, + "logits/rejected": 0.10522178560495377, + "logps/chosen": -1.310528039932251, + "logps/rejected": -1.4806125164031982, + "loss": 2.0596, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.310528039932251, + "rewards/margins": 0.17008444666862488, + "rewards/rejected": -1.4806125164031982, + "semantic_entropy": 0.8099870681762695, + "step": 1130 + }, + { + "epoch": 0.6074594413781569, + "grad_norm": 10.549909775271933, + "learning_rate": 9.683733539658138e-07, + "logits/chosen": -0.02498762123286724, + "logits/rejected": 0.12934377789497375, + "logps/chosen": -1.325805902481079, + "logps/rejected": -1.5380194187164307, + "loss": 2.046, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.325805902481079, + "rewards/margins": 0.21221347153186798, + "rewards/rejected": -1.5380194187164307, + "semantic_entropy": 0.7939231991767883, + "step": 1135 + }, + { + "epoch": 0.6101354741595585, + "grad_norm": 8.453341612686003, + "learning_rate": 9.678259793641178e-07, + "logits/chosen": -0.04770956188440323, + "logits/rejected": -0.01541087031364441, + "logps/chosen": -1.301532506942749, + "logps/rejected": -1.348470687866211, + "loss": 2.1, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.301532506942749, + "rewards/margins": 0.04693824052810669, + "rewards/rejected": -1.348470687866211, + "semantic_entropy": 0.8088309168815613, + "step": 1140 + }, + { + "epoch": 0.61281150694096, + "grad_norm": 7.110999863323454, + "learning_rate": 9.672740659127183e-07, + "logits/chosen": -0.15768319368362427, + "logits/rejected": -0.049283333122730255, + "logps/chosen": -1.320922613143921, + "logps/rejected": -1.4701412916183472, + "loss": 2.0717, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -1.320922613143921, + "rewards/margins": 0.14921870827674866, + "rewards/rejected": -1.4701412916183472, + "semantic_entropy": 0.7909232378005981, + "step": 1145 + }, + { + "epoch": 0.6154875397223616, + "grad_norm": 7.890663915423388, + "learning_rate": 9.667176189662818e-07, + "logits/chosen": -0.13520444929599762, + "logits/rejected": -0.0030995309352874756, + "logps/chosen": -1.2132972478866577, + "logps/rejected": -1.3859599828720093, + "loss": 2.0013, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2132972478866577, + "rewards/margins": 0.1726626455783844, + "rewards/rejected": -1.3859599828720093, + "semantic_entropy": 0.8403644561767578, + "step": 1150 + }, + { + "epoch": 0.6181635725037632, + "grad_norm": 6.803336058728524, + "learning_rate": 9.661566439234592e-07, + "logits/chosen": -0.015679482370615005, + "logits/rejected": 0.06638450920581818, + "logps/chosen": -1.3122317790985107, + "logps/rejected": -1.4226499795913696, + "loss": 2.0871, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.3122317790985107, + "rewards/margins": 0.11041836440563202, + "rewards/rejected": -1.4226499795913696, + "semantic_entropy": 0.8015656471252441, + "step": 1155 + }, + { + "epoch": 0.6208396052851648, + "grad_norm": 7.659599498658578, + "learning_rate": 9.655911462268327e-07, + "logits/chosen": 0.052912432700395584, + "logits/rejected": 0.13948659598827362, + "logps/chosen": -1.2624635696411133, + "logps/rejected": -1.39972984790802, + "loss": 2.0293, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.2624635696411133, + "rewards/margins": 0.13726648688316345, + "rewards/rejected": -1.39972984790802, + "semantic_entropy": 0.8163033723831177, + "step": 1160 + }, + { + "epoch": 0.6235156380665663, + "grad_norm": 5.930536297989213, + "learning_rate": 9.650211313628636e-07, + "logits/chosen": -0.006730362772941589, + "logits/rejected": 0.06927184015512466, + "logps/chosen": -1.2054722309112549, + "logps/rejected": -1.3947350978851318, + "loss": 1.9887, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2054722309112549, + "rewards/margins": 0.18926294147968292, + "rewards/rejected": -1.3947350978851318, + "semantic_entropy": 0.8181927800178528, + "step": 1165 + }, + { + "epoch": 0.6261916708479679, + "grad_norm": 6.258386860243564, + "learning_rate": 9.644466048618386e-07, + "logits/chosen": -0.05160114914178848, + "logits/rejected": 0.10188324749469757, + "logps/chosen": -1.422415018081665, + "logps/rejected": -1.477027416229248, + "loss": 2.1507, + "rewards/accuracies": 0.53125, + "rewards/chosen": -1.422415018081665, + "rewards/margins": 0.05461234971880913, + "rewards/rejected": -1.477027416229248, + "semantic_entropy": 0.7627812027931213, + "step": 1170 + }, + { + "epoch": 0.6288677036293695, + "grad_norm": 5.970398580196082, + "learning_rate": 9.63867572297816e-07, + "logits/chosen": -0.024037795141339302, + "logits/rejected": 0.14671321213245392, + "logps/chosen": -1.252068281173706, + "logps/rejected": -1.376847743988037, + "loss": 2.0549, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.252068281173706, + "rewards/margins": 0.12477940320968628, + "rewards/rejected": -1.376847743988037, + "semantic_entropy": 0.8178479075431824, + "step": 1175 + }, + { + "epoch": 0.631543736410771, + "grad_norm": 6.256496318286121, + "learning_rate": 9.632840392885727e-07, + "logits/chosen": -0.0464647002518177, + "logits/rejected": 0.0785941556096077, + "logps/chosen": -1.3559671640396118, + "logps/rejected": -1.552578091621399, + "loss": 2.0598, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.3559671640396118, + "rewards/margins": 0.19661085307598114, + "rewards/rejected": -1.552578091621399, + "semantic_entropy": 0.7601912021636963, + "step": 1180 + }, + { + "epoch": 0.6342197691921726, + "grad_norm": 9.894684261717916, + "learning_rate": 9.626960114955483e-07, + "logits/chosen": 0.01621159166097641, + "logits/rejected": 0.14661137759685516, + "logps/chosen": -1.3828543424606323, + "logps/rejected": -1.5047622919082642, + "loss": 2.1197, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.3828543424606323, + "rewards/margins": 0.12190792709589005, + "rewards/rejected": -1.5047622919082642, + "semantic_entropy": 0.769521951675415, + "step": 1185 + }, + { + "epoch": 0.6368958019735742, + "grad_norm": 8.232140475800872, + "learning_rate": 9.621034946237909e-07, + "logits/chosen": -0.0651417151093483, + "logits/rejected": 0.06812240183353424, + "logps/chosen": -1.3452720642089844, + "logps/rejected": -1.5318175554275513, + "loss": 2.0596, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.3452720642089844, + "rewards/margins": 0.18654534220695496, + "rewards/rejected": -1.5318175554275513, + "semantic_entropy": 0.7749825716018677, + "step": 1190 + }, + { + "epoch": 0.6395718347549757, + "grad_norm": 10.119925156565927, + "learning_rate": 9.615064944219021e-07, + "logits/chosen": 0.003726619528606534, + "logits/rejected": 0.12318499386310577, + "logps/chosen": -1.2271159887313843, + "logps/rejected": -1.4569180011749268, + "loss": 1.9712, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.2271159887313843, + "rewards/margins": 0.22980189323425293, + "rewards/rejected": -1.4569180011749268, + "semantic_entropy": 0.8198936581611633, + "step": 1195 + }, + { + "epoch": 0.6422478675363773, + "grad_norm": 8.765807332648222, + "learning_rate": 9.609050166819803e-07, + "logits/chosen": -0.046974822878837585, + "logits/rejected": 0.017429247498512268, + "logps/chosen": -1.269205927848816, + "logps/rejected": -1.436964750289917, + "loss": 2.0333, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.269205927848816, + "rewards/margins": 0.1677587926387787, + "rewards/rejected": -1.436964750289917, + "semantic_entropy": 0.8055108189582825, + "step": 1200 + }, + { + "epoch": 0.6422478675363773, + "eval_logits/chosen": 0.32719656825065613, + "eval_logits/rejected": 0.4174032211303711, + "eval_logps/chosen": -1.3249365091323853, + "eval_logps/rejected": -1.5070233345031738, + "eval_loss": 2.0610902309417725, + "eval_rewards/accuracies": 0.5734421610832214, + "eval_rewards/chosen": -1.3249365091323853, + "eval_rewards/margins": 0.18208687007427216, + "eval_rewards/rejected": -1.5070233345031738, + "eval_runtime": 34.5017, + "eval_samples_per_second": 38.984, + "eval_semantic_entropy": 0.7899767756462097, + "eval_steps_per_second": 9.768, + "step": 1200 + }, + { + "epoch": 0.6449239003177789, + "grad_norm": 10.989286448969933, + "learning_rate": 9.602990672395653e-07, + "logits/chosen": -0.12443922460079193, + "logits/rejected": 0.04555289074778557, + "logps/chosen": -1.2795411348342896, + "logps/rejected": -1.4252101182937622, + "loss": 2.0401, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2795411348342896, + "rewards/margins": 0.14566898345947266, + "rewards/rejected": -1.4252101182937622, + "semantic_entropy": 0.8055251836776733, + "step": 1205 + }, + { + "epoch": 0.6475999330991805, + "grad_norm": 10.284102990108195, + "learning_rate": 9.59688651973581e-07, + "logits/chosen": -0.07771871238946915, + "logits/rejected": 0.10380265861749649, + "logps/chosen": -1.3103466033935547, + "logps/rejected": -1.4392341375350952, + "loss": 2.1027, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.3103466033935547, + "rewards/margins": 0.12888756394386292, + "rewards/rejected": -1.4392341375350952, + "semantic_entropy": 0.7999576926231384, + "step": 1210 + }, + { + "epoch": 0.650275965880582, + "grad_norm": 5.798825984707924, + "learning_rate": 9.590737768062792e-07, + "logits/chosen": -0.11275134980678558, + "logits/rejected": 0.0032500834204256535, + "logps/chosen": -1.3132580518722534, + "logps/rejected": -1.3747795820236206, + "loss": 2.0875, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.3132580518722534, + "rewards/margins": 0.061521708965301514, + "rewards/rejected": -1.3747795820236206, + "semantic_entropy": 0.8179332613945007, + "step": 1215 + }, + { + "epoch": 0.6529519986619836, + "grad_norm": 9.076422127592291, + "learning_rate": 9.584544477031816e-07, + "logits/chosen": 0.07115252315998077, + "logits/rejected": 0.17380015552043915, + "logps/chosen": -1.232552170753479, + "logps/rejected": -1.386908769607544, + "loss": 2.0484, + "rewards/accuracies": 0.53125, + "rewards/chosen": -1.232552170753479, + "rewards/margins": 0.15435653924942017, + "rewards/rejected": -1.386908769607544, + "semantic_entropy": 0.8239533305168152, + "step": 1220 + }, + { + "epoch": 0.6556280314433852, + "grad_norm": 6.438707549082974, + "learning_rate": 9.578306706730215e-07, + "logits/chosen": -0.16714417934417725, + "logits/rejected": 0.03010159358382225, + "logps/chosen": -1.3243261575698853, + "logps/rejected": -1.446779489517212, + "loss": 2.0669, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.3243261575698853, + "rewards/margins": 0.12245325744152069, + "rewards/rejected": -1.446779489517212, + "semantic_entropy": 0.7870798110961914, + "step": 1225 + }, + { + "epoch": 0.6583040642247867, + "grad_norm": 8.726078412835406, + "learning_rate": 9.572024517676865e-07, + "logits/chosen": -0.07110466808080673, + "logits/rejected": 0.03021205961704254, + "logps/chosen": -1.2625112533569336, + "logps/rejected": -1.4501873254776, + "loss": 2.0438, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.2625112533569336, + "rewards/margins": 0.18767592310905457, + "rewards/rejected": -1.4501873254776, + "semantic_entropy": 0.8156733512878418, + "step": 1230 + }, + { + "epoch": 0.6609800970061883, + "grad_norm": 6.3660145593188995, + "learning_rate": 9.565697970821593e-07, + "logits/chosen": -0.040026407688856125, + "logits/rejected": 0.08541294187307358, + "logps/chosen": -1.3209205865859985, + "logps/rejected": -1.4224975109100342, + "loss": 2.0924, + "rewards/accuracies": 0.53125, + "rewards/chosen": -1.3209205865859985, + "rewards/margins": 0.10157684236764908, + "rewards/rejected": -1.4224975109100342, + "semantic_entropy": 0.7984825372695923, + "step": 1235 + }, + { + "epoch": 0.6636561297875899, + "grad_norm": 9.056017527039876, + "learning_rate": 9.559327127544585e-07, + "logits/chosen": -0.1611124575138092, + "logits/rejected": -0.035345181822776794, + "logps/chosen": -1.2943966388702393, + "logps/rejected": -1.4472328424453735, + "loss": 2.0583, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.2943966388702393, + "rewards/margins": 0.15283603966236115, + "rewards/rejected": -1.4472328424453735, + "semantic_entropy": 0.8084890246391296, + "step": 1240 + }, + { + "epoch": 0.6663321625689914, + "grad_norm": 8.337516442690244, + "learning_rate": 9.552912049655789e-07, + "logits/chosen": -0.07143320143222809, + "logits/rejected": 0.09474059194326401, + "logps/chosen": -1.3714230060577393, + "logps/rejected": -1.4419299364089966, + "loss": 2.1097, + "rewards/accuracies": 0.48750001192092896, + "rewards/chosen": -1.3714230060577393, + "rewards/margins": 0.07050693780183792, + "rewards/rejected": -1.4419299364089966, + "semantic_entropy": 0.7875084280967712, + "step": 1245 + }, + { + "epoch": 0.669008195350393, + "grad_norm": 11.557141587121166, + "learning_rate": 9.546452799394315e-07, + "logits/chosen": -0.06920245289802551, + "logits/rejected": 0.1126919835805893, + "logps/chosen": -1.355477213859558, + "logps/rejected": -1.4375073909759521, + "loss": 2.109, + "rewards/accuracies": 0.4937500059604645, + "rewards/chosen": -1.355477213859558, + "rewards/margins": 0.0820300504565239, + "rewards/rejected": -1.4375073909759521, + "semantic_entropy": 0.7804659605026245, + "step": 1250 + }, + { + "epoch": 0.6716842281317946, + "grad_norm": 10.611897366182086, + "learning_rate": 9.539949439427846e-07, + "logits/chosen": -0.07876632362604141, + "logits/rejected": 0.041526369750499725, + "logps/chosen": -1.3036260604858398, + "logps/rejected": -1.4317988157272339, + "loss": 2.0595, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.3036260604858398, + "rewards/margins": 0.12817277014255524, + "rewards/rejected": -1.4317988157272339, + "semantic_entropy": 0.8029844164848328, + "step": 1255 + }, + { + "epoch": 0.6743602609131962, + "grad_norm": 7.570396919067868, + "learning_rate": 9.533402032852002e-07, + "logits/chosen": -0.09820413589477539, + "logits/rejected": 0.027981841936707497, + "logps/chosen": -1.2500559091567993, + "logps/rejected": -1.462172269821167, + "loss": 2.0205, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2500559091567993, + "rewards/margins": 0.21211643517017365, + "rewards/rejected": -1.462172269821167, + "semantic_entropy": 0.8136337995529175, + "step": 1260 + }, + { + "epoch": 0.6770362936945977, + "grad_norm": 7.519925985649294, + "learning_rate": 9.526810643189754e-07, + "logits/chosen": -0.0017353773582726717, + "logits/rejected": 0.13616499304771423, + "logps/chosen": -1.289548397064209, + "logps/rejected": -1.444461464881897, + "loss": 2.0378, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.289548397064209, + "rewards/margins": 0.15491308271884918, + "rewards/rejected": -1.444461464881897, + "semantic_entropy": 0.8108280301094055, + "step": 1265 + }, + { + "epoch": 0.6797123264759993, + "grad_norm": 7.8869932020973454, + "learning_rate": 9.52017533439079e-07, + "logits/chosen": -0.06916630268096924, + "logits/rejected": 0.03533398360013962, + "logps/chosen": -1.260999083518982, + "logps/rejected": -1.4440901279449463, + "loss": 2.0333, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.260999083518982, + "rewards/margins": 0.1830909699201584, + "rewards/rejected": -1.4440901279449463, + "semantic_entropy": 0.8088814616203308, + "step": 1270 + }, + { + "epoch": 0.6823883592574009, + "grad_norm": 5.869146541625044, + "learning_rate": 9.513496170830909e-07, + "logits/chosen": -0.04706912487745285, + "logits/rejected": 0.05594928190112114, + "logps/chosen": -1.3073713779449463, + "logps/rejected": -1.4249076843261719, + "loss": 2.0741, + "rewards/accuracies": 0.48750001192092896, + "rewards/chosen": -1.3073713779449463, + "rewards/margins": 0.11753638088703156, + "rewards/rejected": -1.4249076843261719, + "semantic_entropy": 0.8061819076538086, + "step": 1275 + }, + { + "epoch": 0.6850643920388024, + "grad_norm": 8.181696193081255, + "learning_rate": 9.506773217311382e-07, + "logits/chosen": -0.06857715547084808, + "logits/rejected": 0.07334651052951813, + "logps/chosen": -1.381034016609192, + "logps/rejected": -1.4767224788665771, + "loss": 2.1219, + "rewards/accuracies": 0.53125, + "rewards/chosen": -1.381034016609192, + "rewards/margins": 0.09568850696086884, + "rewards/rejected": -1.4767224788665771, + "semantic_entropy": 0.7766083478927612, + "step": 1280 + }, + { + "epoch": 0.687740424820204, + "grad_norm": 9.44924573864344, + "learning_rate": 9.500006539058334e-07, + "logits/chosen": -0.018108461052179337, + "logits/rejected": 0.1121261939406395, + "logps/chosen": -1.2533257007598877, + "logps/rejected": -1.3792779445648193, + "loss": 2.0225, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2533257007598877, + "rewards/margins": 0.12595216929912567, + "rewards/rejected": -1.3792779445648193, + "semantic_entropy": 0.8158677816390991, + "step": 1285 + }, + { + "epoch": 0.6904164576016056, + "grad_norm": 7.4959004943032825, + "learning_rate": 9.493196201722109e-07, + "logits/chosen": -0.15983861684799194, + "logits/rejected": -0.022212965413928032, + "logps/chosen": -1.3256019353866577, + "logps/rejected": -1.3897264003753662, + "loss": 2.1197, + "rewards/accuracies": 0.48750001192092896, + "rewards/chosen": -1.3256019353866577, + "rewards/margins": 0.06412459164857864, + "rewards/rejected": -1.3897264003753662, + "semantic_entropy": 0.7991577386856079, + "step": 1290 + }, + { + "epoch": 0.6930924903830072, + "grad_norm": 6.065432032119847, + "learning_rate": 9.486342271376628e-07, + "logits/chosen": -0.07144840061664581, + "logits/rejected": -0.05996709316968918, + "logps/chosen": -1.2956889867782593, + "logps/rejected": -1.4863463640213013, + "loss": 2.0367, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.2956889867782593, + "rewards/margins": 0.1906573474407196, + "rewards/rejected": -1.4863463640213013, + "semantic_entropy": 0.7861794233322144, + "step": 1295 + }, + { + "epoch": 0.6957685231644087, + "grad_norm": 7.768340559958128, + "learning_rate": 9.479444814518755e-07, + "logits/chosen": -0.057303112000226974, + "logits/rejected": 0.17862965166568756, + "logps/chosen": -1.2886030673980713, + "logps/rejected": -1.4833580255508423, + "loss": 2.0291, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.2886030673980713, + "rewards/margins": 0.19475503265857697, + "rewards/rejected": -1.4833580255508423, + "semantic_entropy": 0.7993043661117554, + "step": 1300 + }, + { + "epoch": 0.6984445559458103, + "grad_norm": 6.829811228292914, + "learning_rate": 9.472503898067645e-07, + "logits/chosen": 0.05778735876083374, + "logits/rejected": 0.10782381147146225, + "logps/chosen": -1.3004398345947266, + "logps/rejected": -1.4733632802963257, + "loss": 2.0328, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.3004398345947266, + "rewards/margins": 0.1729235202074051, + "rewards/rejected": -1.4733632802963257, + "semantic_entropy": 0.8007766008377075, + "step": 1305 + }, + { + "epoch": 0.701120588727212, + "grad_norm": 6.806301756870639, + "learning_rate": 9.465519589364099e-07, + "logits/chosen": 0.03991737216711044, + "logits/rejected": 0.11657501757144928, + "logps/chosen": -1.310382604598999, + "logps/rejected": -1.4569346904754639, + "loss": 2.0875, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.310382604598999, + "rewards/margins": 0.1465521603822708, + "rewards/rejected": -1.4569346904754639, + "semantic_entropy": 0.7917109131813049, + "step": 1310 + }, + { + "epoch": 0.7037966215086134, + "grad_norm": 6.290006408474125, + "learning_rate": 9.458491956169914e-07, + "logits/chosen": -0.06548132747411728, + "logits/rejected": 0.10813206434249878, + "logps/chosen": -1.2515592575073242, + "logps/rejected": -1.487073302268982, + "loss": 2.0074, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2515592575073242, + "rewards/margins": 0.23551401495933533, + "rewards/rejected": -1.487073302268982, + "semantic_entropy": 0.8081199526786804, + "step": 1315 + }, + { + "epoch": 0.706472654290015, + "grad_norm": 5.8407006796066785, + "learning_rate": 9.451421066667215e-07, + "logits/chosen": -0.15501976013183594, + "logits/rejected": 0.03836467117071152, + "logps/chosen": -1.2408206462860107, + "logps/rejected": -1.441028118133545, + "loss": 2.0022, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.2408206462860107, + "rewards/margins": 0.20020751655101776, + "rewards/rejected": -1.441028118133545, + "semantic_entropy": 0.8187205195426941, + "step": 1320 + }, + { + "epoch": 0.7091486870714167, + "grad_norm": 12.171979565537207, + "learning_rate": 9.444306989457805e-07, + "logits/chosen": -0.00010145604755962268, + "logits/rejected": 0.10248501598834991, + "logps/chosen": -1.3242946863174438, + "logps/rejected": -1.4491575956344604, + "loss": 2.0964, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.3242946863174438, + "rewards/margins": 0.12486305087804794, + "rewards/rejected": -1.4491575956344604, + "semantic_entropy": 0.7907122373580933, + "step": 1325 + }, + { + "epoch": 0.7118247198528181, + "grad_norm": 5.813429350518074, + "learning_rate": 9.437149793562489e-07, + "logits/chosen": -0.030075322836637497, + "logits/rejected": 0.08605314791202545, + "logps/chosen": -1.2983765602111816, + "logps/rejected": -1.3950271606445312, + "loss": 2.0709, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.2983765602111816, + "rewards/margins": 0.09665055572986603, + "rewards/rejected": -1.3950271606445312, + "semantic_entropy": 0.8118602633476257, + "step": 1330 + }, + { + "epoch": 0.7145007526342197, + "grad_norm": 6.511695432503269, + "learning_rate": 9.429949548420417e-07, + "logits/chosen": -0.010198342613875866, + "logits/rejected": 0.0688561499118805, + "logps/chosen": -1.3720452785491943, + "logps/rejected": -1.5215284824371338, + "loss": 2.0767, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.3720452785491943, + "rewards/margins": 0.14948336780071259, + "rewards/rejected": -1.5215284824371338, + "semantic_entropy": 0.7785710096359253, + "step": 1335 + }, + { + "epoch": 0.7171767854156214, + "grad_norm": 7.43508239763397, + "learning_rate": 9.422706323888396e-07, + "logits/chosen": -0.021798694506287575, + "logits/rejected": 0.007907522842288017, + "logps/chosen": -1.3327934741973877, + "logps/rejected": -1.4697725772857666, + "loss": 2.0901, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.3327934741973877, + "rewards/margins": 0.1369791030883789, + "rewards/rejected": -1.4697725772857666, + "semantic_entropy": 0.7791727185249329, + "step": 1340 + }, + { + "epoch": 0.719852818197023, + "grad_norm": 5.620204494767762, + "learning_rate": 9.415420190240225e-07, + "logits/chosen": 0.012555956840515137, + "logits/rejected": 0.18066881597042084, + "logps/chosen": -1.3168660402297974, + "logps/rejected": -1.4390350580215454, + "loss": 2.0643, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.3168660402297974, + "rewards/margins": 0.1221691146492958, + "rewards/rejected": -1.4390350580215454, + "semantic_entropy": 0.7964397668838501, + "step": 1345 + }, + { + "epoch": 0.7225288509784245, + "grad_norm": 7.437201739326167, + "learning_rate": 9.408091218166002e-07, + "logits/chosen": 0.035775817930698395, + "logits/rejected": 0.09171895682811737, + "logps/chosen": -1.307445764541626, + "logps/rejected": -1.3502302169799805, + "loss": 2.1128, + "rewards/accuracies": 0.53125, + "rewards/chosen": -1.307445764541626, + "rewards/margins": 0.04278445243835449, + "rewards/rejected": -1.3502302169799805, + "semantic_entropy": 0.8082399368286133, + "step": 1350 + }, + { + "epoch": 0.7252048837598261, + "grad_norm": 6.421590530442405, + "learning_rate": 9.400719478771449e-07, + "logits/chosen": -0.00508537283167243, + "logits/rejected": 0.272622287273407, + "logps/chosen": -1.363750696182251, + "logps/rejected": -1.4495420455932617, + "loss": 2.121, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.363750696182251, + "rewards/margins": 0.08579148352146149, + "rewards/rejected": -1.4495420455932617, + "semantic_entropy": 0.7730823755264282, + "step": 1355 + }, + { + "epoch": 0.7278809165412277, + "grad_norm": 8.172558548172619, + "learning_rate": 9.393305043577209e-07, + "logits/chosen": -0.1188274472951889, + "logits/rejected": 0.022789832204580307, + "logps/chosen": -1.3701941967010498, + "logps/rejected": -1.5333750247955322, + "loss": 2.0899, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -1.3701941967010498, + "rewards/margins": 0.16318091750144958, + "rewards/rejected": -1.5333750247955322, + "semantic_entropy": 0.7707791924476624, + "step": 1360 + }, + { + "epoch": 0.7305569493226292, + "grad_norm": 5.194706880837731, + "learning_rate": 9.38584798451817e-07, + "logits/chosen": 0.0019635481294244528, + "logits/rejected": 0.13548122346401215, + "logps/chosen": -1.303626298904419, + "logps/rejected": -1.4488534927368164, + "loss": 2.0556, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.303626298904419, + "rewards/margins": 0.14522729814052582, + "rewards/rejected": -1.4488534927368164, + "semantic_entropy": 0.7995003461837769, + "step": 1365 + }, + { + "epoch": 0.7332329821040308, + "grad_norm": 8.017472283818565, + "learning_rate": 9.37834837394275e-07, + "logits/chosen": -0.009090607985854149, + "logits/rejected": 0.10668311268091202, + "logps/chosen": -1.3580187559127808, + "logps/rejected": -1.578385591506958, + "loss": 2.0559, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.3580187559127808, + "rewards/margins": 0.2203669250011444, + "rewards/rejected": -1.578385591506958, + "semantic_entropy": 0.7690521478652954, + "step": 1370 + }, + { + "epoch": 0.7359090148854324, + "grad_norm": 6.748183181544977, + "learning_rate": 9.370806284612203e-07, + "logits/chosen": -0.04126691818237305, + "logits/rejected": 0.09672950208187103, + "logps/chosen": -1.2818526029586792, + "logps/rejected": -1.540055513381958, + "loss": 2.0156, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2818526029586792, + "rewards/margins": 0.25820282101631165, + "rewards/rejected": -1.540055513381958, + "semantic_entropy": 0.7934702038764954, + "step": 1375 + }, + { + "epoch": 0.738585047666834, + "grad_norm": 9.607687872946466, + "learning_rate": 9.363221789699912e-07, + "logits/chosen": -0.09153338521718979, + "logits/rejected": 0.023612957447767258, + "logps/chosen": -1.307871699333191, + "logps/rejected": -1.3920269012451172, + "loss": 2.1003, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.307871699333191, + "rewards/margins": 0.08415510505437851, + "rewards/rejected": -1.3920269012451172, + "semantic_entropy": 0.8024643659591675, + "step": 1380 + }, + { + "epoch": 0.7412610804482355, + "grad_norm": 7.5314238148632295, + "learning_rate": 9.355594962790682e-07, + "logits/chosen": -0.0758143737912178, + "logits/rejected": 0.04460468143224716, + "logps/chosen": -1.2546486854553223, + "logps/rejected": -1.397021770477295, + "loss": 2.0542, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.2546486854553223, + "rewards/margins": 0.14237311482429504, + "rewards/rejected": -1.397021770477295, + "semantic_entropy": 0.8238178491592407, + "step": 1385 + }, + { + "epoch": 0.7439371132296371, + "grad_norm": 9.691312986451734, + "learning_rate": 9.34792587788002e-07, + "logits/chosen": 0.007501109037548304, + "logits/rejected": 0.11551366001367569, + "logps/chosen": -1.3173161745071411, + "logps/rejected": -1.4741413593292236, + "loss": 2.0667, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.3173161745071411, + "rewards/margins": 0.15682531893253326, + "rewards/rejected": -1.4741413593292236, + "semantic_entropy": 0.7868748903274536, + "step": 1390 + }, + { + "epoch": 0.7466131460110387, + "grad_norm": 6.04815777040883, + "learning_rate": 9.34021460937342e-07, + "logits/chosen": 0.007738134823739529, + "logits/rejected": 0.09423503279685974, + "logps/chosen": -1.2795543670654297, + "logps/rejected": -1.3807176351547241, + "loss": 2.0639, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.2795543670654297, + "rewards/margins": 0.10116332769393921, + "rewards/rejected": -1.3807176351547241, + "semantic_entropy": 0.807846188545227, + "step": 1395 + }, + { + "epoch": 0.7492891787924402, + "grad_norm": 5.210066011422572, + "learning_rate": 9.332461232085646e-07, + "logits/chosen": -0.1578998863697052, + "logits/rejected": -0.03726965934038162, + "logps/chosen": -1.3505934476852417, + "logps/rejected": -1.4699254035949707, + "loss": 2.0896, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -1.3505934476852417, + "rewards/margins": 0.11933205276727676, + "rewards/rejected": -1.4699254035949707, + "semantic_entropy": 0.7852131128311157, + "step": 1400 + }, + { + "epoch": 0.7519652115738418, + "grad_norm": 8.145037317688463, + "learning_rate": 9.324665821239998e-07, + "logits/chosen": -0.08859096467494965, + "logits/rejected": 0.07373972237110138, + "logps/chosen": -1.2071774005889893, + "logps/rejected": -1.505427598953247, + "loss": 1.981, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.2071774005889893, + "rewards/margins": 0.29825007915496826, + "rewards/rejected": -1.505427598953247, + "semantic_entropy": 0.8246868252754211, + "step": 1405 + }, + { + "epoch": 0.7546412443552434, + "grad_norm": 6.162440905857332, + "learning_rate": 9.316828452467583e-07, + "logits/chosen": -0.12044434249401093, + "logits/rejected": 0.0488150492310524, + "logps/chosen": -1.3291029930114746, + "logps/rejected": -1.5076282024383545, + "loss": 2.0544, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.3291029930114746, + "rewards/margins": 0.17852529883384705, + "rewards/rejected": -1.5076282024383545, + "semantic_entropy": 0.7849763631820679, + "step": 1410 + }, + { + "epoch": 0.7573172771366449, + "grad_norm": 10.836969712453396, + "learning_rate": 9.30894920180659e-07, + "logits/chosen": -0.03222806006669998, + "logits/rejected": 0.10289813578128815, + "logps/chosen": -1.3662335872650146, + "logps/rejected": -1.3669545650482178, + "loss": 2.1405, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -1.3662335872650146, + "rewards/margins": 0.000720818352419883, + "rewards/rejected": -1.3669545650482178, + "semantic_entropy": 0.7890567779541016, + "step": 1415 + }, + { + "epoch": 0.7599933099180465, + "grad_norm": 6.394489482016526, + "learning_rate": 9.301028145701543e-07, + "logits/chosen": 0.004710095934569836, + "logits/rejected": 0.127283975481987, + "logps/chosen": -1.2482645511627197, + "logps/rejected": -1.5582225322723389, + "loss": 2.01, + "rewards/accuracies": 0.5062500238418579, + "rewards/chosen": -1.2482645511627197, + "rewards/margins": 0.3099580407142639, + "rewards/rejected": -1.5582225322723389, + "semantic_entropy": 0.8073336482048035, + "step": 1420 + }, + { + "epoch": 0.7626693426994481, + "grad_norm": 6.29413562581532, + "learning_rate": 9.293065361002563e-07, + "logits/chosen": 0.010683017782866955, + "logits/rejected": 0.08055853843688965, + "logps/chosen": -1.2729721069335938, + "logps/rejected": -1.573007345199585, + "loss": 2.0105, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2729721069335938, + "rewards/margins": 0.3000350594520569, + "rewards/rejected": -1.573007345199585, + "semantic_entropy": 0.7954605221748352, + "step": 1425 + }, + { + "epoch": 0.7653453754808497, + "grad_norm": 7.1164925104779, + "learning_rate": 9.285060924964622e-07, + "logits/chosen": -0.10673652589321136, + "logits/rejected": 0.014233958907425404, + "logps/chosen": -1.3369324207305908, + "logps/rejected": -1.4339196681976318, + "loss": 2.0844, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -1.3369324207305908, + "rewards/margins": 0.09698706865310669, + "rewards/rejected": -1.4339196681976318, + "semantic_entropy": 0.7893370389938354, + "step": 1430 + }, + { + "epoch": 0.7680214082622512, + "grad_norm": 7.812013678094771, + "learning_rate": 9.277014915246792e-07, + "logits/chosen": 0.032980311661958694, + "logits/rejected": 0.08035765588283539, + "logps/chosen": -1.2653449773788452, + "logps/rejected": -1.5054515600204468, + "loss": 2.0513, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.2653449773788452, + "rewards/margins": 0.2401064932346344, + "rewards/rejected": -1.5054515600204468, + "semantic_entropy": 0.8130581974983215, + "step": 1435 + }, + { + "epoch": 0.7706974410436528, + "grad_norm": 6.118702707949794, + "learning_rate": 9.268927409911498e-07, + "logits/chosen": -0.07677887380123138, + "logits/rejected": 0.010664084926247597, + "logps/chosen": -1.324356198310852, + "logps/rejected": -1.4014503955841064, + "loss": 2.0742, + "rewards/accuracies": 0.5062500238418579, + "rewards/chosen": -1.324356198310852, + "rewards/margins": 0.07709423452615738, + "rewards/rejected": -1.4014503955841064, + "semantic_entropy": 0.8042744398117065, + "step": 1440 + }, + { + "epoch": 0.7733734738250544, + "grad_norm": 8.801912177647397, + "learning_rate": 9.260798487423749e-07, + "logits/chosen": -0.10331074148416519, + "logits/rejected": 0.0976705476641655, + "logps/chosen": -1.3601049184799194, + "logps/rejected": -1.4674112796783447, + "loss": 2.1, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.3601049184799194, + "rewards/margins": 0.10730626434087753, + "rewards/rejected": -1.4674112796783447, + "semantic_entropy": 0.7811123728752136, + "step": 1445 + }, + { + "epoch": 0.7760495066064559, + "grad_norm": 16.238716470237314, + "learning_rate": 9.252628226650389e-07, + "logits/chosen": 0.009607335552573204, + "logits/rejected": 0.09555571526288986, + "logps/chosen": -1.2560529708862305, + "logps/rejected": -1.3869524002075195, + "loss": 2.0566, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.2560529708862305, + "rewards/margins": 0.13089947402477264, + "rewards/rejected": -1.3869524002075195, + "semantic_entropy": 0.8172558546066284, + "step": 1450 + }, + { + "epoch": 0.7787255393878575, + "grad_norm": 9.370488834506501, + "learning_rate": 9.244416706859321e-07, + "logits/chosen": -0.02865959145128727, + "logits/rejected": 0.12177982181310654, + "logps/chosen": -1.2793071269989014, + "logps/rejected": -1.4887911081314087, + "loss": 2.0322, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.2793071269989014, + "rewards/margins": 0.2094838172197342, + "rewards/rejected": -1.4887911081314087, + "semantic_entropy": 0.7951102256774902, + "step": 1455 + }, + { + "epoch": 0.7814015721692591, + "grad_norm": 6.519181163142868, + "learning_rate": 9.23616400771875e-07, + "logits/chosen": -0.017458543181419373, + "logits/rejected": 0.12942534685134888, + "logps/chosen": -1.2433431148529053, + "logps/rejected": -1.4446688890457153, + "loss": 2.0067, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2433431148529053, + "rewards/margins": 0.20132584869861603, + "rewards/rejected": -1.4446688890457153, + "semantic_entropy": 0.816464900970459, + "step": 1460 + }, + { + "epoch": 0.7840776049506607, + "grad_norm": 6.515758245208593, + "learning_rate": 9.227870209296395e-07, + "logits/chosen": 0.008127368986606598, + "logits/rejected": 0.10638369619846344, + "logps/chosen": -1.356247067451477, + "logps/rejected": -1.5012516975402832, + "loss": 2.1029, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.356247067451477, + "rewards/margins": 0.14500463008880615, + "rewards/rejected": -1.5012516975402832, + "semantic_entropy": 0.7735931277275085, + "step": 1465 + }, + { + "epoch": 0.7867536377320622, + "grad_norm": 8.64957671159049, + "learning_rate": 9.219535392058728e-07, + "logits/chosen": -0.08659469336271286, + "logits/rejected": -0.0552179217338562, + "logps/chosen": -1.3174679279327393, + "logps/rejected": -1.4319454431533813, + "loss": 2.0845, + "rewards/accuracies": 0.53125, + "rewards/chosen": -1.3174679279327393, + "rewards/margins": 0.11447747051715851, + "rewards/rejected": -1.4319454431533813, + "semantic_entropy": 0.7948979735374451, + "step": 1470 + }, + { + "epoch": 0.7894296705134638, + "grad_norm": 5.8600076062762465, + "learning_rate": 9.211159636870181e-07, + "logits/chosen": -0.08313588052988052, + "logits/rejected": 0.07544960081577301, + "logps/chosen": -1.3022918701171875, + "logps/rejected": -1.4836384057998657, + "loss": 2.0271, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.3022918701171875, + "rewards/margins": 0.18134646117687225, + "rewards/rejected": -1.4836384057998657, + "semantic_entropy": 0.7985160946846008, + "step": 1475 + }, + { + "epoch": 0.7921057032948654, + "grad_norm": 7.922496666081785, + "learning_rate": 9.202743024992367e-07, + "logits/chosen": -0.013608187437057495, + "logits/rejected": 0.08938765525817871, + "logps/chosen": -1.2842402458190918, + "logps/rejected": -1.5133750438690186, + "loss": 2.0415, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.2842402458190918, + "rewards/margins": 0.22913464903831482, + "rewards/rejected": -1.5133750438690186, + "semantic_entropy": 0.7954760789871216, + "step": 1480 + }, + { + "epoch": 0.7947817360762669, + "grad_norm": 8.575669716734673, + "learning_rate": 9.194285638083293e-07, + "logits/chosen": 0.0062799095176160336, + "logits/rejected": 0.15394124388694763, + "logps/chosen": -1.3310755491256714, + "logps/rejected": -1.5266507863998413, + "loss": 2.0564, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.3310755491256714, + "rewards/margins": 0.19557540118694305, + "rewards/rejected": -1.5266507863998413, + "semantic_entropy": 0.783862292766571, + "step": 1485 + }, + { + "epoch": 0.7974577688576685, + "grad_norm": 9.957844967407029, + "learning_rate": 9.185787558196562e-07, + "logits/chosen": -0.07122926414012909, + "logits/rejected": 0.032294441014528275, + "logps/chosen": -1.312718391418457, + "logps/rejected": -1.3787847757339478, + "loss": 2.0974, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -1.312718391418457, + "rewards/margins": 0.0660664290189743, + "rewards/rejected": -1.3787847757339478, + "semantic_entropy": 0.8111189007759094, + "step": 1490 + }, + { + "epoch": 0.8001338016390701, + "grad_norm": 9.791559868001285, + "learning_rate": 9.177248867780583e-07, + "logits/chosen": -0.05186697095632553, + "logits/rejected": 0.04654816910624504, + "logps/chosen": -1.4116029739379883, + "logps/rejected": -1.4615944623947144, + "loss": 2.1464, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.4116029739379883, + "rewards/margins": 0.04999139904975891, + "rewards/rejected": -1.4615944623947144, + "semantic_entropy": 0.7640504837036133, + "step": 1495 + }, + { + "epoch": 0.8028098344204716, + "grad_norm": 9.880609014831158, + "learning_rate": 9.168669649677769e-07, + "logits/chosen": -0.0890929326415062, + "logits/rejected": 0.006459700874984264, + "logps/chosen": -1.2926914691925049, + "logps/rejected": -1.4606952667236328, + "loss": 2.0543, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.2926914691925049, + "rewards/margins": 0.16800378262996674, + "rewards/rejected": -1.4606952667236328, + "semantic_entropy": 0.7951599359512329, + "step": 1500 + }, + { + "epoch": 0.8054858672018732, + "grad_norm": 9.372810455385036, + "learning_rate": 9.16004998712373e-07, + "logits/chosen": 0.004992163274437189, + "logits/rejected": 0.06641830503940582, + "logps/chosen": -1.2445884943008423, + "logps/rejected": -1.5159660577774048, + "loss": 1.9905, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -1.2445884943008423, + "rewards/margins": 0.2713775336742401, + "rewards/rejected": -1.5159660577774048, + "semantic_entropy": 0.8220226168632507, + "step": 1505 + }, + { + "epoch": 0.8081618999832748, + "grad_norm": 6.017423772699667, + "learning_rate": 9.151389963746472e-07, + "logits/chosen": -0.09114827960729599, + "logits/rejected": 0.16105875372886658, + "logps/chosen": -1.3403730392456055, + "logps/rejected": -1.5040563344955444, + "loss": 2.0586, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.3403730392456055, + "rewards/margins": 0.16368329524993896, + "rewards/rejected": -1.5040563344955444, + "semantic_entropy": 0.770714282989502, + "step": 1510 + }, + { + "epoch": 0.8108379327646764, + "grad_norm": 6.37267342623801, + "learning_rate": 9.142689663565577e-07, + "logits/chosen": -0.005736993160098791, + "logits/rejected": 0.06145425885915756, + "logps/chosen": -1.2793912887573242, + "logps/rejected": -1.4553543329238892, + "loss": 2.0492, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -1.2793912887573242, + "rewards/margins": 0.17596301436424255, + "rewards/rejected": -1.4553543329238892, + "semantic_entropy": 0.8051084280014038, + "step": 1515 + }, + { + "epoch": 0.8135139655460779, + "grad_norm": 9.942639013117367, + "learning_rate": 9.133949170991397e-07, + "logits/chosen": -0.007551294751465321, + "logits/rejected": 0.0692036896944046, + "logps/chosen": -1.317042589187622, + "logps/rejected": -1.4655804634094238, + "loss": 2.0592, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.317042589187622, + "rewards/margins": 0.14853790402412415, + "rewards/rejected": -1.4655804634094238, + "semantic_entropy": 0.7813259959220886, + "step": 1520 + }, + { + "epoch": 0.8161899983274795, + "grad_norm": 7.054321515604811, + "learning_rate": 9.125168570824231e-07, + "logits/chosen": -0.038256287574768066, + "logits/rejected": 0.13500620424747467, + "logps/chosen": -1.3114440441131592, + "logps/rejected": -1.4267799854278564, + "loss": 2.1041, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.3114440441131592, + "rewards/margins": 0.11533576250076294, + "rewards/rejected": -1.4267799854278564, + "semantic_entropy": 0.804587185382843, + "step": 1525 + }, + { + "epoch": 0.8188660311088811, + "grad_norm": 8.248911243528406, + "learning_rate": 9.116347948253496e-07, + "logits/chosen": -0.03580842912197113, + "logits/rejected": 0.06639468669891357, + "logps/chosen": -1.3471962213516235, + "logps/rejected": -1.468531608581543, + "loss": 2.0808, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -1.3471962213516235, + "rewards/margins": 0.12133540958166122, + "rewards/rejected": -1.468531608581543, + "semantic_entropy": 0.7958540916442871, + "step": 1530 + }, + { + "epoch": 0.8215420638902826, + "grad_norm": 9.11613499956825, + "learning_rate": 9.107487388856916e-07, + "logits/chosen": -0.07226817309856415, + "logits/rejected": 0.0961885154247284, + "logps/chosen": -1.2631547451019287, + "logps/rejected": -1.463041067123413, + "loss": 2.0131, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.2631547451019287, + "rewards/margins": 0.19988617300987244, + "rewards/rejected": -1.463041067123413, + "semantic_entropy": 0.7994940876960754, + "step": 1535 + }, + { + "epoch": 0.8242180966716842, + "grad_norm": 8.024790708944156, + "learning_rate": 9.098586978599673e-07, + "logits/chosen": 0.0032311968971043825, + "logits/rejected": 0.15999026596546173, + "logps/chosen": -1.2885329723358154, + "logps/rejected": -1.5601550340652466, + "loss": 2.0041, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2885329723358154, + "rewards/margins": 0.2716220021247864, + "rewards/rejected": -1.5601550340652466, + "semantic_entropy": 0.7936614751815796, + "step": 1540 + }, + { + "epoch": 0.8268941294530858, + "grad_norm": 6.98164321849856, + "learning_rate": 9.089646803833588e-07, + "logits/chosen": 0.008408062160015106, + "logits/rejected": 0.16597135365009308, + "logps/chosen": -1.2991634607315063, + "logps/rejected": -1.4175589084625244, + "loss": 2.0821, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.2991634607315063, + "rewards/margins": 0.11839548498392105, + "rewards/rejected": -1.4175589084625244, + "semantic_entropy": 0.8139492273330688, + "step": 1545 + }, + { + "epoch": 0.8295701622344873, + "grad_norm": 7.811228517744147, + "learning_rate": 9.080666951296276e-07, + "logits/chosen": -0.13311918079853058, + "logits/rejected": 0.13327452540397644, + "logps/chosen": -1.3553602695465088, + "logps/rejected": -1.5140632390975952, + "loss": 2.0947, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.3553602695465088, + "rewards/margins": 0.1587028056383133, + "rewards/rejected": -1.5140632390975952, + "semantic_entropy": 0.7813464403152466, + "step": 1550 + }, + { + "epoch": 0.8322461950158889, + "grad_norm": 4.678684985118599, + "learning_rate": 9.071647508110305e-07, + "logits/chosen": -0.09198766946792603, + "logits/rejected": 0.15760457515716553, + "logps/chosen": -1.387629508972168, + "logps/rejected": -1.5720300674438477, + "loss": 2.1029, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.387629508972168, + "rewards/margins": 0.18440048396587372, + "rewards/rejected": -1.5720300674438477, + "semantic_entropy": 0.7744709849357605, + "step": 1555 + }, + { + "epoch": 0.8349222277972905, + "grad_norm": 7.168853445949985, + "learning_rate": 9.062588561782354e-07, + "logits/chosen": 0.026187364012002945, + "logits/rejected": 0.09108327329158783, + "logps/chosen": -1.3502163887023926, + "logps/rejected": -1.5217186212539673, + "loss": 2.0609, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.3502163887023926, + "rewards/margins": 0.17150214314460754, + "rewards/rejected": -1.5217186212539673, + "semantic_entropy": 0.7796697616577148, + "step": 1560 + }, + { + "epoch": 0.8375982605786921, + "grad_norm": 6.101638528531929, + "learning_rate": 9.053490200202358e-07, + "logits/chosen": 0.002669230103492737, + "logits/rejected": 0.09444611519575119, + "logps/chosen": -1.3471533060073853, + "logps/rejected": -1.483074426651001, + "loss": 2.0915, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.3471533060073853, + "rewards/margins": 0.13592123985290527, + "rewards/rejected": -1.483074426651001, + "semantic_entropy": 0.7881175875663757, + "step": 1565 + }, + { + "epoch": 0.8402742933600936, + "grad_norm": 9.652107285256822, + "learning_rate": 9.044352511642661e-07, + "logits/chosen": 0.0475144125521183, + "logits/rejected": 0.05616613104939461, + "logps/chosen": -1.246050477027893, + "logps/rejected": -1.417238712310791, + "loss": 2.0355, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.246050477027893, + "rewards/margins": 0.17118819057941437, + "rewards/rejected": -1.417238712310791, + "semantic_entropy": 0.8253181576728821, + "step": 1570 + }, + { + "epoch": 0.8429503261414952, + "grad_norm": 8.0454186757764, + "learning_rate": 9.03517558475716e-07, + "logits/chosen": 0.008113816380500793, + "logits/rejected": 0.11139838397502899, + "logps/chosen": -1.3103927373886108, + "logps/rejected": -1.4039795398712158, + "loss": 2.0769, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.3103927373886108, + "rewards/margins": 0.09358695894479752, + "rewards/rejected": -1.4039795398712158, + "semantic_entropy": 0.8116000294685364, + "step": 1575 + }, + { + "epoch": 0.8456263589228968, + "grad_norm": 6.658449200158449, + "learning_rate": 9.025959508580436e-07, + "logits/chosen": 0.0631398856639862, + "logits/rejected": 0.29851242899894714, + "logps/chosen": -1.3078067302703857, + "logps/rejected": -1.4754230976104736, + "loss": 2.0336, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.3078067302703857, + "rewards/margins": 0.16761623322963715, + "rewards/rejected": -1.4754230976104736, + "semantic_entropy": 0.8020066022872925, + "step": 1580 + }, + { + "epoch": 0.8483023917042983, + "grad_norm": 6.859574582510765, + "learning_rate": 9.016704372526905e-07, + "logits/chosen": 0.017910093069076538, + "logits/rejected": 0.1749291867017746, + "logps/chosen": -1.2329002618789673, + "logps/rejected": -1.524070143699646, + "loss": 1.9995, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2329002618789673, + "rewards/margins": 0.29117000102996826, + "rewards/rejected": -1.524070143699646, + "semantic_entropy": 0.8060545921325684, + "step": 1585 + }, + { + "epoch": 0.8509784244856999, + "grad_norm": 7.229156363043744, + "learning_rate": 9.007410266389934e-07, + "logits/chosen": -0.04009784385561943, + "logits/rejected": 0.05179458111524582, + "logps/chosen": -1.2556885480880737, + "logps/rejected": -1.3836594820022583, + "loss": 2.0359, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.2556885480880737, + "rewards/margins": 0.1279708296060562, + "rewards/rejected": -1.3836594820022583, + "semantic_entropy": 0.8144699931144714, + "step": 1590 + }, + { + "epoch": 0.8536544572671015, + "grad_norm": 9.068092685958812, + "learning_rate": 8.998077280340981e-07, + "logits/chosen": 0.03531334921717644, + "logits/rejected": 0.11247481405735016, + "logps/chosen": -1.3818706274032593, + "logps/rejected": -1.414201021194458, + "loss": 2.1353, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -1.3818706274032593, + "rewards/margins": 0.0323302187025547, + "rewards/rejected": -1.414201021194458, + "semantic_entropy": 0.7828024625778198, + "step": 1595 + }, + { + "epoch": 0.8563304900485031, + "grad_norm": 6.487284359042391, + "learning_rate": 8.988705504928722e-07, + "logits/chosen": -0.07598348706960678, + "logits/rejected": 0.10973948240280151, + "logps/chosen": -1.3424265384674072, + "logps/rejected": -1.5661863088607788, + "loss": 2.0467, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.3424265384674072, + "rewards/margins": 0.223759725689888, + "rewards/rejected": -1.5661863088607788, + "semantic_entropy": 0.7749485969543457, + "step": 1600 + }, + { + "epoch": 0.8563304900485031, + "eval_logits/chosen": 0.2518528699874878, + "eval_logits/rejected": 0.33622872829437256, + "eval_logps/chosen": -1.3256367444992065, + "eval_logps/rejected": -1.5169671773910522, + "eval_loss": 2.056356430053711, + "eval_rewards/accuracies": 0.5764095187187195, + "eval_rewards/chosen": -1.3256367444992065, + "eval_rewards/margins": 0.19133047759532928, + "eval_rewards/rejected": -1.5169671773910522, + "eval_runtime": 34.6083, + "eval_samples_per_second": 38.864, + "eval_semantic_entropy": 0.7919985055923462, + "eval_steps_per_second": 9.738, + "step": 1600 + }, + { + "epoch": 0.8590065228299046, + "grad_norm": 5.214394261380477, + "learning_rate": 8.979295031078157e-07, + "logits/chosen": -0.07590194046497345, + "logits/rejected": 0.14043793082237244, + "logps/chosen": -1.2876328229904175, + "logps/rejected": -1.5351704359054565, + "loss": 2.012, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.2876328229904175, + "rewards/margins": 0.24753758311271667, + "rewards/rejected": -1.5351704359054565, + "semantic_entropy": 0.8124788403511047, + "step": 1605 + }, + { + "epoch": 0.8616825556113062, + "grad_norm": 5.950197629128667, + "learning_rate": 8.969845950089751e-07, + "logits/chosen": -0.09720132499933243, + "logits/rejected": 0.07550375908613205, + "logps/chosen": -1.269771933555603, + "logps/rejected": -1.4900038242340088, + "loss": 2.0197, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.269771933555603, + "rewards/margins": 0.22023192048072815, + "rewards/rejected": -1.4900038242340088, + "semantic_entropy": 0.8048882484436035, + "step": 1610 + }, + { + "epoch": 0.8643585883927078, + "grad_norm": 7.206569071318787, + "learning_rate": 8.960358353638526e-07, + "logits/chosen": -0.05517633631825447, + "logits/rejected": 0.043794918805360794, + "logps/chosen": -1.3460371494293213, + "logps/rejected": -1.574812889099121, + "loss": 2.0663, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.3460371494293213, + "rewards/margins": 0.2287757396697998, + "rewards/rejected": -1.574812889099121, + "semantic_entropy": 0.7761720418930054, + "step": 1615 + }, + { + "epoch": 0.8670346211741093, + "grad_norm": 6.789900956245166, + "learning_rate": 8.950832333773184e-07, + "logits/chosen": -0.002611640142276883, + "logits/rejected": 0.12673768401145935, + "logps/chosen": -1.2162271738052368, + "logps/rejected": -1.507699966430664, + "loss": 1.9731, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2162271738052368, + "rewards/margins": 0.2914729714393616, + "rewards/rejected": -1.507699966430664, + "semantic_entropy": 0.8164950609207153, + "step": 1620 + }, + { + "epoch": 0.869710653955511, + "grad_norm": 7.59597640909701, + "learning_rate": 8.941267982915213e-07, + "logits/chosen": 0.08257068693637848, + "logits/rejected": 0.1271582841873169, + "logps/chosen": -1.3797576427459717, + "logps/rejected": -1.525801658630371, + "loss": 2.0991, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.3797576427459717, + "rewards/margins": 0.14604386687278748, + "rewards/rejected": -1.525801658630371, + "semantic_entropy": 0.7759956121444702, + "step": 1625 + }, + { + "epoch": 0.8723866867369126, + "grad_norm": 8.434961396653323, + "learning_rate": 8.931665393857983e-07, + "logits/chosen": 0.0038153603672981262, + "logits/rejected": 0.15201179683208466, + "logps/chosen": -1.3320643901824951, + "logps/rejected": -1.4746291637420654, + "loss": 2.0713, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.3320643901824951, + "rewards/margins": 0.1425647884607315, + "rewards/rejected": -1.4746291637420654, + "semantic_entropy": 0.7873384952545166, + "step": 1630 + }, + { + "epoch": 0.875062719518314, + "grad_norm": 10.044887524931466, + "learning_rate": 8.922024659765861e-07, + "logits/chosen": -0.0778026208281517, + "logits/rejected": 0.035470522940158844, + "logps/chosen": -1.2441432476043701, + "logps/rejected": -1.4577016830444336, + "loss": 2.0174, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.2441432476043701, + "rewards/margins": 0.2135585993528366, + "rewards/rejected": -1.4577016830444336, + "semantic_entropy": 0.8132489919662476, + "step": 1635 + }, + { + "epoch": 0.8777387522997157, + "grad_norm": 6.639101328261095, + "learning_rate": 8.912345874173288e-07, + "logits/chosen": -0.0712386816740036, + "logits/rejected": 0.037735819816589355, + "logps/chosen": -1.2489250898361206, + "logps/rejected": -1.4933502674102783, + "loss": 2.0323, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.2489250898361206, + "rewards/margins": 0.24442513287067413, + "rewards/rejected": -1.4933502674102783, + "semantic_entropy": 0.8062291145324707, + "step": 1640 + }, + { + "epoch": 0.8804147850811173, + "grad_norm": 11.912160547547236, + "learning_rate": 8.902629130983885e-07, + "logits/chosen": 0.0011784828966483474, + "logits/rejected": 0.05601752549409866, + "logps/chosen": -1.2473455667495728, + "logps/rejected": -1.4221245050430298, + "loss": 2.0301, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2473455667495728, + "rewards/margins": 0.17477881908416748, + "rewards/rejected": -1.4221245050430298, + "semantic_entropy": 0.825102686882019, + "step": 1645 + }, + { + "epoch": 0.8830908178625189, + "grad_norm": 10.842471269722136, + "learning_rate": 8.892874524469537e-07, + "logits/chosen": 0.07691788673400879, + "logits/rejected": 0.13914301991462708, + "logps/chosen": -1.270114779472351, + "logps/rejected": -1.4903980493545532, + "loss": 2.0299, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.270114779472351, + "rewards/margins": 0.22028322517871857, + "rewards/rejected": -1.4903980493545532, + "semantic_entropy": 0.7986249327659607, + "step": 1650 + }, + { + "epoch": 0.8857668506439204, + "grad_norm": 9.732866542987194, + "learning_rate": 8.883082149269478e-07, + "logits/chosen": -0.026226142421364784, + "logits/rejected": 0.08298458904027939, + "logps/chosen": -1.3272168636322021, + "logps/rejected": -1.469519019126892, + "loss": 2.1097, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -1.3272168636322021, + "rewards/margins": 0.14230214059352875, + "rewards/rejected": -1.469519019126892, + "semantic_entropy": 0.7965654730796814, + "step": 1655 + }, + { + "epoch": 0.888442883425322, + "grad_norm": 6.200821594341999, + "learning_rate": 8.873252100389377e-07, + "logits/chosen": 0.053123731166124344, + "logits/rejected": 0.06377711892127991, + "logps/chosen": -1.256958246231079, + "logps/rejected": -1.4503594636917114, + "loss": 2.0351, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.256958246231079, + "rewards/margins": 0.19340117275714874, + "rewards/rejected": -1.4503594636917114, + "semantic_entropy": 0.8116283416748047, + "step": 1660 + }, + { + "epoch": 0.8911189162067236, + "grad_norm": 7.294249630027461, + "learning_rate": 8.863384473200411e-07, + "logits/chosen": 0.02938307449221611, + "logits/rejected": 0.09694571793079376, + "logps/chosen": -1.3356560468673706, + "logps/rejected": -1.4853121042251587, + "loss": 2.0737, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.3356560468673706, + "rewards/margins": 0.14965587854385376, + "rewards/rejected": -1.4853121042251587, + "semantic_entropy": 0.7866509556770325, + "step": 1665 + }, + { + "epoch": 0.8937949489881251, + "grad_norm": 6.133334888255327, + "learning_rate": 8.853479363438342e-07, + "logits/chosen": 0.06209275871515274, + "logits/rejected": 0.22910866141319275, + "logps/chosen": -1.360039234161377, + "logps/rejected": -1.456312894821167, + "loss": 2.1464, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.360039234161377, + "rewards/margins": 0.09627362340688705, + "rewards/rejected": -1.456312894821167, + "semantic_entropy": 0.7923996448516846, + "step": 1670 + }, + { + "epoch": 0.8964709817695267, + "grad_norm": 5.3571289027092766, + "learning_rate": 8.843536867202588e-07, + "logits/chosen": 0.058256424963474274, + "logits/rejected": 0.2694912850856781, + "logps/chosen": -1.3328102827072144, + "logps/rejected": -1.606268286705017, + "loss": 2.075, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.3328102827072144, + "rewards/margins": 0.27345791459083557, + "rewards/rejected": -1.606268286705017, + "semantic_entropy": 0.7753814458847046, + "step": 1675 + }, + { + "epoch": 0.8991470145509283, + "grad_norm": 9.514567268775329, + "learning_rate": 8.833557080955292e-07, + "logits/chosen": -0.06700204312801361, + "logits/rejected": 0.04518015310168266, + "logps/chosen": -1.3658771514892578, + "logps/rejected": -1.5113369226455688, + "loss": 2.083, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.3658771514892578, + "rewards/margins": 0.14545971155166626, + "rewards/rejected": -1.5113369226455688, + "semantic_entropy": 0.7836839556694031, + "step": 1680 + }, + { + "epoch": 0.9018230473323299, + "grad_norm": 8.79669468104146, + "learning_rate": 8.823540101520381e-07, + "logits/chosen": -0.04735895246267319, + "logits/rejected": 0.17872780561447144, + "logps/chosen": -1.3166942596435547, + "logps/rejected": -1.4989925622940063, + "loss": 2.0616, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.3166942596435547, + "rewards/margins": 0.18229825794696808, + "rewards/rejected": -1.4989925622940063, + "semantic_entropy": 0.8012421727180481, + "step": 1685 + }, + { + "epoch": 0.9044990801137314, + "grad_norm": 7.070667857103815, + "learning_rate": 8.813486026082637e-07, + "logits/chosen": -0.050979845225811005, + "logits/rejected": 0.13490286469459534, + "logps/chosen": -1.2526905536651611, + "logps/rejected": -1.4703716039657593, + "loss": 2.0054, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.2526905536651611, + "rewards/margins": 0.21768124401569366, + "rewards/rejected": -1.4703716039657593, + "semantic_entropy": 0.8110173940658569, + "step": 1690 + }, + { + "epoch": 0.907175112895133, + "grad_norm": 12.349116290155283, + "learning_rate": 8.803394952186742e-07, + "logits/chosen": -0.17546625435352325, + "logits/rejected": -0.032556891441345215, + "logps/chosen": -1.3627218008041382, + "logps/rejected": -1.4914048910140991, + "loss": 2.085, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.3627218008041382, + "rewards/margins": 0.12868306040763855, + "rewards/rejected": -1.4914048910140991, + "semantic_entropy": 0.7780641317367554, + "step": 1695 + }, + { + "epoch": 0.9098511456765346, + "grad_norm": 9.21235392234306, + "learning_rate": 8.793266977736342e-07, + "logits/chosen": -0.01133874524384737, + "logits/rejected": -0.05470942333340645, + "logps/chosen": -1.3494858741760254, + "logps/rejected": -1.4131577014923096, + "loss": 2.1016, + "rewards/accuracies": 0.48750001192092896, + "rewards/chosen": -1.3494858741760254, + "rewards/margins": 0.0636717677116394, + "rewards/rejected": -1.4131577014923096, + "semantic_entropy": 0.7917024493217468, + "step": 1700 + }, + { + "epoch": 0.9125271784579361, + "grad_norm": 8.882528510651557, + "learning_rate": 8.783102200993085e-07, + "logits/chosen": -0.002578629646450281, + "logits/rejected": 0.13472047448158264, + "logps/chosen": -1.3230177164077759, + "logps/rejected": -1.435495138168335, + "loss": 2.0797, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.3230177164077759, + "rewards/margins": 0.11247744411230087, + "rewards/rejected": -1.435495138168335, + "semantic_entropy": 0.8014957308769226, + "step": 1705 + }, + { + "epoch": 0.9152032112393377, + "grad_norm": 7.565053404244324, + "learning_rate": 8.772900720575683e-07, + "logits/chosen": -0.04457225650548935, + "logits/rejected": 0.03571139648556709, + "logps/chosen": -1.2638635635375977, + "logps/rejected": -1.428993582725525, + "loss": 2.0241, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.2638635635375977, + "rewards/margins": 0.16512994468212128, + "rewards/rejected": -1.428993582725525, + "semantic_entropy": 0.8144906163215637, + "step": 1710 + }, + { + "epoch": 0.9178792440207393, + "grad_norm": 6.65669576170742, + "learning_rate": 8.762662635458944e-07, + "logits/chosen": -0.02387130819261074, + "logits/rejected": 0.1807420551776886, + "logps/chosen": -1.3656136989593506, + "logps/rejected": -1.4667924642562866, + "loss": 2.1195, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.3656136989593506, + "rewards/margins": 0.10117874294519424, + "rewards/rejected": -1.4667924642562866, + "semantic_entropy": 0.7991411089897156, + "step": 1715 + }, + { + "epoch": 0.9205552768021408, + "grad_norm": 6.801503977545353, + "learning_rate": 8.752388044972811e-07, + "logits/chosen": 0.005317692644894123, + "logits/rejected": 0.06661224365234375, + "logps/chosen": -1.20073401927948, + "logps/rejected": -1.474448561668396, + "loss": 1.9924, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.20073401927948, + "rewards/margins": 0.2737146019935608, + "rewards/rejected": -1.474448561668396, + "semantic_entropy": 0.8212075233459473, + "step": 1720 + }, + { + "epoch": 0.9232313095835424, + "grad_norm": 6.786794245858191, + "learning_rate": 8.74207704880141e-07, + "logits/chosen": -0.014214864000678062, + "logits/rejected": 0.08873619884252548, + "logps/chosen": -1.3316367864608765, + "logps/rejected": -1.5858074426651, + "loss": 2.0367, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.3316367864608765, + "rewards/margins": 0.2541707754135132, + "rewards/rejected": -1.5858074426651, + "semantic_entropy": 0.7817917466163635, + "step": 1725 + }, + { + "epoch": 0.925907342364944, + "grad_norm": 9.01368242373458, + "learning_rate": 8.731729746982068e-07, + "logits/chosen": 0.07874997705221176, + "logits/rejected": 0.1306711733341217, + "logps/chosen": -1.3039653301239014, + "logps/rejected": -1.4204844236373901, + "loss": 2.0723, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.3039653301239014, + "rewards/margins": 0.11651904881000519, + "rewards/rejected": -1.4204844236373901, + "semantic_entropy": 0.7985782027244568, + "step": 1730 + }, + { + "epoch": 0.9285833751463456, + "grad_norm": 7.397503812781015, + "learning_rate": 8.721346239904355e-07, + "logits/chosen": -0.09926575422286987, + "logits/rejected": 0.058978479355573654, + "logps/chosen": -1.2411620616912842, + "logps/rejected": -1.6494128704071045, + "loss": 1.957, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2411620616912842, + "rewards/margins": 0.40825071930885315, + "rewards/rejected": -1.6494128704071045, + "semantic_entropy": 0.801047146320343, + "step": 1735 + }, + { + "epoch": 0.9312594079277471, + "grad_norm": 7.049995899457256, + "learning_rate": 8.710926628309101e-07, + "logits/chosen": -0.020289301872253418, + "logits/rejected": 0.11928510665893555, + "logps/chosen": -1.2865359783172607, + "logps/rejected": -1.4603403806686401, + "loss": 2.0347, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.2865359783172607, + "rewards/margins": 0.17380447685718536, + "rewards/rejected": -1.4603403806686401, + "semantic_entropy": 0.8009761571884155, + "step": 1740 + }, + { + "epoch": 0.9339354407091487, + "grad_norm": 6.429018945726928, + "learning_rate": 8.700471013287424e-07, + "logits/chosen": 0.053223930299282074, + "logits/rejected": 0.06973040848970413, + "logps/chosen": -1.2947371006011963, + "logps/rejected": -1.4536296129226685, + "loss": 2.0479, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2947371006011963, + "rewards/margins": 0.1588924676179886, + "rewards/rejected": -1.4536296129226685, + "semantic_entropy": 0.8022210001945496, + "step": 1745 + }, + { + "epoch": 0.9366114734905503, + "grad_norm": 10.83261920976737, + "learning_rate": 8.689979496279746e-07, + "logits/chosen": -0.05656473711133003, + "logits/rejected": 0.0013455540174618363, + "logps/chosen": -1.3060810565948486, + "logps/rejected": -1.5460128784179688, + "loss": 2.0247, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.3060810565948486, + "rewards/margins": 0.2399318963289261, + "rewards/rejected": -1.5460128784179688, + "semantic_entropy": 0.7921985387802124, + "step": 1750 + }, + { + "epoch": 0.9392875062719518, + "grad_norm": 5.858542388966641, + "learning_rate": 8.679452179074811e-07, + "logits/chosen": -0.03769116848707199, + "logits/rejected": 0.05500766634941101, + "logps/chosen": -1.2704050540924072, + "logps/rejected": -1.412957787513733, + "loss": 2.0511, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.2704050540924072, + "rewards/margins": 0.14255282282829285, + "rewards/rejected": -1.412957787513733, + "semantic_entropy": 0.8166753053665161, + "step": 1755 + }, + { + "epoch": 0.9419635390533534, + "grad_norm": 5.872711066743105, + "learning_rate": 8.668889163808698e-07, + "logits/chosen": -0.010468644089996815, + "logits/rejected": 0.11073984205722809, + "logps/chosen": -1.2528936862945557, + "logps/rejected": -1.4464694261550903, + "loss": 2.0165, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2528936862945557, + "rewards/margins": 0.1935756951570511, + "rewards/rejected": -1.4464694261550903, + "semantic_entropy": 0.8102561235427856, + "step": 1760 + }, + { + "epoch": 0.944639571834755, + "grad_norm": 6.529911274017464, + "learning_rate": 8.658290552963827e-07, + "logits/chosen": -0.0010960951913148165, + "logits/rejected": 0.01580260880291462, + "logps/chosen": -1.2851160764694214, + "logps/rejected": -1.490731954574585, + "loss": 2.0604, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2851160764694214, + "rewards/margins": 0.20561587810516357, + "rewards/rejected": -1.490731954574585, + "semantic_entropy": 0.8163287043571472, + "step": 1765 + }, + { + "epoch": 0.9473156046161565, + "grad_norm": 5.639046399121027, + "learning_rate": 8.647656449367966e-07, + "logits/chosen": -0.009508803486824036, + "logits/rejected": 0.13738605380058289, + "logps/chosen": -1.3116872310638428, + "logps/rejected": -1.4082788228988647, + "loss": 2.0579, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.3116872310638428, + "rewards/margins": 0.09659181535243988, + "rewards/rejected": -1.4082788228988647, + "semantic_entropy": 0.7913161516189575, + "step": 1770 + }, + { + "epoch": 0.9499916373975581, + "grad_norm": 9.492998520557995, + "learning_rate": 8.636986956193235e-07, + "logits/chosen": -0.060478221625089645, + "logits/rejected": 0.026750242337584496, + "logps/chosen": -1.262905478477478, + "logps/rejected": -1.4266475439071655, + "loss": 2.0257, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.262905478477478, + "rewards/margins": 0.16374202072620392, + "rewards/rejected": -1.4266475439071655, + "semantic_entropy": 0.8194987177848816, + "step": 1775 + }, + { + "epoch": 0.9526676701789597, + "grad_norm": 6.987088294233002, + "learning_rate": 8.626282176955104e-07, + "logits/chosen": -0.061791181564331055, + "logits/rejected": 0.05312754586338997, + "logps/chosen": -1.2885668277740479, + "logps/rejected": -1.507681131362915, + "loss": 2.0293, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.2885668277740479, + "rewards/margins": 0.2191143035888672, + "rewards/rejected": -1.507681131362915, + "semantic_entropy": 0.8043468594551086, + "step": 1780 + }, + { + "epoch": 0.9553437029603613, + "grad_norm": 7.2172640331427695, + "learning_rate": 8.615542215511389e-07, + "logits/chosen": 0.04582851380109787, + "logits/rejected": 0.11913655698299408, + "logps/chosen": -1.2383428812026978, + "logps/rejected": -1.3574765920639038, + "loss": 2.058, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.2383428812026978, + "rewards/margins": 0.11913367360830307, + "rewards/rejected": -1.3574765920639038, + "semantic_entropy": 0.8183929324150085, + "step": 1785 + }, + { + "epoch": 0.9580197357417628, + "grad_norm": 8.066411676659767, + "learning_rate": 8.604767176061241e-07, + "logits/chosen": 0.0605916753411293, + "logits/rejected": 0.10191599279642105, + "logps/chosen": -1.311759114265442, + "logps/rejected": -1.4463961124420166, + "loss": 2.0579, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -1.311759114265442, + "rewards/margins": 0.13463714718818665, + "rewards/rejected": -1.4463961124420166, + "semantic_entropy": 0.7989506721496582, + "step": 1790 + }, + { + "epoch": 0.9606957685231644, + "grad_norm": 7.885711535048902, + "learning_rate": 8.593957163144141e-07, + "logits/chosen": -0.08106566965579987, + "logits/rejected": 0.053579796105623245, + "logps/chosen": -1.2748843431472778, + "logps/rejected": -1.4692093133926392, + "loss": 2.0301, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2748843431472778, + "rewards/margins": 0.19432485103607178, + "rewards/rejected": -1.4692093133926392, + "semantic_entropy": 0.8063615560531616, + "step": 1795 + }, + { + "epoch": 0.963371801304566, + "grad_norm": 5.108281485121351, + "learning_rate": 8.58311228163888e-07, + "logits/chosen": 0.007795440498739481, + "logits/rejected": 0.08127208799123764, + "logps/chosen": -1.273447871208191, + "logps/rejected": -1.4030935764312744, + "loss": 2.0492, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.273447871208191, + "rewards/margins": 0.12964563071727753, + "rewards/rejected": -1.4030935764312744, + "semantic_entropy": 0.8049715757369995, + "step": 1800 + }, + { + "epoch": 0.9660478340859675, + "grad_norm": 6.028916082322847, + "learning_rate": 8.57223263676255e-07, + "logits/chosen": -0.12820856273174286, + "logits/rejected": -0.002626272616907954, + "logps/chosen": -1.2371407747268677, + "logps/rejected": -1.5107574462890625, + "loss": 1.9814, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.2371407747268677, + "rewards/margins": 0.2736164927482605, + "rewards/rejected": -1.5107574462890625, + "semantic_entropy": 0.8165399432182312, + "step": 1805 + }, + { + "epoch": 0.9687238668673691, + "grad_norm": 7.801922833669752, + "learning_rate": 8.561318334069511e-07, + "logits/chosen": -0.01773199997842312, + "logits/rejected": 0.11925344169139862, + "logps/chosen": -1.2662723064422607, + "logps/rejected": -1.418873906135559, + "loss": 2.0459, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.2662723064422607, + "rewards/margins": 0.15260140597820282, + "rewards/rejected": -1.418873906135559, + "semantic_entropy": 0.8175581097602844, + "step": 1810 + }, + { + "epoch": 0.9713998996487707, + "grad_norm": 6.630422780032403, + "learning_rate": 8.550369479450375e-07, + "logits/chosen": -0.039072971791028976, + "logits/rejected": 0.09387413412332535, + "logps/chosen": -1.2889964580535889, + "logps/rejected": -1.4401779174804688, + "loss": 2.0326, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2889964580535889, + "rewards/margins": 0.15118160843849182, + "rewards/rejected": -1.4401779174804688, + "semantic_entropy": 0.8038996458053589, + "step": 1815 + }, + { + "epoch": 0.9740759324301723, + "grad_norm": 7.132366412010857, + "learning_rate": 8.539386179130977e-07, + "logits/chosen": -0.010481951758265495, + "logits/rejected": 0.05843520164489746, + "logps/chosen": -1.2933626174926758, + "logps/rejected": -1.4336893558502197, + "loss": 2.0552, + "rewards/accuracies": 0.53125, + "rewards/chosen": -1.2933626174926758, + "rewards/margins": 0.1403266340494156, + "rewards/rejected": -1.4336893558502197, + "semantic_entropy": 0.8076707124710083, + "step": 1820 + }, + { + "epoch": 0.9767519652115738, + "grad_norm": 8.28316100471642, + "learning_rate": 8.528368539671347e-07, + "logits/chosen": -0.06624146550893784, + "logits/rejected": 0.08388527482748032, + "logps/chosen": -1.2708795070648193, + "logps/rejected": -1.5188466310501099, + "loss": 2.0311, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.2708795070648193, + "rewards/margins": 0.24796704947948456, + "rewards/rejected": -1.5188466310501099, + "semantic_entropy": 0.8095704913139343, + "step": 1825 + }, + { + "epoch": 0.9794279979929754, + "grad_norm": 5.971238453093119, + "learning_rate": 8.51731666796467e-07, + "logits/chosen": 0.1276635229587555, + "logits/rejected": 0.15978917479515076, + "logps/chosen": -1.339593768119812, + "logps/rejected": -1.4184544086456299, + "loss": 2.0979, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -1.339593768119812, + "rewards/margins": 0.07886041700839996, + "rewards/rejected": -1.4184544086456299, + "semantic_entropy": 0.7924166917800903, + "step": 1830 + }, + { + "epoch": 0.982104030774377, + "grad_norm": 6.927656624192063, + "learning_rate": 8.506230671236254e-07, + "logits/chosen": -0.013933452777564526, + "logits/rejected": 0.051707565784454346, + "logps/chosen": -1.3016908168792725, + "logps/rejected": -1.355043649673462, + "loss": 2.08, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.3016908168792725, + "rewards/margins": 0.05335277318954468, + "rewards/rejected": -1.355043649673462, + "semantic_entropy": 0.8091516494750977, + "step": 1835 + }, + { + "epoch": 0.9847800635557785, + "grad_norm": 6.708651387780945, + "learning_rate": 8.495110657042488e-07, + "logits/chosen": 0.022884510457515717, + "logits/rejected": 0.12576675415039062, + "logps/chosen": -1.3016811609268188, + "logps/rejected": -1.5378698110580444, + "loss": 2.0291, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.3016811609268188, + "rewards/margins": 0.23618856072425842, + "rewards/rejected": -1.5378698110580444, + "semantic_entropy": 0.8028618097305298, + "step": 1840 + }, + { + "epoch": 0.9874560963371801, + "grad_norm": 8.94078484600404, + "learning_rate": 8.483956733269799e-07, + "logits/chosen": -0.034597523510456085, + "logits/rejected": 0.05511583760380745, + "logps/chosen": -1.322399377822876, + "logps/rejected": -1.4499645233154297, + "loss": 2.0718, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.322399377822876, + "rewards/margins": 0.1275651454925537, + "rewards/rejected": -1.4499645233154297, + "semantic_entropy": 0.7913865447044373, + "step": 1845 + }, + { + "epoch": 0.9901321291185817, + "grad_norm": 7.903504654248402, + "learning_rate": 8.472769008133602e-07, + "logits/chosen": -0.1613004505634308, + "logits/rejected": -0.03331545740365982, + "logps/chosen": -1.3431861400604248, + "logps/rejected": -1.3820966482162476, + "loss": 2.1119, + "rewards/accuracies": 0.5062500238418579, + "rewards/chosen": -1.3431861400604248, + "rewards/margins": 0.03891069442033768, + "rewards/rejected": -1.3820966482162476, + "semantic_entropy": 0.8067256808280945, + "step": 1850 + }, + { + "epoch": 0.9928081618999832, + "grad_norm": 6.836428084610912, + "learning_rate": 8.461547590177259e-07, + "logits/chosen": -0.02012103796005249, + "logits/rejected": 0.08856208622455597, + "logps/chosen": -1.2511141300201416, + "logps/rejected": -1.4271875619888306, + "loss": 2.032, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.2511141300201416, + "rewards/margins": 0.17607346177101135, + "rewards/rejected": -1.4271875619888306, + "semantic_entropy": 0.80317622423172, + "step": 1855 + }, + { + "epoch": 0.9954841946813848, + "grad_norm": 8.191047160297591, + "learning_rate": 8.450292588271014e-07, + "logits/chosen": -0.010422793217003345, + "logits/rejected": 0.07045050710439682, + "logps/chosen": -1.3515708446502686, + "logps/rejected": -1.4826161861419678, + "loss": 2.0701, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.3515708446502686, + "rewards/margins": 0.13104534149169922, + "rewards/rejected": -1.4826161861419678, + "semantic_entropy": 0.7868179082870483, + "step": 1860 + }, + { + "epoch": 0.9981602274627864, + "grad_norm": 7.277078462725191, + "learning_rate": 8.439004111610945e-07, + "logits/chosen": -0.04057621955871582, + "logits/rejected": 0.03624313324689865, + "logps/chosen": -1.207397699356079, + "logps/rejected": -1.4739655256271362, + "loss": 1.989, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.207397699356079, + "rewards/margins": 0.2665678858757019, + "rewards/rejected": -1.4739655256271362, + "semantic_entropy": 0.8229459524154663, + "step": 1865 + }, + { + "epoch": 1.000836260244188, + "grad_norm": 7.269625131401338, + "learning_rate": 8.427682269717901e-07, + "logits/chosen": -0.061928071081638336, + "logits/rejected": 0.09646090120077133, + "logps/chosen": -1.3524137735366821, + "logps/rejected": -1.469217300415039, + "loss": 2.0911, + "rewards/accuracies": 0.48750001192092896, + "rewards/chosen": -1.3524137735366821, + "rewards/margins": 0.11680366098880768, + "rewards/rejected": -1.469217300415039, + "semantic_entropy": 0.7895178198814392, + "step": 1870 + }, + { + "epoch": 1.0035122930255895, + "grad_norm": 6.234940546892951, + "learning_rate": 8.416327172436446e-07, + "logits/chosen": -0.0901242271065712, + "logits/rejected": 0.04721410945057869, + "logps/chosen": -1.3372011184692383, + "logps/rejected": -1.4556587934494019, + "loss": 2.0741, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.3372011184692383, + "rewards/margins": 0.1184576004743576, + "rewards/rejected": -1.4556587934494019, + "semantic_entropy": 0.8036667704582214, + "step": 1875 + }, + { + "epoch": 1.0061883258069912, + "grad_norm": 10.862639044006587, + "learning_rate": 8.404938929933778e-07, + "logits/chosen": 0.009253310039639473, + "logits/rejected": 0.1762274205684662, + "logps/chosen": -1.2881697416305542, + "logps/rejected": -1.610016107559204, + "loss": 2.0204, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.2881697416305542, + "rewards/margins": 0.3218461871147156, + "rewards/rejected": -1.610016107559204, + "semantic_entropy": 0.7875820398330688, + "step": 1880 + }, + { + "epoch": 1.0088643585883927, + "grad_norm": 7.426144032081365, + "learning_rate": 8.39351765269868e-07, + "logits/chosen": -0.04300857335329056, + "logits/rejected": 0.03298963978886604, + "logps/chosen": -1.2392830848693848, + "logps/rejected": -1.4619297981262207, + "loss": 2.0175, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.2392830848693848, + "rewards/margins": 0.2226467877626419, + "rewards/rejected": -1.4619297981262207, + "semantic_entropy": 0.8182765245437622, + "step": 1885 + }, + { + "epoch": 1.0115403913697942, + "grad_norm": 6.766335194681512, + "learning_rate": 8.382063451540431e-07, + "logits/chosen": -0.025482019409537315, + "logits/rejected": 0.16996082663536072, + "logps/chosen": -1.2850615978240967, + "logps/rejected": -1.4646522998809814, + "loss": 2.07, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.2850615978240967, + "rewards/margins": 0.17959053814411163, + "rewards/rejected": -1.4646522998809814, + "semantic_entropy": 0.7993168830871582, + "step": 1890 + }, + { + "epoch": 1.014216424151196, + "grad_norm": 8.042008089739506, + "learning_rate": 8.370576437587742e-07, + "logits/chosen": 0.036351293325424194, + "logits/rejected": 0.09470394253730774, + "logps/chosen": -1.2692261934280396, + "logps/rejected": -1.434579610824585, + "loss": 2.0235, + "rewards/accuracies": 0.65625, + "rewards/chosen": -1.2692261934280396, + "rewards/margins": 0.16535338759422302, + "rewards/rejected": -1.434579610824585, + "semantic_entropy": 0.809338390827179, + "step": 1895 + }, + { + "epoch": 1.0168924569325974, + "grad_norm": 6.394105938084402, + "learning_rate": 8.359056722287674e-07, + "logits/chosen": -0.10826126486063004, + "logits/rejected": 0.14603324234485626, + "logps/chosen": -1.3070743083953857, + "logps/rejected": -1.4380731582641602, + "loss": 2.063, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.3070743083953857, + "rewards/margins": 0.13099884986877441, + "rewards/rejected": -1.4380731582641602, + "semantic_entropy": 0.7941070795059204, + "step": 1900 + }, + { + "epoch": 1.019568489713999, + "grad_norm": 6.349824057450553, + "learning_rate": 8.347504417404553e-07, + "logits/chosen": -0.007749214768409729, + "logits/rejected": 0.13887669146060944, + "logps/chosen": -1.3314285278320312, + "logps/rejected": -1.4583871364593506, + "loss": 2.0747, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.3314285278320312, + "rewards/margins": 0.12695881724357605, + "rewards/rejected": -1.4583871364593506, + "semantic_entropy": 0.791080117225647, + "step": 1905 + }, + { + "epoch": 1.0222445224954007, + "grad_norm": 7.4786128948638515, + "learning_rate": 8.335919635018893e-07, + "logits/chosen": -0.10085531324148178, + "logits/rejected": 0.018569733947515488, + "logps/chosen": -1.2909607887268066, + "logps/rejected": -1.4696437120437622, + "loss": 2.0521, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.2909607887268066, + "rewards/margins": 0.1786828637123108, + "rewards/rejected": -1.4696437120437622, + "semantic_entropy": 0.791267454624176, + "step": 1910 + }, + { + "epoch": 1.0249205552768021, + "grad_norm": 5.803922148585013, + "learning_rate": 8.324302487526303e-07, + "logits/chosen": -0.04810156300663948, + "logits/rejected": 0.024439021944999695, + "logps/chosen": -1.2567697763442993, + "logps/rejected": -1.391143560409546, + "loss": 2.035, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.2567697763442993, + "rewards/margins": 0.13437364995479584, + "rewards/rejected": -1.391143560409546, + "semantic_entropy": 0.8175903558731079, + "step": 1915 + }, + { + "epoch": 1.0275965880582036, + "grad_norm": 6.84748741629461, + "learning_rate": 8.312653087636398e-07, + "logits/chosen": -0.08552031219005585, + "logits/rejected": -0.009279889054596424, + "logps/chosen": -1.1754251718521118, + "logps/rejected": -1.4189541339874268, + "loss": 1.956, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.1754251718521118, + "rewards/margins": 0.24352887272834778, + "rewards/rejected": -1.4189541339874268, + "semantic_entropy": 0.8310561180114746, + "step": 1920 + }, + { + "epoch": 1.0302726208396054, + "grad_norm": 7.9402409829710425, + "learning_rate": 8.300971548371711e-07, + "logits/chosen": -0.2018776386976242, + "logits/rejected": 0.003242161823436618, + "logps/chosen": -1.3566595315933228, + "logps/rejected": -1.4750354290008545, + "loss": 2.0812, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.3566595315933228, + "rewards/margins": 0.11837591975927353, + "rewards/rejected": -1.4750354290008545, + "semantic_entropy": 0.7911876440048218, + "step": 1925 + }, + { + "epoch": 1.0329486536210069, + "grad_norm": 7.251246592815628, + "learning_rate": 8.289257983066582e-07, + "logits/chosen": -0.1044299378991127, + "logits/rejected": 0.03499078005552292, + "logps/chosen": -1.2216999530792236, + "logps/rejected": -1.4600203037261963, + "loss": 1.9767, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.2216999530792236, + "rewards/margins": 0.23832039535045624, + "rewards/rejected": -1.4600203037261963, + "semantic_entropy": 0.8182482719421387, + "step": 1930 + }, + { + "epoch": 1.0356246864024083, + "grad_norm": 8.134051978549717, + "learning_rate": 8.277512505366077e-07, + "logits/chosen": -0.1392371654510498, + "logits/rejected": 0.031945519149303436, + "logps/chosen": -1.2987267971038818, + "logps/rejected": -1.5219374895095825, + "loss": 2.0236, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.2987267971038818, + "rewards/margins": 0.22321060299873352, + "rewards/rejected": -1.5219374895095825, + "semantic_entropy": 0.8038732409477234, + "step": 1935 + }, + { + "epoch": 1.03830071918381, + "grad_norm": 8.837560923069026, + "learning_rate": 8.265735229224868e-07, + "logits/chosen": -0.07545419037342072, + "logits/rejected": 0.03142388537526131, + "logps/chosen": -1.3016760349273682, + "logps/rejected": -1.5333623886108398, + "loss": 2.0343, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.3016760349273682, + "rewards/margins": 0.23168623447418213, + "rewards/rejected": -1.5333623886108398, + "semantic_entropy": 0.7905539870262146, + "step": 1940 + }, + { + "epoch": 1.0409767519652116, + "grad_norm": 13.135999992185194, + "learning_rate": 8.253926268906144e-07, + "logits/chosen": -0.14592385292053223, + "logits/rejected": 0.019411101937294006, + "logps/chosen": -1.2989904880523682, + "logps/rejected": -1.5625553131103516, + "loss": 2.0003, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.2989904880523682, + "rewards/margins": 0.2635648250579834, + "rewards/rejected": -1.5625553131103516, + "semantic_entropy": 0.7920844554901123, + "step": 1945 + }, + { + "epoch": 1.043652784746613, + "grad_norm": 5.779607975582833, + "learning_rate": 8.242085738980487e-07, + "logits/chosen": -0.06912654638290405, + "logits/rejected": 0.13862493634223938, + "logps/chosen": -1.3420753479003906, + "logps/rejected": -1.5418751239776611, + "loss": 2.0447, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.3420753479003906, + "rewards/margins": 0.19979973137378693, + "rewards/rejected": -1.5418751239776611, + "semantic_entropy": 0.7756515741348267, + "step": 1950 + }, + { + "epoch": 1.0463288175280148, + "grad_norm": 7.5077098873095665, + "learning_rate": 8.230213754324772e-07, + "logits/chosen": -0.013625724241137505, + "logits/rejected": 0.055890560150146484, + "logps/chosen": -1.2228213548660278, + "logps/rejected": -1.4776057004928589, + "loss": 1.97, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.2228213548660278, + "rewards/margins": 0.2547842860221863, + "rewards/rejected": -1.4776057004928589, + "semantic_entropy": 0.8201814889907837, + "step": 1955 + }, + { + "epoch": 1.0490048503094163, + "grad_norm": 7.321166037812535, + "learning_rate": 8.218310430121045e-07, + "logits/chosen": -0.07903581112623215, + "logits/rejected": -0.04870419204235077, + "logps/chosen": -1.2837450504302979, + "logps/rejected": -1.4462873935699463, + "loss": 2.0399, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2837450504302979, + "rewards/margins": 0.1625424325466156, + "rewards/rejected": -1.4462873935699463, + "semantic_entropy": 0.789503276348114, + "step": 1960 + }, + { + "epoch": 1.051680883090818, + "grad_norm": 7.931221556919084, + "learning_rate": 8.20637588185541e-07, + "logits/chosen": 0.03140250965952873, + "logits/rejected": 0.10349716991186142, + "logps/chosen": -1.226862907409668, + "logps/rejected": -1.5837271213531494, + "loss": 1.942, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.226862907409668, + "rewards/margins": 0.35686445236206055, + "rewards/rejected": -1.5837271213531494, + "semantic_entropy": 0.8064924478530884, + "step": 1965 + }, + { + "epoch": 1.0543569158722195, + "grad_norm": 8.900927053864605, + "learning_rate": 8.194410225316906e-07, + "logits/chosen": -0.04091443866491318, + "logits/rejected": 0.10504456609487534, + "logps/chosen": -1.2857110500335693, + "logps/rejected": -1.512974500656128, + "loss": 2.0268, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.2857110500335693, + "rewards/margins": 0.2272634208202362, + "rewards/rejected": -1.512974500656128, + "semantic_entropy": 0.7864609956741333, + "step": 1970 + }, + { + "epoch": 1.057032948653621, + "grad_norm": 7.377180436735642, + "learning_rate": 8.182413576596385e-07, + "logits/chosen": 0.031790219247341156, + "logits/rejected": 0.12696322798728943, + "logps/chosen": -1.224799394607544, + "logps/rejected": -1.4564673900604248, + "loss": 1.9856, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.224799394607544, + "rewards/margins": 0.23166804015636444, + "rewards/rejected": -1.4564673900604248, + "semantic_entropy": 0.8123728036880493, + "step": 1975 + }, + { + "epoch": 1.0597089814350227, + "grad_norm": 8.157139086496546, + "learning_rate": 8.170386052085389e-07, + "logits/chosen": 0.032434239983558655, + "logits/rejected": 0.1504647582769394, + "logps/chosen": -1.2983589172363281, + "logps/rejected": -1.518869400024414, + "loss": 2.0384, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.2983589172363281, + "rewards/margins": 0.2205105721950531, + "rewards/rejected": -1.518869400024414, + "semantic_entropy": 0.7908387184143066, + "step": 1980 + }, + { + "epoch": 1.0623850142164242, + "grad_norm": 7.536876492898136, + "learning_rate": 8.158327768475008e-07, + "logits/chosen": -0.02274470403790474, + "logits/rejected": 0.13517804443836212, + "logps/chosen": -1.3373349905014038, + "logps/rejected": -1.4611749649047852, + "loss": 2.0847, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.3373349905014038, + "rewards/margins": 0.1238398551940918, + "rewards/rejected": -1.4611749649047852, + "semantic_entropy": 0.782866358757019, + "step": 1985 + }, + { + "epoch": 1.0650610469978257, + "grad_norm": 7.9100389576496, + "learning_rate": 8.146238842754767e-07, + "logits/chosen": -0.04121997952461243, + "logits/rejected": 0.05382202938199043, + "logps/chosen": -1.326257586479187, + "logps/rejected": -1.5107661485671997, + "loss": 2.0481, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.326257586479187, + "rewards/margins": 0.18450842797756195, + "rewards/rejected": -1.5107661485671997, + "semantic_entropy": 0.7959458827972412, + "step": 1990 + }, + { + "epoch": 1.0677370797792274, + "grad_norm": 7.393372917797862, + "learning_rate": 8.134119392211476e-07, + "logits/chosen": 0.04782380536198616, + "logits/rejected": 0.19727477431297302, + "logps/chosen": -1.2395175695419312, + "logps/rejected": -1.5628464221954346, + "loss": 1.9818, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2395175695419312, + "rewards/margins": 0.3233289122581482, + "rewards/rejected": -1.5628464221954346, + "semantic_entropy": 0.8064082860946655, + "step": 1995 + }, + { + "epoch": 1.0704131125606289, + "grad_norm": 15.049418095336085, + "learning_rate": 8.121969534428094e-07, + "logits/chosen": -0.048318177461624146, + "logits/rejected": 0.09915760904550552, + "logps/chosen": -1.3366820812225342, + "logps/rejected": -1.4741013050079346, + "loss": 2.1246, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -1.3366820812225342, + "rewards/margins": 0.13741900026798248, + "rewards/rejected": -1.4741013050079346, + "semantic_entropy": 0.799536943435669, + "step": 2000 + }, + { + "epoch": 1.0704131125606289, + "eval_logits/chosen": 0.36331358551979065, + "eval_logits/rejected": 0.4596547782421112, + "eval_logps/chosen": -1.3185980319976807, + "eval_logps/rejected": -1.5339993238449097, + "eval_loss": 2.0504531860351562, + "eval_rewards/accuracies": 0.577151358127594, + "eval_rewards/chosen": -1.3185980319976807, + "eval_rewards/margins": 0.21540121734142303, + "eval_rewards/rejected": -1.5339993238449097, + "eval_runtime": 34.7693, + "eval_samples_per_second": 38.684, + "eval_semantic_entropy": 0.7883589863777161, + "eval_steps_per_second": 9.692, + "step": 2000 + }, + { + "epoch": 1.0730891453420304, + "grad_norm": 9.046281060813591, + "learning_rate": 8.109789387282599e-07, + "logits/chosen": 0.03845970332622528, + "logits/rejected": 0.11954043060541153, + "logps/chosen": -1.3330227136611938, + "logps/rejected": -1.4478117227554321, + "loss": 2.0881, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.3330227136611938, + "rewards/margins": 0.11478898674249649, + "rewards/rejected": -1.4478117227554321, + "semantic_entropy": 0.805001437664032, + "step": 2005 + }, + { + "epoch": 1.075765178123432, + "grad_norm": 10.256124217400894, + "learning_rate": 8.097579068946827e-07, + "logits/chosen": 0.04520503804087639, + "logits/rejected": 0.15148118138313293, + "logps/chosen": -1.2535759210586548, + "logps/rejected": -1.433410882949829, + "loss": 2.0351, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.2535759210586548, + "rewards/margins": 0.17983505129814148, + "rewards/rejected": -1.433410882949829, + "semantic_entropy": 0.8159645199775696, + "step": 2010 + }, + { + "epoch": 1.0784412109048336, + "grad_norm": 7.344453328822589, + "learning_rate": 8.085338697885344e-07, + "logits/chosen": -0.02262263372540474, + "logits/rejected": 0.12338912487030029, + "logps/chosen": -1.2342798709869385, + "logps/rejected": -1.444035530090332, + "loss": 2.0145, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2342798709869385, + "rewards/margins": 0.20975565910339355, + "rewards/rejected": -1.444035530090332, + "semantic_entropy": 0.8241077661514282, + "step": 2015 + }, + { + "epoch": 1.081117243686235, + "grad_norm": 7.496592903248984, + "learning_rate": 8.073068392854282e-07, + "logits/chosen": -0.049902092665433884, + "logits/rejected": 0.13317637145519257, + "logps/chosen": -1.325439691543579, + "logps/rejected": -1.5331872701644897, + "loss": 2.0448, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.325439691543579, + "rewards/margins": 0.2077476978302002, + "rewards/rejected": -1.5331872701644897, + "semantic_entropy": 0.7916399240493774, + "step": 2020 + }, + { + "epoch": 1.0837932764676368, + "grad_norm": 6.372316019210651, + "learning_rate": 8.060768272900193e-07, + "logits/chosen": 0.04581315070390701, + "logits/rejected": 0.1767156422138214, + "logps/chosen": -1.2968662977218628, + "logps/rejected": -1.532001256942749, + "loss": 2.024, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2968662977218628, + "rewards/margins": 0.2351350039243698, + "rewards/rejected": -1.532001256942749, + "semantic_entropy": 0.7861354351043701, + "step": 2025 + }, + { + "epoch": 1.0864693092490383, + "grad_norm": 6.788766807266465, + "learning_rate": 8.0484384573589e-07, + "logits/chosen": -0.04738964885473251, + "logits/rejected": -0.025576937943696976, + "logps/chosen": -1.2473602294921875, + "logps/rejected": -1.433471918106079, + "loss": 2.0237, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.2473602294921875, + "rewards/margins": 0.18611164391040802, + "rewards/rejected": -1.433471918106079, + "semantic_entropy": 0.8121569752693176, + "step": 2030 + }, + { + "epoch": 1.0891453420304398, + "grad_norm": 7.346242709370745, + "learning_rate": 8.03607906585432e-07, + "logits/chosen": -0.06825561821460724, + "logits/rejected": 0.0949849784374237, + "logps/chosen": -1.247241735458374, + "logps/rejected": -1.4357866048812866, + "loss": 2.0112, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.247241735458374, + "rewards/margins": 0.1885448694229126, + "rewards/rejected": -1.4357866048812866, + "semantic_entropy": 0.8164774179458618, + "step": 2035 + }, + { + "epoch": 1.0918213748118415, + "grad_norm": 9.962051708647408, + "learning_rate": 8.023690218297329e-07, + "logits/chosen": -0.1509312242269516, + "logits/rejected": -0.08851532638072968, + "logps/chosen": -1.274482011795044, + "logps/rejected": -1.4187238216400146, + "loss": 2.0289, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.274482011795044, + "rewards/margins": 0.14424176514148712, + "rewards/rejected": -1.4187238216400146, + "semantic_entropy": 0.7936094403266907, + "step": 2040 + }, + { + "epoch": 1.094497407593243, + "grad_norm": 11.622046675876497, + "learning_rate": 8.01127203488458e-07, + "logits/chosen": -0.0111696757376194, + "logits/rejected": 0.03334783762693405, + "logps/chosen": -1.2602503299713135, + "logps/rejected": -1.4635040760040283, + "loss": 2.025, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2602503299713135, + "rewards/margins": 0.20325377583503723, + "rewards/rejected": -1.4635040760040283, + "semantic_entropy": 0.8120501637458801, + "step": 2045 + }, + { + "epoch": 1.0971734403746445, + "grad_norm": 9.748815999988883, + "learning_rate": 7.998824636097339e-07, + "logits/chosen": -0.12438543885946274, + "logits/rejected": -0.0023122006095945835, + "logps/chosen": -1.3013404607772827, + "logps/rejected": -1.4263468980789185, + "loss": 2.0826, + "rewards/accuracies": 0.53125, + "rewards/chosen": -1.3013404607772827, + "rewards/margins": 0.12500649690628052, + "rewards/rejected": -1.4263468980789185, + "semantic_entropy": 0.7970176339149475, + "step": 2050 + }, + { + "epoch": 1.0998494731560462, + "grad_norm": 8.91959075076058, + "learning_rate": 7.986348142700328e-07, + "logits/chosen": -0.023914365097880363, + "logits/rejected": 0.09678036719560623, + "logps/chosen": -1.2690775394439697, + "logps/rejected": -1.427825689315796, + "loss": 2.0212, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.2690775394439697, + "rewards/margins": 0.15874823927879333, + "rewards/rejected": -1.427825689315796, + "semantic_entropy": 0.8028534054756165, + "step": 2055 + }, + { + "epoch": 1.1025255059374477, + "grad_norm": 9.230908880998141, + "learning_rate": 7.973842675740539e-07, + "logits/chosen": -0.013653213158249855, + "logits/rejected": 0.03546537458896637, + "logps/chosen": -1.3021272420883179, + "logps/rejected": -1.5597717761993408, + "loss": 2.0238, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.3021272420883179, + "rewards/margins": 0.257644385099411, + "rewards/rejected": -1.5597717761993408, + "semantic_entropy": 0.7928792238235474, + "step": 2060 + }, + { + "epoch": 1.1052015387188494, + "grad_norm": 8.470596064865148, + "learning_rate": 7.961308356546066e-07, + "logits/chosen": -0.05723114684224129, + "logits/rejected": 0.06854735314846039, + "logps/chosen": -1.2799628973007202, + "logps/rejected": -1.4498138427734375, + "loss": 2.0457, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.2799628973007202, + "rewards/margins": 0.16985103487968445, + "rewards/rejected": -1.4498138427734375, + "semantic_entropy": 0.8039534687995911, + "step": 2065 + }, + { + "epoch": 1.107877571500251, + "grad_norm": 6.879708138429533, + "learning_rate": 7.948745306724931e-07, + "logits/chosen": -0.06296677142381668, + "logits/rejected": 0.08676932007074356, + "logps/chosen": -1.2351148128509521, + "logps/rejected": -1.5215286016464233, + "loss": 1.9843, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2351148128509521, + "rewards/margins": 0.28641384840011597, + "rewards/rejected": -1.5215286016464233, + "semantic_entropy": 0.8108504414558411, + "step": 2070 + }, + { + "epoch": 1.1105536042816524, + "grad_norm": 10.281724019392756, + "learning_rate": 7.936153648163897e-07, + "logits/chosen": -0.0729776918888092, + "logits/rejected": 0.02537897601723671, + "logps/chosen": -1.285667061805725, + "logps/rejected": -1.5334974527359009, + "loss": 2.0094, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.285667061805725, + "rewards/margins": 0.2478303164243698, + "rewards/rejected": -1.5334974527359009, + "semantic_entropy": 0.7966839075088501, + "step": 2075 + }, + { + "epoch": 1.1132296370630541, + "grad_norm": 5.658066237322829, + "learning_rate": 7.92353350302729e-07, + "logits/chosen": -0.105830118060112, + "logits/rejected": 0.049424778670072556, + "logps/chosen": -1.2078402042388916, + "logps/rejected": -1.5260502099990845, + "loss": 1.9342, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -1.2078402042388916, + "rewards/margins": 0.3182099163532257, + "rewards/rejected": -1.5260502099990845, + "semantic_entropy": 0.8191325068473816, + "step": 2080 + }, + { + "epoch": 1.1159056698444556, + "grad_norm": 12.694433127022538, + "learning_rate": 7.910884993755816e-07, + "logits/chosen": -0.10474447160959244, + "logits/rejected": 0.0023083791602402925, + "logps/chosen": -1.2569905519485474, + "logps/rejected": -1.488846778869629, + "loss": 2.0114, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.2569905519485474, + "rewards/margins": 0.23185637593269348, + "rewards/rejected": -1.488846778869629, + "semantic_entropy": 0.8038360476493835, + "step": 2085 + }, + { + "epoch": 1.118581702625857, + "grad_norm": 9.38006102694871, + "learning_rate": 7.898208243065367e-07, + "logits/chosen": -0.16282737255096436, + "logits/rejected": -0.1553858518600464, + "logps/chosen": -1.237786054611206, + "logps/rejected": -1.4228355884552002, + "loss": 2.0154, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.237786054611206, + "rewards/margins": 0.18504968285560608, + "rewards/rejected": -1.4228355884552002, + "semantic_entropy": 0.8132780194282532, + "step": 2090 + }, + { + "epoch": 1.1212577354072588, + "grad_norm": 6.012799912827595, + "learning_rate": 7.88550337394583e-07, + "logits/chosen": -0.0706481784582138, + "logits/rejected": 0.06993953138589859, + "logps/chosen": -1.393044114112854, + "logps/rejected": -1.5504987239837646, + "loss": 2.1148, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.393044114112854, + "rewards/margins": 0.157454714179039, + "rewards/rejected": -1.5504987239837646, + "semantic_entropy": 0.7678667902946472, + "step": 2095 + }, + { + "epoch": 1.1239337681886603, + "grad_norm": 9.165376113792247, + "learning_rate": 7.872770509659905e-07, + "logits/chosen": 0.02150576561689377, + "logits/rejected": 0.07556779682636261, + "logps/chosen": -1.3992087841033936, + "logps/rejected": -1.5149985551834106, + "loss": 2.107, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.3992087841033936, + "rewards/margins": 0.1157897338271141, + "rewards/rejected": -1.5149985551834106, + "semantic_entropy": 0.756324052810669, + "step": 2100 + }, + { + "epoch": 1.1266098009700618, + "grad_norm": 7.418887878348668, + "learning_rate": 7.860009773741896e-07, + "logits/chosen": 0.025624120607972145, + "logits/rejected": 0.14256080985069275, + "logps/chosen": -1.3226193189620972, + "logps/rejected": -1.5097525119781494, + "loss": 2.0465, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.3226193189620972, + "rewards/margins": 0.18713310360908508, + "rewards/rejected": -1.5097525119781494, + "semantic_entropy": 0.787027895450592, + "step": 2105 + }, + { + "epoch": 1.1292858337514635, + "grad_norm": 9.246072581454596, + "learning_rate": 7.84722128999652e-07, + "logits/chosen": -0.04906727001070976, + "logits/rejected": 0.11425349861383438, + "logps/chosen": -1.2612407207489014, + "logps/rejected": -1.6420915126800537, + "loss": 1.9762, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2612407207489014, + "rewards/margins": 0.3808509409427643, + "rewards/rejected": -1.6420915126800537, + "semantic_entropy": 0.7930108904838562, + "step": 2110 + }, + { + "epoch": 1.131961866532865, + "grad_norm": 8.482976009607345, + "learning_rate": 7.834405182497699e-07, + "logits/chosen": 0.058322250843048096, + "logits/rejected": 0.11610279232263565, + "logps/chosen": -1.2708790302276611, + "logps/rejected": -1.4597584009170532, + "loss": 2.0116, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.2708790302276611, + "rewards/margins": 0.1888793557882309, + "rewards/rejected": -1.4597584009170532, + "semantic_entropy": 0.7919198274612427, + "step": 2115 + }, + { + "epoch": 1.1346378993142665, + "grad_norm": 8.859454340426378, + "learning_rate": 7.821561575587368e-07, + "logits/chosen": -0.02575806714594364, + "logits/rejected": 0.013861206360161304, + "logps/chosen": -1.313529372215271, + "logps/rejected": -1.4739990234375, + "loss": 2.0587, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.313529372215271, + "rewards/margins": 0.16046956181526184, + "rewards/rejected": -1.4739990234375, + "semantic_entropy": 0.7935684323310852, + "step": 2120 + }, + { + "epoch": 1.1373139320956682, + "grad_norm": 6.909830428655394, + "learning_rate": 7.808690593874254e-07, + "logits/chosen": -0.008657902479171753, + "logits/rejected": 0.05696826055645943, + "logps/chosen": -1.2222939729690552, + "logps/rejected": -1.5031594038009644, + "loss": 1.9572, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": -1.2222939729690552, + "rewards/margins": 0.2808656096458435, + "rewards/rejected": -1.5031594038009644, + "semantic_entropy": 0.8107255101203918, + "step": 2125 + }, + { + "epoch": 1.1399899648770697, + "grad_norm": 10.724106791378182, + "learning_rate": 7.79579236223268e-07, + "logits/chosen": -0.0033174841664731503, + "logits/rejected": 0.23977334797382355, + "logps/chosen": -1.316141963005066, + "logps/rejected": -1.5274380445480347, + "loss": 2.0362, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.316141963005066, + "rewards/margins": 0.21129612624645233, + "rewards/rejected": -1.5274380445480347, + "semantic_entropy": 0.788119912147522, + "step": 2130 + }, + { + "epoch": 1.1426659976584714, + "grad_norm": 7.046612138948062, + "learning_rate": 7.782867005801346e-07, + "logits/chosen": -0.00464610755443573, + "logits/rejected": 0.1614435464143753, + "logps/chosen": -1.315813422203064, + "logps/rejected": -1.5444300174713135, + "loss": 2.0404, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.315813422203064, + "rewards/margins": 0.22861655056476593, + "rewards/rejected": -1.5444300174713135, + "semantic_entropy": 0.7877010107040405, + "step": 2135 + }, + { + "epoch": 1.145342030439873, + "grad_norm": 8.597150217290602, + "learning_rate": 7.769914649982117e-07, + "logits/chosen": -0.055145204067230225, + "logits/rejected": 0.08367632329463959, + "logps/chosen": -1.29514479637146, + "logps/rejected": -1.4626885652542114, + "loss": 2.0509, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.29514479637146, + "rewards/margins": 0.16754384338855743, + "rewards/rejected": -1.4626885652542114, + "semantic_entropy": 0.8096902966499329, + "step": 2140 + }, + { + "epoch": 1.1480180632212744, + "grad_norm": 8.339971979435767, + "learning_rate": 7.756935420438803e-07, + "logits/chosen": -0.004235397092998028, + "logits/rejected": 0.08324207365512848, + "logps/chosen": -1.1817706823349, + "logps/rejected": -1.4775861501693726, + "loss": 1.954, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.1817706823349, + "rewards/margins": 0.29581543803215027, + "rewards/rejected": -1.4775861501693726, + "semantic_entropy": 0.819560170173645, + "step": 2145 + }, + { + "epoch": 1.1506940960026761, + "grad_norm": 5.499676423671464, + "learning_rate": 7.743929443095951e-07, + "logits/chosen": -0.0550382137298584, + "logits/rejected": 0.001202210783958435, + "logps/chosen": -1.3445276021957397, + "logps/rejected": -1.547282099723816, + "loss": 2.0586, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.3445276021957397, + "rewards/margins": 0.2027544230222702, + "rewards/rejected": -1.547282099723816, + "semantic_entropy": 0.7729575634002686, + "step": 2150 + }, + { + "epoch": 1.1533701287840776, + "grad_norm": 6.473506673489069, + "learning_rate": 7.730896844137609e-07, + "logits/chosen": 0.031207948923110962, + "logits/rejected": 0.09783817827701569, + "logps/chosen": -1.3366258144378662, + "logps/rejected": -1.5274697542190552, + "loss": 2.0341, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.3366258144378662, + "rewards/margins": 0.19084373116493225, + "rewards/rejected": -1.5274697542190552, + "semantic_entropy": 0.7779184579849243, + "step": 2155 + }, + { + "epoch": 1.1560461615654791, + "grad_norm": 8.835825021503956, + "learning_rate": 7.717837750006106e-07, + "logits/chosen": -0.0980040580034256, + "logits/rejected": 0.01028486154973507, + "logps/chosen": -1.2548919916152954, + "logps/rejected": -1.509281039237976, + "loss": 1.9958, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2548919916152954, + "rewards/margins": 0.2543889880180359, + "rewards/rejected": -1.509281039237976, + "semantic_entropy": 0.7877534627914429, + "step": 2160 + }, + { + "epoch": 1.1587221943468808, + "grad_norm": 8.271327898672205, + "learning_rate": 7.704752287400832e-07, + "logits/chosen": -0.0610247440636158, + "logits/rejected": 0.11114144325256348, + "logps/chosen": -1.2976696491241455, + "logps/rejected": -1.5627309083938599, + "loss": 2.0145, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -1.2976696491241455, + "rewards/margins": 0.2650611996650696, + "rewards/rejected": -1.5627309083938599, + "semantic_entropy": 0.7958112359046936, + "step": 2165 + }, + { + "epoch": 1.1613982271282823, + "grad_norm": 7.367775189859744, + "learning_rate": 7.691640583277004e-07, + "logits/chosen": -0.036937564611434937, + "logits/rejected": 0.12986062467098236, + "logps/chosen": -1.2528033256530762, + "logps/rejected": -1.5739809274673462, + "loss": 1.9942, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.2528033256530762, + "rewards/margins": 0.32117748260498047, + "rewards/rejected": -1.5739809274673462, + "semantic_entropy": 0.8013005256652832, + "step": 2170 + }, + { + "epoch": 1.1640742599096838, + "grad_norm": 6.321138290200016, + "learning_rate": 7.678502764844433e-07, + "logits/chosen": -0.06491942703723907, + "logits/rejected": 0.10677101463079453, + "logps/chosen": -1.3190665245056152, + "logps/rejected": -1.4719650745391846, + "loss": 2.0565, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.3190665245056152, + "rewards/margins": 0.1528986245393753, + "rewards/rejected": -1.4719650745391846, + "semantic_entropy": 0.7967050075531006, + "step": 2175 + }, + { + "epoch": 1.1667502926910855, + "grad_norm": 7.660731521759613, + "learning_rate": 7.665338959566288e-07, + "logits/chosen": -0.033620767295360565, + "logits/rejected": 0.047455258667469025, + "logps/chosen": -1.2621140480041504, + "logps/rejected": -1.4954121112823486, + "loss": 2.0003, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2621140480041504, + "rewards/margins": 0.2332981824874878, + "rewards/rejected": -1.4954121112823486, + "semantic_entropy": 0.8004210591316223, + "step": 2180 + }, + { + "epoch": 1.169426325472487, + "grad_norm": 9.018605412309794, + "learning_rate": 7.652149295157868e-07, + "logits/chosen": 0.06298673897981644, + "logits/rejected": 0.208059623837471, + "logps/chosen": -1.2907410860061646, + "logps/rejected": -1.4562091827392578, + "loss": 2.0281, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.2907410860061646, + "rewards/margins": 0.16546815633773804, + "rewards/rejected": -1.4562091827392578, + "semantic_entropy": 0.7983787655830383, + "step": 2185 + }, + { + "epoch": 1.1721023582538885, + "grad_norm": 7.8766805185523046, + "learning_rate": 7.638933899585354e-07, + "logits/chosen": 0.14884532988071442, + "logits/rejected": 0.19815945625305176, + "logps/chosen": -1.2859556674957275, + "logps/rejected": -1.4986616373062134, + "loss": 2.0256, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.2859556674957275, + "rewards/margins": 0.21270573139190674, + "rewards/rejected": -1.4986616373062134, + "semantic_entropy": 0.7964907884597778, + "step": 2190 + }, + { + "epoch": 1.1747783910352902, + "grad_norm": 10.1996833053212, + "learning_rate": 7.625692901064573e-07, + "logits/chosen": 0.08653564751148224, + "logits/rejected": 0.18244849145412445, + "logps/chosen": -1.2636315822601318, + "logps/rejected": -1.5912013053894043, + "loss": 1.9884, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2636315822601318, + "rewards/margins": 0.3275696933269501, + "rewards/rejected": -1.5912013053894043, + "semantic_entropy": 0.7970950603485107, + "step": 2195 + }, + { + "epoch": 1.1774544238166917, + "grad_norm": 7.299173024972554, + "learning_rate": 7.61242642805975e-07, + "logits/chosen": -0.008318186737596989, + "logits/rejected": -0.021589037030935287, + "logps/chosen": -1.2870399951934814, + "logps/rejected": -1.5103387832641602, + "loss": 2.0217, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.2870399951934814, + "rewards/margins": 0.22329886257648468, + "rewards/rejected": -1.5103387832641602, + "semantic_entropy": 0.7910685539245605, + "step": 2200 + }, + { + "epoch": 1.1801304565980932, + "grad_norm": 7.213949372344186, + "learning_rate": 7.599134609282266e-07, + "logits/chosen": -0.04548101872205734, + "logits/rejected": 0.1449245810508728, + "logps/chosen": -1.2256078720092773, + "logps/rejected": -1.4781060218811035, + "loss": 1.9978, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.2256078720092773, + "rewards/margins": 0.25249817967414856, + "rewards/rejected": -1.4781060218811035, + "semantic_entropy": 0.8226034045219421, + "step": 2205 + }, + { + "epoch": 1.182806489379495, + "grad_norm": 9.18705823903751, + "learning_rate": 7.585817573689402e-07, + "logits/chosen": -0.10582272708415985, + "logits/rejected": 0.008244603872299194, + "logps/chosen": -1.1650890111923218, + "logps/rejected": -1.5030286312103271, + "loss": 1.9383, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.1650890111923218, + "rewards/margins": 0.33793967962265015, + "rewards/rejected": -1.5030286312103271, + "semantic_entropy": 0.8308181762695312, + "step": 2210 + }, + { + "epoch": 1.1854825221608964, + "grad_norm": 8.865593844658033, + "learning_rate": 7.572475450483098e-07, + "logits/chosen": -0.0699770599603653, + "logits/rejected": 0.01554514653980732, + "logps/chosen": -1.3917657136917114, + "logps/rejected": -1.6706657409667969, + "loss": 2.0636, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.3917657136917114, + "rewards/margins": 0.278900146484375, + "rewards/rejected": -1.6706657409667969, + "semantic_entropy": 0.7539466619491577, + "step": 2215 + }, + { + "epoch": 1.188158554942298, + "grad_norm": 11.056070419744794, + "learning_rate": 7.559108369108689e-07, + "logits/chosen": -0.1278703808784485, + "logits/rejected": -0.0024462491273880005, + "logps/chosen": -1.2356823682785034, + "logps/rejected": -1.4762585163116455, + "loss": 2.0046, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2356823682785034, + "rewards/margins": 0.24057602882385254, + "rewards/rejected": -1.4762585163116455, + "semantic_entropy": 0.814600944519043, + "step": 2220 + }, + { + "epoch": 1.1908345877236997, + "grad_norm": 8.271989533397697, + "learning_rate": 7.54571645925366e-07, + "logits/chosen": -0.09756255894899368, + "logits/rejected": 0.10917206108570099, + "logps/chosen": -1.2540271282196045, + "logps/rejected": -1.5666847229003906, + "loss": 1.9733, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2540271282196045, + "rewards/margins": 0.3126576244831085, + "rewards/rejected": -1.5666847229003906, + "semantic_entropy": 0.8026211857795715, + "step": 2225 + }, + { + "epoch": 1.1935106205051011, + "grad_norm": 14.450927596316225, + "learning_rate": 7.532299850846378e-07, + "logits/chosen": -0.11772173643112183, + "logits/rejected": 0.016855159774422646, + "logps/chosen": -1.3069515228271484, + "logps/rejected": -1.6907488107681274, + "loss": 2.0293, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.3069515228271484, + "rewards/margins": 0.3837973475456238, + "rewards/rejected": -1.6907488107681274, + "semantic_entropy": 0.783217191696167, + "step": 2230 + }, + { + "epoch": 1.1961866532865026, + "grad_norm": 9.066081790085091, + "learning_rate": 7.518858674054838e-07, + "logits/chosen": -0.060464583337306976, + "logits/rejected": 0.12888970971107483, + "logps/chosen": -1.2510316371917725, + "logps/rejected": -1.5621397495269775, + "loss": 1.994, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2510316371917725, + "rewards/margins": 0.3111079931259155, + "rewards/rejected": -1.5621397495269775, + "semantic_entropy": 0.8048974275588989, + "step": 2235 + }, + { + "epoch": 1.1988626860679044, + "grad_norm": 9.594387405303804, + "learning_rate": 7.505393059285394e-07, + "logits/chosen": -0.046233952045440674, + "logits/rejected": 0.10925587266683578, + "logps/chosen": -1.2652842998504639, + "logps/rejected": -1.5169308185577393, + "loss": 1.984, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -1.2652842998504639, + "rewards/margins": 0.2516464591026306, + "rewards/rejected": -1.5169308185577393, + "semantic_entropy": 0.7933940887451172, + "step": 2240 + }, + { + "epoch": 1.2015387188493059, + "grad_norm": 9.062658843804321, + "learning_rate": 7.491903137181501e-07, + "logits/chosen": -0.012463415041565895, + "logits/rejected": 0.03613627701997757, + "logps/chosen": -1.2368113994598389, + "logps/rejected": -1.4751683473587036, + "loss": 2.0087, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.2368113994598389, + "rewards/margins": 0.2383570671081543, + "rewards/rejected": -1.4751683473587036, + "semantic_entropy": 0.8038736581802368, + "step": 2245 + }, + { + "epoch": 1.2042147516307076, + "grad_norm": 8.361895591161364, + "learning_rate": 7.478389038622441e-07, + "logits/chosen": 0.04597688838839531, + "logits/rejected": 0.06491976976394653, + "logps/chosen": -1.2090892791748047, + "logps/rejected": -1.5203975439071655, + "loss": 1.9498, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.2090892791748047, + "rewards/margins": 0.311308354139328, + "rewards/rejected": -1.5203975439071655, + "semantic_entropy": 0.8206877708435059, + "step": 2250 + }, + { + "epoch": 1.206890784412109, + "grad_norm": 8.36617213318591, + "learning_rate": 7.46485089472206e-07, + "logits/chosen": -0.02347579412162304, + "logits/rejected": 0.08216370642185211, + "logps/chosen": -1.3362557888031006, + "logps/rejected": -1.4455738067626953, + "loss": 2.0886, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -1.3362557888031006, + "rewards/margins": 0.10931817442178726, + "rewards/rejected": -1.4455738067626953, + "semantic_entropy": 0.7876633405685425, + "step": 2255 + }, + { + "epoch": 1.2095668171935106, + "grad_norm": 7.947935461378185, + "learning_rate": 7.451288836827487e-07, + "logits/chosen": 0.022869199514389038, + "logits/rejected": 0.00841585360467434, + "logps/chosen": -1.2855030298233032, + "logps/rejected": -1.4511579275131226, + "loss": 2.0189, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.2855030298233032, + "rewards/margins": 0.16565480828285217, + "rewards/rejected": -1.4511579275131226, + "semantic_entropy": 0.8032156825065613, + "step": 2260 + }, + { + "epoch": 1.2122428499749123, + "grad_norm": 8.103829288787791, + "learning_rate": 7.437702996517869e-07, + "logits/chosen": -0.07720647007226944, + "logits/rejected": 0.021549483761191368, + "logps/chosen": -1.3225862979888916, + "logps/rejected": -1.4839069843292236, + "loss": 2.0504, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -1.3225862979888916, + "rewards/margins": 0.16132058203220367, + "rewards/rejected": -1.4839069843292236, + "semantic_entropy": 0.7962474822998047, + "step": 2265 + }, + { + "epoch": 1.2149188827563138, + "grad_norm": 8.813740593042496, + "learning_rate": 7.424093505603087e-07, + "logits/chosen": -0.16498062014579773, + "logits/rejected": 0.010989139787852764, + "logps/chosen": -1.264560580253601, + "logps/rejected": -1.5233685970306396, + "loss": 1.9983, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.264560580253601, + "rewards/margins": 0.2588079273700714, + "rewards/rejected": -1.5233685970306396, + "semantic_entropy": 0.7983893156051636, + "step": 2270 + }, + { + "epoch": 1.2175949155377153, + "grad_norm": 8.927905431133137, + "learning_rate": 7.410460496122482e-07, + "logits/chosen": -0.031638335436582565, + "logits/rejected": 0.09178587049245834, + "logps/chosen": -1.2444086074829102, + "logps/rejected": -1.5470324754714966, + "loss": 1.9644, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2444086074829102, + "rewards/margins": 0.30262380838394165, + "rewards/rejected": -1.5470324754714966, + "semantic_entropy": 0.787211537361145, + "step": 2275 + }, + { + "epoch": 1.220270948319117, + "grad_norm": 10.498951525504093, + "learning_rate": 7.396804100343572e-07, + "logits/chosen": -0.09466014802455902, + "logits/rejected": 0.07289015501737595, + "logps/chosen": -1.1898757219314575, + "logps/rejected": -1.4143764972686768, + "loss": 1.9808, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.1898757219314575, + "rewards/margins": 0.22450082004070282, + "rewards/rejected": -1.4143764972686768, + "semantic_entropy": 0.8181502223014832, + "step": 2280 + }, + { + "epoch": 1.2229469811005185, + "grad_norm": 8.619007313844211, + "learning_rate": 7.383124450760768e-07, + "logits/chosen": -0.02043074555695057, + "logits/rejected": 0.18445360660552979, + "logps/chosen": -1.3112406730651855, + "logps/rejected": -1.5405431985855103, + "loss": 2.0467, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.3112406730651855, + "rewards/margins": 0.22930267453193665, + "rewards/rejected": -1.5405431985855103, + "semantic_entropy": 0.7737176418304443, + "step": 2285 + }, + { + "epoch": 1.22562301388192, + "grad_norm": 11.166453971795224, + "learning_rate": 7.369421680094091e-07, + "logits/chosen": -0.11235109716653824, + "logits/rejected": 0.04711022973060608, + "logps/chosen": -1.177172303199768, + "logps/rejected": -1.4215004444122314, + "loss": 1.9917, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.177172303199768, + "rewards/margins": 0.24432817101478577, + "rewards/rejected": -1.4215004444122314, + "semantic_entropy": 0.8288639783859253, + "step": 2290 + }, + { + "epoch": 1.2282990466633217, + "grad_norm": 8.551694391398613, + "learning_rate": 7.355695921287881e-07, + "logits/chosen": -0.08187909424304962, + "logits/rejected": 0.006174634210765362, + "logps/chosen": -1.265074372291565, + "logps/rejected": -1.4966572523117065, + "loss": 2.0164, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.265074372291565, + "rewards/margins": 0.23158295452594757, + "rewards/rejected": -1.4966572523117065, + "semantic_entropy": 0.8085952997207642, + "step": 2295 + }, + { + "epoch": 1.2309750794447232, + "grad_norm": 10.584942593444998, + "learning_rate": 7.341947307509513e-07, + "logits/chosen": -0.03479890897870064, + "logits/rejected": 0.09125839173793793, + "logps/chosen": -1.2871692180633545, + "logps/rejected": -1.4339960813522339, + "loss": 2.0749, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.2871692180633545, + "rewards/margins": 0.14682689309120178, + "rewards/rejected": -1.4339960813522339, + "semantic_entropy": 0.8087296485900879, + "step": 2300 + }, + { + "epoch": 1.233651112226125, + "grad_norm": 7.3984945064866965, + "learning_rate": 7.328175972148094e-07, + "logits/chosen": -0.08773527294397354, + "logits/rejected": 0.05295741558074951, + "logps/chosen": -1.3983827829360962, + "logps/rejected": -1.5909889936447144, + "loss": 2.0939, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.3983827829360962, + "rewards/margins": 0.1926061362028122, + "rewards/rejected": -1.5909889936447144, + "semantic_entropy": 0.7503756284713745, + "step": 2305 + }, + { + "epoch": 1.2363271450075264, + "grad_norm": 13.241352238082227, + "learning_rate": 7.314382048813185e-07, + "logits/chosen": -0.05741016939282417, + "logits/rejected": 0.21026751399040222, + "logps/chosen": -1.3185993432998657, + "logps/rejected": -1.5880451202392578, + "loss": 2.0274, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.3185993432998657, + "rewards/margins": 0.2694457173347473, + "rewards/rejected": -1.5880451202392578, + "semantic_entropy": 0.7889279127120972, + "step": 2310 + }, + { + "epoch": 1.2390031777889279, + "grad_norm": 9.867053081624858, + "learning_rate": 7.300565671333486e-07, + "logits/chosen": -0.03601957857608795, + "logits/rejected": 0.1427186280488968, + "logps/chosen": -1.3128204345703125, + "logps/rejected": -1.576196551322937, + "loss": 2.0158, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.3128204345703125, + "rewards/margins": 0.26337605714797974, + "rewards/rejected": -1.576196551322937, + "semantic_entropy": 0.7808772325515747, + "step": 2315 + }, + { + "epoch": 1.2416792105703296, + "grad_norm": 7.953399629652333, + "learning_rate": 7.286726973755554e-07, + "logits/chosen": 0.061737217009067535, + "logits/rejected": 0.08541327714920044, + "logps/chosen": -1.3010971546173096, + "logps/rejected": -1.557395577430725, + "loss": 2.0297, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.3010971546173096, + "rewards/margins": 0.2562984526157379, + "rewards/rejected": -1.557395577430725, + "semantic_entropy": 0.7937830090522766, + "step": 2320 + }, + { + "epoch": 1.244355243351731, + "grad_norm": 10.194369450500247, + "learning_rate": 7.272866090342493e-07, + "logits/chosen": 0.08444453775882721, + "logits/rejected": 0.16419485211372375, + "logps/chosen": -1.3324655294418335, + "logps/rejected": -1.6092456579208374, + "loss": 2.0442, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.3324655294418335, + "rewards/margins": 0.27678006887435913, + "rewards/rejected": -1.6092456579208374, + "semantic_entropy": 0.7854612469673157, + "step": 2325 + }, + { + "epoch": 1.2470312761331326, + "grad_norm": 8.188144233030942, + "learning_rate": 7.258983155572656e-07, + "logits/chosen": -0.10631588846445084, + "logits/rejected": 0.006649285554885864, + "logps/chosen": -1.2845680713653564, + "logps/rejected": -1.5136401653289795, + "loss": 2.027, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.2845680713653564, + "rewards/margins": 0.22907209396362305, + "rewards/rejected": -1.5136401653289795, + "semantic_entropy": 0.7916483879089355, + "step": 2330 + }, + { + "epoch": 1.2497073089145343, + "grad_norm": 9.980220667454514, + "learning_rate": 7.245078304138335e-07, + "logits/chosen": 0.05152348801493645, + "logits/rejected": 0.11100976169109344, + "logps/chosen": -1.278947114944458, + "logps/rejected": -1.57237708568573, + "loss": 1.9967, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.278947114944458, + "rewards/margins": 0.2934300899505615, + "rewards/rejected": -1.57237708568573, + "semantic_entropy": 0.7966108918190002, + "step": 2335 + }, + { + "epoch": 1.2523833416959358, + "grad_norm": 6.364400964164751, + "learning_rate": 7.231151670944462e-07, + "logits/chosen": -0.12852008640766144, + "logits/rejected": 0.05516228824853897, + "logps/chosen": -1.2988489866256714, + "logps/rejected": -1.5035803318023682, + "loss": 2.059, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.2988489866256714, + "rewards/margins": 0.20473137497901917, + "rewards/rejected": -1.5035803318023682, + "semantic_entropy": 0.8019092679023743, + "step": 2340 + }, + { + "epoch": 1.2550593744773373, + "grad_norm": 8.247743814735342, + "learning_rate": 7.217203391107291e-07, + "logits/chosen": -0.03583495691418648, + "logits/rejected": 0.12947800755500793, + "logps/chosen": -1.2738090753555298, + "logps/rejected": -1.526583194732666, + "loss": 2.0255, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.2738090753555298, + "rewards/margins": 0.25277405977249146, + "rewards/rejected": -1.526583194732666, + "semantic_entropy": 0.7844155430793762, + "step": 2345 + }, + { + "epoch": 1.257735407258739, + "grad_norm": 7.081772579883173, + "learning_rate": 7.203233599953096e-07, + "logits/chosen": -0.035393621772527695, + "logits/rejected": 0.10859780013561249, + "logps/chosen": -1.3218568563461304, + "logps/rejected": -1.488811731338501, + "loss": 2.0513, + "rewards/accuracies": 0.53125, + "rewards/chosen": -1.3218568563461304, + "rewards/margins": 0.16695484519004822, + "rewards/rejected": -1.488811731338501, + "semantic_entropy": 0.7921909689903259, + "step": 2350 + }, + { + "epoch": 1.2604114400401405, + "grad_norm": 10.049995364195077, + "learning_rate": 7.189242433016852e-07, + "logits/chosen": 0.032818764448165894, + "logits/rejected": 0.16986875236034393, + "logps/chosen": -1.2119081020355225, + "logps/rejected": -1.5190856456756592, + "loss": 1.9815, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2119081020355225, + "rewards/margins": 0.3071775436401367, + "rewards/rejected": -1.5190856456756592, + "semantic_entropy": 0.8008241653442383, + "step": 2355 + }, + { + "epoch": 1.263087472821542, + "grad_norm": 9.041411333486012, + "learning_rate": 7.17523002604092e-07, + "logits/chosen": 0.006859609391540289, + "logits/rejected": 0.13708624243736267, + "logps/chosen": -1.2518107891082764, + "logps/rejected": -1.6344480514526367, + "loss": 1.9757, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2518107891082764, + "rewards/margins": 0.38263726234436035, + "rewards/rejected": -1.6344480514526367, + "semantic_entropy": 0.7938586473464966, + "step": 2360 + }, + { + "epoch": 1.2657635056029437, + "grad_norm": 6.068539137807391, + "learning_rate": 7.161196514973734e-07, + "logits/chosen": 0.029105842113494873, + "logits/rejected": 0.17803865671157837, + "logps/chosen": -1.2924420833587646, + "logps/rejected": -1.551058053970337, + "loss": 1.9978, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.2924420833587646, + "rewards/margins": 0.25861606001853943, + "rewards/rejected": -1.551058053970337, + "semantic_entropy": 0.7914843559265137, + "step": 2365 + }, + { + "epoch": 1.2684395383843452, + "grad_norm": 8.797661180013133, + "learning_rate": 7.147142035968483e-07, + "logits/chosen": 0.05001775175333023, + "logits/rejected": 0.1784544289112091, + "logps/chosen": -1.2478030920028687, + "logps/rejected": -1.5050370693206787, + "loss": 1.9815, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2478030920028687, + "rewards/margins": 0.2572340667247772, + "rewards/rejected": -1.5050370693206787, + "semantic_entropy": 0.8060957193374634, + "step": 2370 + }, + { + "epoch": 1.2711155711657467, + "grad_norm": 8.101720923817023, + "learning_rate": 7.133066725381781e-07, + "logits/chosen": -0.1124950498342514, + "logits/rejected": 0.06909557431936264, + "logps/chosen": -1.1976344585418701, + "logps/rejected": -1.3552278280258179, + "loss": 1.9959, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.1976344585418701, + "rewards/margins": 0.1575932800769806, + "rewards/rejected": -1.3552278280258179, + "semantic_entropy": 0.8371988534927368, + "step": 2375 + }, + { + "epoch": 1.2737916039471484, + "grad_norm": 9.301816034600291, + "learning_rate": 7.118970719772354e-07, + "logits/chosen": -0.08418088406324387, + "logits/rejected": 0.13591249287128448, + "logps/chosen": -1.3185670375823975, + "logps/rejected": -1.5692827701568604, + "loss": 2.0367, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.3185670375823975, + "rewards/margins": 0.2507156431674957, + "rewards/rejected": -1.5692827701568604, + "semantic_entropy": 0.7790293097496033, + "step": 2380 + }, + { + "epoch": 1.27646763672855, + "grad_norm": 9.894290297080861, + "learning_rate": 7.104854155899711e-07, + "logits/chosen": 0.022958554327487946, + "logits/rejected": 0.12819671630859375, + "logps/chosen": -1.2903987169265747, + "logps/rejected": -1.5150518417358398, + "loss": 2.0494, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.2903987169265747, + "rewards/margins": 0.2246532440185547, + "rewards/rejected": -1.5150518417358398, + "semantic_entropy": 0.7933465838432312, + "step": 2385 + }, + { + "epoch": 1.2791436695099514, + "grad_norm": 7.120856691958988, + "learning_rate": 7.090717170722817e-07, + "logits/chosen": 0.02720743417739868, + "logits/rejected": 0.09357274323701859, + "logps/chosen": -1.2580147981643677, + "logps/rejected": -1.5381114482879639, + "loss": 1.9848, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2580147981643677, + "rewards/margins": 0.28009670972824097, + "rewards/rejected": -1.5381114482879639, + "semantic_entropy": 0.8013747930526733, + "step": 2390 + }, + { + "epoch": 1.2818197022913531, + "grad_norm": 11.88828279536148, + "learning_rate": 7.076559901398762e-07, + "logits/chosen": -0.19155053794384003, + "logits/rejected": -0.06659739464521408, + "logps/chosen": -1.1978116035461426, + "logps/rejected": -1.4624278545379639, + "loss": 1.9853, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.1978116035461426, + "rewards/margins": 0.2646161615848541, + "rewards/rejected": -1.4624278545379639, + "semantic_entropy": 0.8260299563407898, + "step": 2395 + }, + { + "epoch": 1.2844957350727546, + "grad_norm": 9.950995582560179, + "learning_rate": 7.062382485281436e-07, + "logits/chosen": -0.035932350903749466, + "logits/rejected": 0.08602927625179291, + "logps/chosen": -1.2562742233276367, + "logps/rejected": -1.5017836093902588, + "loss": 2.0207, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.2562742233276367, + "rewards/margins": 0.24550941586494446, + "rewards/rejected": -1.5017836093902588, + "semantic_entropy": 0.8130423426628113, + "step": 2400 + }, + { + "epoch": 1.2844957350727546, + "eval_logits/chosen": 0.2933657169342041, + "eval_logits/rejected": 0.3859518766403198, + "eval_logps/chosen": -1.330057144165039, + "eval_logps/rejected": -1.577328085899353, + "eval_loss": 2.0472466945648193, + "eval_rewards/accuracies": 0.5875371098518372, + "eval_rewards/chosen": -1.330057144165039, + "eval_rewards/margins": 0.24727098643779755, + "eval_rewards/rejected": -1.577328085899353, + "eval_runtime": 34.6857, + "eval_samples_per_second": 38.777, + "eval_semantic_entropy": 0.7809399366378784, + "eval_steps_per_second": 9.716, + "step": 2400 + }, + { + "epoch": 1.287171767854156, + "grad_norm": 8.736922052582077, + "learning_rate": 7.048185059920193e-07, + "logits/chosen": -0.0388624481856823, + "logits/rejected": 0.1136031299829483, + "logps/chosen": -1.3021931648254395, + "logps/rejected": -1.598894715309143, + "loss": 2.0115, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.3021931648254395, + "rewards/margins": 0.29670166969299316, + "rewards/rejected": -1.598894715309143, + "semantic_entropy": 0.7815759181976318, + "step": 2405 + }, + { + "epoch": 1.2898478006355578, + "grad_norm": 5.844097600671448, + "learning_rate": 7.033967763058516e-07, + "logits/chosen": -0.18287523090839386, + "logits/rejected": 0.016366440802812576, + "logps/chosen": -1.2722828388214111, + "logps/rejected": -1.3984174728393555, + "loss": 2.064, + "rewards/accuracies": 0.53125, + "rewards/chosen": -1.2722828388214111, + "rewards/margins": 0.12613460421562195, + "rewards/rejected": -1.3984174728393555, + "semantic_entropy": 0.8081968426704407, + "step": 2410 + }, + { + "epoch": 1.2925238334169593, + "grad_norm": 6.860452427864468, + "learning_rate": 7.019730732632681e-07, + "logits/chosen": 0.014550815336406231, + "logits/rejected": 0.08596150577068329, + "logps/chosen": -1.1932735443115234, + "logps/rejected": -1.5706613063812256, + "loss": 1.9274, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.1932735443115234, + "rewards/margins": 0.3773877024650574, + "rewards/rejected": -1.5706613063812256, + "semantic_entropy": 0.8176706433296204, + "step": 2415 + }, + { + "epoch": 1.2951998661983608, + "grad_norm": 8.814610765856207, + "learning_rate": 7.005474106770418e-07, + "logits/chosen": -0.1114654541015625, + "logits/rejected": 0.013631993904709816, + "logps/chosen": -1.3067487478256226, + "logps/rejected": -1.6183964014053345, + "loss": 1.9872, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.3067487478256226, + "rewards/margins": 0.3116476535797119, + "rewards/rejected": -1.6183964014053345, + "semantic_entropy": 0.768617570400238, + "step": 2420 + }, + { + "epoch": 1.2978758989797625, + "grad_norm": 8.996235476343262, + "learning_rate": 6.991198023789577e-07, + "logits/chosen": -0.0221896730363369, + "logits/rejected": 0.056027401238679886, + "logps/chosen": -1.2366410493850708, + "logps/rejected": -1.4527819156646729, + "loss": 1.9974, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.2366410493850708, + "rewards/margins": 0.21614094078540802, + "rewards/rejected": -1.4527819156646729, + "semantic_entropy": 0.8054319620132446, + "step": 2425 + }, + { + "epoch": 1.300551931761164, + "grad_norm": 9.971777847894783, + "learning_rate": 6.976902622196776e-07, + "logits/chosen": 0.007852977141737938, + "logits/rejected": 0.058755289763212204, + "logps/chosen": -1.3803184032440186, + "logps/rejected": -1.5980432033538818, + "loss": 2.0738, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.3803184032440186, + "rewards/margins": 0.2177247703075409, + "rewards/rejected": -1.5980432033538818, + "semantic_entropy": 0.7618513107299805, + "step": 2430 + }, + { + "epoch": 1.3032279645425655, + "grad_norm": 5.783046917505834, + "learning_rate": 6.962588040686064e-07, + "logits/chosen": 0.024611715227365494, + "logits/rejected": 0.15348206460475922, + "logps/chosen": -1.2611327171325684, + "logps/rejected": -1.4416377544403076, + "loss": 2.0195, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.2611327171325684, + "rewards/margins": 0.18050506711006165, + "rewards/rejected": -1.4416377544403076, + "semantic_entropy": 0.8049419522285461, + "step": 2435 + }, + { + "epoch": 1.3059039973239672, + "grad_norm": 8.973601983735593, + "learning_rate": 6.948254418137573e-07, + "logits/chosen": -0.10572667419910431, + "logits/rejected": 0.0066367872059345245, + "logps/chosen": -1.2377347946166992, + "logps/rejected": -1.4910590648651123, + "loss": 1.998, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.2377347946166992, + "rewards/margins": 0.2533242404460907, + "rewards/rejected": -1.4910590648651123, + "semantic_entropy": 0.8126150369644165, + "step": 2440 + }, + { + "epoch": 1.3085800301053687, + "grad_norm": 7.804619647636399, + "learning_rate": 6.933901893616174e-07, + "logits/chosen": -0.07127559185028076, + "logits/rejected": 0.07549577951431274, + "logps/chosen": -1.2812305688858032, + "logps/rejected": -1.4737753868103027, + "loss": 2.0333, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.2812305688858032, + "rewards/margins": 0.19254475831985474, + "rewards/rejected": -1.4737753868103027, + "semantic_entropy": 0.8040180206298828, + "step": 2445 + }, + { + "epoch": 1.3112560628867704, + "grad_norm": 6.889523714270502, + "learning_rate": 6.919530606370121e-07, + "logits/chosen": -0.027874654158949852, + "logits/rejected": 0.13589277863502502, + "logps/chosen": -1.247702956199646, + "logps/rejected": -1.584751009941101, + "loss": 1.9647, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.247702956199646, + "rewards/margins": 0.33704808354377747, + "rewards/rejected": -1.584751009941101, + "semantic_entropy": 0.8122938275337219, + "step": 2450 + }, + { + "epoch": 1.313932095668172, + "grad_norm": 5.118254464470323, + "learning_rate": 6.905140695829706e-07, + "logits/chosen": -0.06709776073694229, + "logits/rejected": 0.17308706045150757, + "logps/chosen": -1.3292291164398193, + "logps/rejected": -1.5290602445602417, + "loss": 2.0415, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.3292291164398193, + "rewards/margins": 0.19983109831809998, + "rewards/rejected": -1.5290602445602417, + "semantic_entropy": 0.7836787700653076, + "step": 2455 + }, + { + "epoch": 1.3166081284495736, + "grad_norm": 8.789989155313604, + "learning_rate": 6.890732301605904e-07, + "logits/chosen": -0.00417876522988081, + "logits/rejected": 0.1012817770242691, + "logps/chosen": -1.3258728981018066, + "logps/rejected": -1.4803338050842285, + "loss": 2.0531, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.3258728981018066, + "rewards/margins": 0.15446093678474426, + "rewards/rejected": -1.4803338050842285, + "semantic_entropy": 0.7863596081733704, + "step": 2460 + }, + { + "epoch": 1.3192841612309751, + "grad_norm": 7.69060213765338, + "learning_rate": 6.876305563489021e-07, + "logits/chosen": 0.011214707978069782, + "logits/rejected": 0.09302522987127304, + "logps/chosen": -1.2799160480499268, + "logps/rejected": -1.5444409847259521, + "loss": 2.0133, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.2799160480499268, + "rewards/margins": 0.26452499628067017, + "rewards/rejected": -1.5444409847259521, + "semantic_entropy": 0.8035866022109985, + "step": 2465 + }, + { + "epoch": 1.3219601940123766, + "grad_norm": 10.928279384945997, + "learning_rate": 6.861860621447331e-07, + "logits/chosen": -0.13337582349777222, + "logits/rejected": -0.002061316277831793, + "logps/chosen": -1.2584311962127686, + "logps/rejected": -1.4112979173660278, + "loss": 2.0326, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.2584311962127686, + "rewards/margins": 0.15286675095558167, + "rewards/rejected": -1.4112979173660278, + "semantic_entropy": 0.8194764256477356, + "step": 2470 + }, + { + "epoch": 1.3246362267937783, + "grad_norm": 7.148688930285466, + "learning_rate": 6.847397615625725e-07, + "logits/chosen": -0.013584035448729992, + "logits/rejected": 0.05738170072436333, + "logps/chosen": -1.2745800018310547, + "logps/rejected": -1.5176098346710205, + "loss": 2.0152, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.2745800018310547, + "rewards/margins": 0.24302978813648224, + "rewards/rejected": -1.5176098346710205, + "semantic_entropy": 0.7967642545700073, + "step": 2475 + }, + { + "epoch": 1.3273122595751798, + "grad_norm": 6.877033441682886, + "learning_rate": 6.83291668634435e-07, + "logits/chosen": -0.1592644453048706, + "logits/rejected": 0.020404372364282608, + "logps/chosen": -1.302016019821167, + "logps/rejected": -1.5877892971038818, + "loss": 1.994, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.302016019821167, + "rewards/margins": 0.2857731580734253, + "rewards/rejected": -1.5877892971038818, + "semantic_entropy": 0.7693041563034058, + "step": 2480 + }, + { + "epoch": 1.3299882923565813, + "grad_norm": 7.9456489134187205, + "learning_rate": 6.818417974097246e-07, + "logits/chosen": 0.0417151153087616, + "logits/rejected": 0.22702746093273163, + "logps/chosen": -1.3054397106170654, + "logps/rejected": -1.6041944026947021, + "loss": 2.0443, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.3054397106170654, + "rewards/margins": 0.2987547218799591, + "rewards/rejected": -1.6041944026947021, + "semantic_entropy": 0.787520706653595, + "step": 2485 + }, + { + "epoch": 1.332664325137983, + "grad_norm": 7.924827416846528, + "learning_rate": 6.803901619550981e-07, + "logits/chosen": -0.08295613527297974, + "logits/rejected": -0.02383790723979473, + "logps/chosen": -1.302221655845642, + "logps/rejected": -1.5922086238861084, + "loss": 2.0096, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.302221655845642, + "rewards/margins": 0.289986789226532, + "rewards/rejected": -1.5922086238861084, + "semantic_entropy": 0.7813442945480347, + "step": 2490 + }, + { + "epoch": 1.3353403579193845, + "grad_norm": 7.910158635170881, + "learning_rate": 6.789367763543292e-07, + "logits/chosen": 0.026933297514915466, + "logits/rejected": 0.03556499630212784, + "logps/chosen": -1.2895691394805908, + "logps/rejected": -1.5162808895111084, + "loss": 2.021, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2895691394805908, + "rewards/margins": 0.2267116755247116, + "rewards/rejected": -1.5162808895111084, + "semantic_entropy": 0.7934345602989197, + "step": 2495 + }, + { + "epoch": 1.338016390700786, + "grad_norm": 8.084202427838166, + "learning_rate": 6.774816547081714e-07, + "logits/chosen": 0.05614136531949043, + "logits/rejected": 0.22107501327991486, + "logps/chosen": -1.2359117269515991, + "logps/rejected": -1.4719440937042236, + "loss": 1.9882, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2359117269515991, + "rewards/margins": 0.23603227734565735, + "rewards/rejected": -1.4719440937042236, + "semantic_entropy": 0.8171242475509644, + "step": 2500 + }, + { + "epoch": 1.3406924234821878, + "grad_norm": 7.735130816689047, + "learning_rate": 6.760248111342211e-07, + "logits/chosen": 0.009106556884944439, + "logits/rejected": 0.1891101449728012, + "logps/chosen": -1.24410080909729, + "logps/rejected": -1.4906928539276123, + "loss": 1.9864, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.24410080909729, + "rewards/margins": 0.24659200012683868, + "rewards/rejected": -1.4906928539276123, + "semantic_entropy": 0.8127717971801758, + "step": 2505 + }, + { + "epoch": 1.3433684562635893, + "grad_norm": 12.384562731005605, + "learning_rate": 6.745662597667813e-07, + "logits/chosen": -0.08548152446746826, + "logits/rejected": 0.04962456226348877, + "logps/chosen": -1.232684850692749, + "logps/rejected": -1.4977141618728638, + "loss": 1.9834, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.232684850692749, + "rewards/margins": 0.26502925157546997, + "rewards/rejected": -1.4977141618728638, + "semantic_entropy": 0.8126821517944336, + "step": 2510 + }, + { + "epoch": 1.3460444890449907, + "grad_norm": 6.177445891852524, + "learning_rate": 6.731060147567236e-07, + "logits/chosen": 0.008504378609359264, + "logits/rejected": 0.1291893571615219, + "logps/chosen": -1.3171823024749756, + "logps/rejected": -1.4955008029937744, + "loss": 2.0666, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.3171823024749756, + "rewards/margins": 0.17831860482692719, + "rewards/rejected": -1.4955008029937744, + "semantic_entropy": 0.7949143052101135, + "step": 2515 + }, + { + "epoch": 1.3487205218263925, + "grad_norm": 5.895468706608909, + "learning_rate": 6.716440902713515e-07, + "logits/chosen": -0.06746038049459457, + "logits/rejected": 0.014505298808217049, + "logps/chosen": -1.2996203899383545, + "logps/rejected": -1.5103718042373657, + "loss": 2.0256, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2996203899383545, + "rewards/margins": 0.2107514888048172, + "rewards/rejected": -1.5103718042373657, + "semantic_entropy": 0.7952625155448914, + "step": 2520 + }, + { + "epoch": 1.351396554607794, + "grad_norm": 12.24033172449453, + "learning_rate": 6.701805004942627e-07, + "logits/chosen": -0.06568801403045654, + "logits/rejected": 0.014113761484622955, + "logps/chosen": -1.324389934539795, + "logps/rejected": -1.5580135583877563, + "loss": 2.0244, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.324389934539795, + "rewards/margins": 0.23362377285957336, + "rewards/rejected": -1.5580135583877563, + "semantic_entropy": 0.7667808532714844, + "step": 2525 + }, + { + "epoch": 1.3540725873891954, + "grad_norm": 9.655691096098971, + "learning_rate": 6.687152596252119e-07, + "logits/chosen": -0.08653044700622559, + "logits/rejected": -0.03837307170033455, + "logps/chosen": -1.2778962850570679, + "logps/rejected": -1.4817698001861572, + "loss": 2.0308, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.2778962850570679, + "rewards/margins": 0.20387335121631622, + "rewards/rejected": -1.4817698001861572, + "semantic_entropy": 0.8072389364242554, + "step": 2530 + }, + { + "epoch": 1.3567486201705972, + "grad_norm": 6.229786698881941, + "learning_rate": 6.672483818799722e-07, + "logits/chosen": -0.10884450376033783, + "logits/rejected": 0.031116357073187828, + "logps/chosen": -1.2840421199798584, + "logps/rejected": -1.5486438274383545, + "loss": 2.0033, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2840421199798584, + "rewards/margins": 0.2646019756793976, + "rewards/rejected": -1.5486438274383545, + "semantic_entropy": 0.7965590357780457, + "step": 2535 + }, + { + "epoch": 1.3594246529519987, + "grad_norm": 11.08324125453285, + "learning_rate": 6.657798814901978e-07, + "logits/chosen": -0.030958428978919983, + "logits/rejected": 0.1495594084262848, + "logps/chosen": -1.3801937103271484, + "logps/rejected": -1.5122677087783813, + "loss": 2.1004, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.3801937103271484, + "rewards/margins": 0.1320740431547165, + "rewards/rejected": -1.5122677087783813, + "semantic_entropy": 0.7679113149642944, + "step": 2540 + }, + { + "epoch": 1.3621006857334002, + "grad_norm": 7.463242215270924, + "learning_rate": 6.643097727032863e-07, + "logits/chosen": -0.011565339751541615, + "logits/rejected": 0.16919642686843872, + "logps/chosen": -1.2788037061691284, + "logps/rejected": -1.5690239667892456, + "loss": 2.0136, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.2788037061691284, + "rewards/margins": 0.2902204394340515, + "rewards/rejected": -1.5690239667892456, + "semantic_entropy": 0.7988485097885132, + "step": 2545 + }, + { + "epoch": 1.3647767185148019, + "grad_norm": 8.401641521017426, + "learning_rate": 6.628380697822392e-07, + "logits/chosen": -0.03879554197192192, + "logits/rejected": 0.1336180716753006, + "logps/chosen": -1.2727123498916626, + "logps/rejected": -1.4142420291900635, + "loss": 2.0508, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.2727123498916626, + "rewards/margins": 0.1415296494960785, + "rewards/rejected": -1.4142420291900635, + "semantic_entropy": 0.814642071723938, + "step": 2550 + }, + { + "epoch": 1.3674527512962034, + "grad_norm": 10.17566266604566, + "learning_rate": 6.61364787005525e-07, + "logits/chosen": 0.024192985147237778, + "logits/rejected": 0.11562930047512054, + "logps/chosen": -1.212947130203247, + "logps/rejected": -1.5666682720184326, + "loss": 1.9549, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.212947130203247, + "rewards/margins": 0.3537212312221527, + "rewards/rejected": -1.5666682720184326, + "semantic_entropy": 0.8101435899734497, + "step": 2555 + }, + { + "epoch": 1.3701287840776049, + "grad_norm": 8.236825612826049, + "learning_rate": 6.598899386669395e-07, + "logits/chosen": 0.03629554063081741, + "logits/rejected": 0.16760225594043732, + "logps/chosen": -1.2631809711456299, + "logps/rejected": -1.5157405138015747, + "loss": 1.998, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.2631809711456299, + "rewards/margins": 0.2525593638420105, + "rewards/rejected": -1.5157405138015747, + "semantic_entropy": 0.8036211133003235, + "step": 2560 + }, + { + "epoch": 1.3728048168590066, + "grad_norm": 10.644311265208762, + "learning_rate": 6.584135390754679e-07, + "logits/chosen": 0.02121199481189251, + "logits/rejected": 0.15904763340950012, + "logps/chosen": -1.2449887990951538, + "logps/rejected": -1.528660774230957, + "loss": 1.9903, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.2449887990951538, + "rewards/margins": 0.28367194533348083, + "rewards/rejected": -1.528660774230957, + "semantic_entropy": 0.8081823587417603, + "step": 2565 + }, + { + "epoch": 1.375480849640408, + "grad_norm": 5.671198047003181, + "learning_rate": 6.569356025551454e-07, + "logits/chosen": 0.08652433753013611, + "logits/rejected": 0.15835759043693542, + "logps/chosen": -1.2589101791381836, + "logps/rejected": -1.5290107727050781, + "loss": 1.9852, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2589101791381836, + "rewards/margins": 0.27010056376457214, + "rewards/rejected": -1.5290107727050781, + "semantic_entropy": 0.8052641153335571, + "step": 2570 + }, + { + "epoch": 1.3781568824218096, + "grad_norm": 10.644019148144604, + "learning_rate": 6.554561434449186e-07, + "logits/chosen": -0.07601086795330048, + "logits/rejected": 0.0795358270406723, + "logps/chosen": -1.2429871559143066, + "logps/rejected": -1.529396414756775, + "loss": 1.9813, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.2429871559143066, + "rewards/margins": 0.28640928864479065, + "rewards/rejected": -1.529396414756775, + "semantic_entropy": 0.8111133575439453, + "step": 2575 + }, + { + "epoch": 1.3808329152032113, + "grad_norm": 11.586334506338712, + "learning_rate": 6.539751760985063e-07, + "logits/chosen": -0.006502258591353893, + "logits/rejected": 0.07593898475170135, + "logps/chosen": -1.338326096534729, + "logps/rejected": -1.5027631521224976, + "loss": 2.0704, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.338326096534729, + "rewards/margins": 0.1644371747970581, + "rewards/rejected": -1.5027631521224976, + "semantic_entropy": 0.7761968374252319, + "step": 2580 + }, + { + "epoch": 1.3835089479846128, + "grad_norm": 7.7658484961464085, + "learning_rate": 6.524927148842602e-07, + "logits/chosen": 0.09149026870727539, + "logits/rejected": 0.2646264433860779, + "logps/chosen": -1.1860164403915405, + "logps/rejected": -1.5092562437057495, + "loss": 1.9556, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.1860164403915405, + "rewards/margins": 0.32323986291885376, + "rewards/rejected": -1.5092562437057495, + "semantic_entropy": 0.8078700304031372, + "step": 2585 + }, + { + "epoch": 1.3861849807660143, + "grad_norm": 9.208964534441007, + "learning_rate": 6.510087741850254e-07, + "logits/chosen": -0.021234478801488876, + "logits/rejected": 0.12869387865066528, + "logps/chosen": -1.2381582260131836, + "logps/rejected": -1.4897607564926147, + "loss": 2.0092, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.2381582260131836, + "rewards/margins": 0.2516025900840759, + "rewards/rejected": -1.4897607564926147, + "semantic_entropy": 0.8143725395202637, + "step": 2590 + }, + { + "epoch": 1.388861013547416, + "grad_norm": 10.626958785316473, + "learning_rate": 6.495233683980012e-07, + "logits/chosen": 0.013047700747847557, + "logits/rejected": 0.05873823165893555, + "logps/chosen": -1.249180555343628, + "logps/rejected": -1.4837150573730469, + "loss": 1.9945, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.249180555343628, + "rewards/margins": 0.23453423380851746, + "rewards/rejected": -1.4837150573730469, + "semantic_entropy": 0.8165580034255981, + "step": 2595 + }, + { + "epoch": 1.3915370463288175, + "grad_norm": 10.315240324352786, + "learning_rate": 6.480365119346011e-07, + "logits/chosen": 0.07690244913101196, + "logits/rejected": 0.2094108760356903, + "logps/chosen": -1.2769521474838257, + "logps/rejected": -1.4337047338485718, + "loss": 2.0389, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.2769521474838257, + "rewards/margins": 0.15675252676010132, + "rewards/rejected": -1.4337047338485718, + "semantic_entropy": 0.803144633769989, + "step": 2600 + }, + { + "epoch": 1.394213079110219, + "grad_norm": 8.152851427219868, + "learning_rate": 6.465482192203129e-07, + "logits/chosen": 0.06097614765167236, + "logits/rejected": 0.10995031893253326, + "logps/chosen": -1.2642452716827393, + "logps/rejected": -1.460809350013733, + "loss": 2.0252, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.2642452716827393, + "rewards/margins": 0.19656404852867126, + "rewards/rejected": -1.460809350013733, + "semantic_entropy": 0.8018202781677246, + "step": 2605 + }, + { + "epoch": 1.3968891118916207, + "grad_norm": 9.626880922808345, + "learning_rate": 6.45058504694559e-07, + "logits/chosen": 0.07766013592481613, + "logits/rejected": 0.15449103713035583, + "logps/chosen": -1.3124924898147583, + "logps/rejected": -1.5030378103256226, + "loss": 2.0732, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.3124924898147583, + "rewards/margins": 0.1905454695224762, + "rewards/rejected": -1.5030378103256226, + "semantic_entropy": 0.7874075770378113, + "step": 2610 + }, + { + "epoch": 1.3995651446730222, + "grad_norm": 12.498925680101337, + "learning_rate": 6.435673828105564e-07, + "logits/chosen": -0.04191456735134125, + "logits/rejected": 0.10613715648651123, + "logps/chosen": -1.2324085235595703, + "logps/rejected": -1.5605417490005493, + "loss": 1.9814, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2324085235595703, + "rewards/margins": 0.3281332850456238, + "rewards/rejected": -1.5605417490005493, + "semantic_entropy": 0.8089901804924011, + "step": 2615 + }, + { + "epoch": 1.402241177454424, + "grad_norm": 9.860457798037306, + "learning_rate": 6.420748680351763e-07, + "logits/chosen": -0.04338609799742699, + "logits/rejected": -0.04294615983963013, + "logps/chosen": -1.3530676364898682, + "logps/rejected": -1.4674022197723389, + "loss": 2.1339, + "rewards/accuracies": 0.46875, + "rewards/chosen": -1.3530676364898682, + "rewards/margins": 0.11433436721563339, + "rewards/rejected": -1.4674022197723389, + "semantic_entropy": 0.7847703695297241, + "step": 2620 + }, + { + "epoch": 1.4049172102358254, + "grad_norm": 13.825014034037942, + "learning_rate": 6.405809748488032e-07, + "logits/chosen": 0.0195016972720623, + "logits/rejected": 0.1728542298078537, + "logps/chosen": -1.2841010093688965, + "logps/rejected": -1.5404409170150757, + "loss": 2.0065, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.2841010093688965, + "rewards/margins": 0.25633999705314636, + "rewards/rejected": -1.5404409170150757, + "semantic_entropy": 0.8087183237075806, + "step": 2625 + }, + { + "epoch": 1.4075932430172269, + "grad_norm": 10.399756704030894, + "learning_rate": 6.390857177451956e-07, + "logits/chosen": -0.1649092584848404, + "logits/rejected": 0.03741155192255974, + "logps/chosen": -1.3439829349517822, + "logps/rejected": -1.505408525466919, + "loss": 2.0719, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.3439829349517822, + "rewards/margins": 0.16142557561397552, + "rewards/rejected": -1.505408525466919, + "semantic_entropy": 0.7817584276199341, + "step": 2630 + }, + { + "epoch": 1.4102692757986286, + "grad_norm": 10.735155487069681, + "learning_rate": 6.375891112313445e-07, + "logits/chosen": -0.05074986815452576, + "logits/rejected": 0.03829338029026985, + "logps/chosen": -1.2775318622589111, + "logps/rejected": -1.5022495985031128, + "loss": 1.9985, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2775318622589111, + "rewards/margins": 0.22471757233142853, + "rewards/rejected": -1.5022495985031128, + "semantic_entropy": 0.8041666746139526, + "step": 2635 + }, + { + "epoch": 1.41294530858003, + "grad_norm": 7.290901946889319, + "learning_rate": 6.360911698273326e-07, + "logits/chosen": 0.020272737368941307, + "logits/rejected": 0.10198897123336792, + "logps/chosen": -1.337398886680603, + "logps/rejected": -1.5660853385925293, + "loss": 2.0534, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.337398886680603, + "rewards/margins": 0.22868652641773224, + "rewards/rejected": -1.5660853385925293, + "semantic_entropy": 0.7864670157432556, + "step": 2640 + }, + { + "epoch": 1.4156213413614318, + "grad_norm": 9.046840482319494, + "learning_rate": 6.345919080661944e-07, + "logits/chosen": -0.037472423166036606, + "logits/rejected": 0.04499778151512146, + "logps/chosen": -1.266202688217163, + "logps/rejected": -1.5562307834625244, + "loss": 1.9872, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.266202688217163, + "rewards/margins": 0.2900282144546509, + "rewards/rejected": -1.5562307834625244, + "semantic_entropy": 0.8044629096984863, + "step": 2645 + }, + { + "epoch": 1.4182973741428333, + "grad_norm": 7.03261181708054, + "learning_rate": 6.330913404937737e-07, + "logits/chosen": -0.08614523708820343, + "logits/rejected": 0.07208752632141113, + "logps/chosen": -1.2745702266693115, + "logps/rejected": -1.69574773311615, + "loss": 2.01, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.2745702266693115, + "rewards/margins": 0.42117738723754883, + "rewards/rejected": -1.69574773311615, + "semantic_entropy": 0.7874399423599243, + "step": 2650 + }, + { + "epoch": 1.4209734069242348, + "grad_norm": 9.051777151633472, + "learning_rate": 6.315894816685838e-07, + "logits/chosen": -0.006546747870743275, + "logits/rejected": 0.14669081568717957, + "logps/chosen": -1.173689365386963, + "logps/rejected": -1.4159897565841675, + "loss": 1.9562, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.173689365386963, + "rewards/margins": 0.24230046570301056, + "rewards/rejected": -1.4159897565841675, + "semantic_entropy": 0.8332114219665527, + "step": 2655 + }, + { + "epoch": 1.4236494397056365, + "grad_norm": 8.990035974128704, + "learning_rate": 6.300863461616657e-07, + "logits/chosen": 0.06433503329753876, + "logits/rejected": 0.11419510841369629, + "logps/chosen": -1.165921926498413, + "logps/rejected": -1.4911032915115356, + "loss": 1.9433, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.165921926498413, + "rewards/margins": 0.32518142461776733, + "rewards/rejected": -1.4911032915115356, + "semantic_entropy": 0.8362231254577637, + "step": 2660 + }, + { + "epoch": 1.426325472487038, + "grad_norm": 8.014043391058564, + "learning_rate": 6.285819485564465e-07, + "logits/chosen": -0.10654006153345108, + "logits/rejected": 0.01912633329629898, + "logps/chosen": -1.3031725883483887, + "logps/rejected": -1.5577466487884521, + "loss": 2.0142, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.3031725883483887, + "rewards/margins": 0.2545742392539978, + "rewards/rejected": -1.5577466487884521, + "semantic_entropy": 0.7930922508239746, + "step": 2665 + }, + { + "epoch": 1.4290015052684395, + "grad_norm": 9.160732444578377, + "learning_rate": 6.270763034485986e-07, + "logits/chosen": 0.029206525534391403, + "logits/rejected": 0.1290557086467743, + "logps/chosen": -1.3868186473846436, + "logps/rejected": -1.5575685501098633, + "loss": 2.1021, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.3868186473846436, + "rewards/margins": 0.1707497388124466, + "rewards/rejected": -1.5575685501098633, + "semantic_entropy": 0.7730140686035156, + "step": 2670 + }, + { + "epoch": 1.4316775380498412, + "grad_norm": 11.463702138070392, + "learning_rate": 6.255694254458972e-07, + "logits/chosen": -0.03042779304087162, + "logits/rejected": 0.13861264288425446, + "logps/chosen": -1.3073450326919556, + "logps/rejected": -1.4852049350738525, + "loss": 2.0481, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.3073450326919556, + "rewards/margins": 0.17785976827144623, + "rewards/rejected": -1.4852049350738525, + "semantic_entropy": 0.8076550364494324, + "step": 2675 + }, + { + "epoch": 1.4343535708312427, + "grad_norm": 9.484512890852951, + "learning_rate": 6.240613291680795e-07, + "logits/chosen": -0.04733417183160782, + "logits/rejected": 0.13355064392089844, + "logps/chosen": -1.2979787588119507, + "logps/rejected": -1.4991651773452759, + "loss": 2.0435, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2979787588119507, + "rewards/margins": 0.20118634402751923, + "rewards/rejected": -1.4991651773452759, + "semantic_entropy": 0.7871995568275452, + "step": 2680 + }, + { + "epoch": 1.4370296036126442, + "grad_norm": 9.553528285797984, + "learning_rate": 6.225520292467021e-07, + "logits/chosen": -0.0540904700756073, + "logits/rejected": 0.16049322485923767, + "logps/chosen": -1.2885100841522217, + "logps/rejected": -1.4493513107299805, + "loss": 2.0401, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.2885100841522217, + "rewards/margins": 0.16084131598472595, + "rewards/rejected": -1.4493513107299805, + "semantic_entropy": 0.8091254234313965, + "step": 2685 + }, + { + "epoch": 1.439705636394046, + "grad_norm": 13.887500536632503, + "learning_rate": 6.210415403249993e-07, + "logits/chosen": -0.18344011902809143, + "logits/rejected": 0.06428229063749313, + "logps/chosen": -1.2974615097045898, + "logps/rejected": -1.6270719766616821, + "loss": 2.0105, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.2974615097045898, + "rewards/margins": 0.32961034774780273, + "rewards/rejected": -1.6270719766616821, + "semantic_entropy": 0.7873474359512329, + "step": 2690 + }, + { + "epoch": 1.4423816691754474, + "grad_norm": 6.378348148603819, + "learning_rate": 6.195298770577415e-07, + "logits/chosen": 0.02969251200556755, + "logits/rejected": 0.05600305646657944, + "logps/chosen": -1.2695014476776123, + "logps/rejected": -1.5602524280548096, + "loss": 1.9927, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2695014476776123, + "rewards/margins": 0.2907510995864868, + "rewards/rejected": -1.5602524280548096, + "semantic_entropy": 0.8114160299301147, + "step": 2695 + }, + { + "epoch": 1.445057701956849, + "grad_norm": 8.270961707183284, + "learning_rate": 6.180170541110923e-07, + "logits/chosen": -0.0491093285381794, + "logits/rejected": 0.14849188923835754, + "logps/chosen": -1.313826560974121, + "logps/rejected": -1.5336029529571533, + "loss": 2.0368, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.313826560974121, + "rewards/margins": 0.21977631747722626, + "rewards/rejected": -1.5336029529571533, + "semantic_entropy": 0.7848860025405884, + "step": 2700 + }, + { + "epoch": 1.4477337347382506, + "grad_norm": 6.020138414515257, + "learning_rate": 6.165030861624663e-07, + "logits/chosen": -0.08611325919628143, + "logits/rejected": 0.14223533868789673, + "logps/chosen": -1.1841992139816284, + "logps/rejected": -1.5576660633087158, + "loss": 1.9085, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.1841992139816284, + "rewards/margins": 0.3734667897224426, + "rewards/rejected": -1.5576660633087158, + "semantic_entropy": 0.821341872215271, + "step": 2705 + }, + { + "epoch": 1.4504097675196521, + "grad_norm": 7.926721115653813, + "learning_rate": 6.149879879003876e-07, + "logits/chosen": 0.040991488844156265, + "logits/rejected": 0.0697018951177597, + "logps/chosen": -1.2791852951049805, + "logps/rejected": -1.5475891828536987, + "loss": 1.9896, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2791852951049805, + "rewards/margins": 0.26840391755104065, + "rewards/rejected": -1.5475891828536987, + "semantic_entropy": 0.8002132177352905, + "step": 2710 + }, + { + "epoch": 1.4530858003010536, + "grad_norm": 8.1304948439871, + "learning_rate": 6.13471774024346e-07, + "logits/chosen": -0.09874364733695984, + "logits/rejected": 0.00951166171580553, + "logps/chosen": -1.2177903652191162, + "logps/rejected": -1.4703280925750732, + "loss": 1.9766, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.2177903652191162, + "rewards/margins": 0.2525377869606018, + "rewards/rejected": -1.4703280925750732, + "semantic_entropy": 0.8077287673950195, + "step": 2715 + }, + { + "epoch": 1.4557618330824553, + "grad_norm": 8.550253666468736, + "learning_rate": 6.119544592446551e-07, + "logits/chosen": -0.06126248836517334, + "logits/rejected": 0.05804147571325302, + "logps/chosen": -1.2580182552337646, + "logps/rejected": -1.4090564250946045, + "loss": 2.0226, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.2580182552337646, + "rewards/margins": 0.1510380655527115, + "rewards/rejected": -1.4090564250946045, + "semantic_entropy": 0.8237816095352173, + "step": 2720 + }, + { + "epoch": 1.4584378658638568, + "grad_norm": 9.360312673012285, + "learning_rate": 6.104360582823096e-07, + "logits/chosen": -0.02586626447737217, + "logits/rejected": 0.09422776848077774, + "logps/chosen": -1.2580927610397339, + "logps/rejected": -1.5287189483642578, + "loss": 2.0074, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.2580927610397339, + "rewards/margins": 0.2706260681152344, + "rewards/rejected": -1.5287189483642578, + "semantic_entropy": 0.7973588705062866, + "step": 2725 + }, + { + "epoch": 1.4611138986452583, + "grad_norm": 9.839600679806647, + "learning_rate": 6.089165858688423e-07, + "logits/chosen": -0.05331653356552124, + "logits/rejected": 0.13488610088825226, + "logps/chosen": -1.267230749130249, + "logps/rejected": -1.5607540607452393, + "loss": 2.0204, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.267230749130249, + "rewards/margins": 0.2935234308242798, + "rewards/rejected": -1.5607540607452393, + "semantic_entropy": 0.7877390384674072, + "step": 2730 + }, + { + "epoch": 1.46378993142666, + "grad_norm": 8.101147304031956, + "learning_rate": 6.073960567461811e-07, + "logits/chosen": -0.040810726583004, + "logits/rejected": 0.16771776974201202, + "logps/chosen": -1.1728408336639404, + "logps/rejected": -1.5638388395309448, + "loss": 1.9287, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.1728408336639404, + "rewards/margins": 0.39099812507629395, + "rewards/rejected": -1.5638388395309448, + "semantic_entropy": 0.8042163848876953, + "step": 2735 + }, + { + "epoch": 1.4664659642080615, + "grad_norm": 10.135341944238796, + "learning_rate": 6.058744856665065e-07, + "logits/chosen": -0.0530366376042366, + "logits/rejected": 0.06937140226364136, + "logps/chosen": -1.2185879945755005, + "logps/rejected": -1.5651237964630127, + "loss": 1.9801, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2185879945755005, + "rewards/margins": 0.3465357720851898, + "rewards/rejected": -1.5651237964630127, + "semantic_entropy": 0.8120279312133789, + "step": 2740 + }, + { + "epoch": 1.469141996989463, + "grad_norm": 8.081130306261906, + "learning_rate": 6.043518873921074e-07, + "logits/chosen": -0.0669885128736496, + "logits/rejected": 0.07204239070415497, + "logps/chosen": -1.2283408641815186, + "logps/rejected": -1.4145268201828003, + "loss": 1.9954, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.2283408641815186, + "rewards/margins": 0.18618597090244293, + "rewards/rejected": -1.4145268201828003, + "semantic_entropy": 0.8127212524414062, + "step": 2745 + }, + { + "epoch": 1.4718180297708647, + "grad_norm": 8.6742003855211, + "learning_rate": 6.028282766952393e-07, + "logits/chosen": -0.0030053220689296722, + "logits/rejected": 0.10681009292602539, + "logps/chosen": -1.3068584203720093, + "logps/rejected": -1.609853744506836, + "loss": 1.992, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.3068584203720093, + "rewards/margins": 0.3029954433441162, + "rewards/rejected": -1.609853744506836, + "semantic_entropy": 0.7740689516067505, + "step": 2750 + }, + { + "epoch": 1.4744940625522662, + "grad_norm": 12.341995821257413, + "learning_rate": 6.013036683579798e-07, + "logits/chosen": 0.025272101163864136, + "logits/rejected": 0.17669710516929626, + "logps/chosen": -1.2535477876663208, + "logps/rejected": -1.443696141242981, + "loss": 2.0065, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2535477876663208, + "rewards/margins": 0.19014835357666016, + "rewards/rejected": -1.443696141242981, + "semantic_entropy": 0.8169199228286743, + "step": 2755 + }, + { + "epoch": 1.4771700953336677, + "grad_norm": 9.236585928461313, + "learning_rate": 5.997780771720854e-07, + "logits/chosen": -0.1045481339097023, + "logits/rejected": 0.08253008872270584, + "logps/chosen": -1.2917675971984863, + "logps/rejected": -1.6060972213745117, + "loss": 2.018, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2917675971984863, + "rewards/margins": 0.3143296539783478, + "rewards/rejected": -1.6060972213745117, + "semantic_entropy": 0.7867814898490906, + "step": 2760 + }, + { + "epoch": 1.4798461281150694, + "grad_norm": 9.887674753689055, + "learning_rate": 5.982515179388486e-07, + "logits/chosen": 0.012802052311599255, + "logits/rejected": 0.15946759283542633, + "logps/chosen": -1.214952826499939, + "logps/rejected": -1.472954511642456, + "loss": 1.9618, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.214952826499939, + "rewards/margins": 0.25800177454948425, + "rewards/rejected": -1.472954511642456, + "semantic_entropy": 0.8072951436042786, + "step": 2765 + }, + { + "epoch": 1.482522160896471, + "grad_norm": 9.546052846550923, + "learning_rate": 5.967240054689541e-07, + "logits/chosen": -0.06388188898563385, + "logits/rejected": 0.01827801950275898, + "logps/chosen": -1.2447795867919922, + "logps/rejected": -1.3526091575622559, + "loss": 2.0449, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.2447795867919922, + "rewards/margins": 0.1078295111656189, + "rewards/rejected": -1.3526091575622559, + "semantic_entropy": 0.8271406292915344, + "step": 2770 + }, + { + "epoch": 1.4851981936778724, + "grad_norm": 6.197965780271343, + "learning_rate": 5.951955545823342e-07, + "logits/chosen": -0.01683109626173973, + "logits/rejected": 0.030049163848161697, + "logps/chosen": -1.2154908180236816, + "logps/rejected": -1.4938910007476807, + "loss": 1.9702, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2154908180236816, + "rewards/margins": 0.2784002125263214, + "rewards/rejected": -1.4938910007476807, + "semantic_entropy": 0.8202184438705444, + "step": 2775 + }, + { + "epoch": 1.4878742264592741, + "grad_norm": 5.786231493222208, + "learning_rate": 5.936661801080263e-07, + "logits/chosen": -0.02936570905148983, + "logits/rejected": 0.07508943974971771, + "logps/chosen": -1.3776317834854126, + "logps/rejected": -1.574289321899414, + "loss": 2.0838, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.3776317834854126, + "rewards/margins": 0.1966576725244522, + "rewards/rejected": -1.574289321899414, + "semantic_entropy": 0.7650582194328308, + "step": 2780 + }, + { + "epoch": 1.4905502592406756, + "grad_norm": 8.22099606926176, + "learning_rate": 5.92135896884028e-07, + "logits/chosen": -0.0665474608540535, + "logits/rejected": 0.07018626481294632, + "logps/chosen": -1.3456532955169678, + "logps/rejected": -1.592674732208252, + "loss": 2.0659, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.3456532955169678, + "rewards/margins": 0.24702143669128418, + "rewards/rejected": -1.592674732208252, + "semantic_entropy": 0.7766121625900269, + "step": 2785 + }, + { + "epoch": 1.4932262920220774, + "grad_norm": 13.009909043873694, + "learning_rate": 5.906047197571541e-07, + "logits/chosen": -0.002771927509456873, + "logits/rejected": -0.02358252741396427, + "logps/chosen": -1.2346235513687134, + "logps/rejected": -1.4999468326568604, + "loss": 1.9989, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2346235513687134, + "rewards/margins": 0.2653234004974365, + "rewards/rejected": -1.4999468326568604, + "semantic_entropy": 0.8137930631637573, + "step": 2790 + }, + { + "epoch": 1.4959023248034788, + "grad_norm": 7.600751860354241, + "learning_rate": 5.890726635828919e-07, + "logits/chosen": 0.08247774839401245, + "logits/rejected": 0.09755951166152954, + "logps/chosen": -1.2009843587875366, + "logps/rejected": -1.4116266965866089, + "loss": 2.0229, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.2009843587875366, + "rewards/margins": 0.21064254641532898, + "rewards/rejected": -1.4116266965866089, + "semantic_entropy": 0.8201521039009094, + "step": 2795 + }, + { + "epoch": 1.4985783575848803, + "grad_norm": 7.9667357221109905, + "learning_rate": 5.875397432252569e-07, + "logits/chosen": -0.10117968171834946, + "logits/rejected": -0.018336813896894455, + "logps/chosen": -1.3454875946044922, + "logps/rejected": -1.6285938024520874, + "loss": 2.0557, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.3454875946044922, + "rewards/margins": 0.2831062972545624, + "rewards/rejected": -1.6285938024520874, + "semantic_entropy": 0.7585874199867249, + "step": 2800 + }, + { + "epoch": 1.4985783575848803, + "eval_logits/chosen": 0.3579806685447693, + "eval_logits/rejected": 0.4562504291534424, + "eval_logps/chosen": -1.3279922008514404, + "eval_logps/rejected": -1.587229609489441, + "eval_loss": 2.043422222137451, + "eval_rewards/accuracies": 0.5845697522163391, + "eval_rewards/chosen": -1.3279922008514404, + "eval_rewards/margins": 0.2592373490333557, + "eval_rewards/rejected": -1.587229609489441, + "eval_runtime": 34.5745, + "eval_samples_per_second": 38.902, + "eval_semantic_entropy": 0.7788424491882324, + "eval_steps_per_second": 9.747, + "step": 2800 + }, + { + "epoch": 1.5012543903662818, + "grad_norm": 7.132846494980536, + "learning_rate": 5.860059735566491e-07, + "logits/chosen": -0.17691846191883087, + "logits/rejected": -0.016455356031656265, + "logps/chosen": -1.176946997642517, + "logps/rejected": -1.488738775253296, + "loss": 1.946, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -1.176946997642517, + "rewards/margins": 0.3117918372154236, + "rewards/rejected": -1.488738775253296, + "semantic_entropy": 0.8184489011764526, + "step": 2805 + }, + { + "epoch": 1.5039304231476835, + "grad_norm": 17.289937745543508, + "learning_rate": 5.844713694577087e-07, + "logits/chosen": -0.042862921953201294, + "logits/rejected": 0.0031647428404539824, + "logps/chosen": -1.3153235912322998, + "logps/rejected": -1.5819628238677979, + "loss": 2.0377, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.3153235912322998, + "rewards/margins": 0.2666395306587219, + "rewards/rejected": -1.5819628238677979, + "semantic_entropy": 0.7687281966209412, + "step": 2810 + }, + { + "epoch": 1.5066064559290853, + "grad_norm": 7.73698652084708, + "learning_rate": 5.829359458171714e-07, + "logits/chosen": 0.019528161734342575, + "logits/rejected": 0.14965340495109558, + "logps/chosen": -1.308516502380371, + "logps/rejected": -1.5800927877426147, + "loss": 2.0036, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.308516502380371, + "rewards/margins": 0.2715762257575989, + "rewards/rejected": -1.5800927877426147, + "semantic_entropy": 0.7879072427749634, + "step": 2815 + }, + { + "epoch": 1.5092824887104868, + "grad_norm": 6.855133463366414, + "learning_rate": 5.81399717531724e-07, + "logits/chosen": 0.004591824021190405, + "logits/rejected": 0.19138944149017334, + "logps/chosen": -1.275131344795227, + "logps/rejected": -1.4118839502334595, + "loss": 2.0545, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.275131344795227, + "rewards/margins": 0.13675281405448914, + "rewards/rejected": -1.4118839502334595, + "semantic_entropy": 0.8109269142150879, + "step": 2820 + }, + { + "epoch": 1.5119585214918883, + "grad_norm": 8.422960613579917, + "learning_rate": 5.798626995058602e-07, + "logits/chosen": -0.07312478125095367, + "logits/rejected": 0.10833685100078583, + "logps/chosen": -1.3415075540542603, + "logps/rejected": -1.6823097467422485, + "loss": 2.0298, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.3415075540542603, + "rewards/margins": 0.3408019542694092, + "rewards/rejected": -1.6823097467422485, + "semantic_entropy": 0.7743151783943176, + "step": 2825 + }, + { + "epoch": 1.51463455427329, + "grad_norm": 7.278580407920762, + "learning_rate": 5.783249066517354e-07, + "logits/chosen": 0.008635302074253559, + "logits/rejected": 0.1758573055267334, + "logps/chosen": -1.3390541076660156, + "logps/rejected": -1.4160137176513672, + "loss": 2.1078, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -1.3390541076660156, + "rewards/margins": 0.07695959508419037, + "rewards/rejected": -1.4160137176513672, + "semantic_entropy": 0.7865692377090454, + "step": 2830 + }, + { + "epoch": 1.5173105870546915, + "grad_norm": 10.345320269143327, + "learning_rate": 5.767863538890228e-07, + "logits/chosen": -0.028357133269309998, + "logits/rejected": 0.14543204009532928, + "logps/chosen": -1.264595627784729, + "logps/rejected": -1.498169183731079, + "loss": 2.0073, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.264595627784729, + "rewards/margins": 0.23357343673706055, + "rewards/rejected": -1.498169183731079, + "semantic_entropy": 0.8044542074203491, + "step": 2835 + }, + { + "epoch": 1.519986619836093, + "grad_norm": 9.129057194395378, + "learning_rate": 5.75247056144768e-07, + "logits/chosen": 0.0071360827423632145, + "logits/rejected": 0.11982695013284683, + "logps/chosen": -1.2875782251358032, + "logps/rejected": -1.480312705039978, + "loss": 2.0147, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2875782251358032, + "rewards/margins": 0.19273433089256287, + "rewards/rejected": -1.480312705039978, + "semantic_entropy": 0.7945183515548706, + "step": 2840 + }, + { + "epoch": 1.5226626526174947, + "grad_norm": 9.43707371227767, + "learning_rate": 5.737070283532444e-07, + "logits/chosen": -0.01443692110478878, + "logits/rejected": 0.07831753045320511, + "logps/chosen": -1.2341654300689697, + "logps/rejected": -1.5503387451171875, + "loss": 1.9832, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.2341654300689697, + "rewards/margins": 0.31617334485054016, + "rewards/rejected": -1.5503387451171875, + "semantic_entropy": 0.8021982908248901, + "step": 2845 + }, + { + "epoch": 1.5253386853988962, + "grad_norm": 13.105256203412493, + "learning_rate": 5.721662854558084e-07, + "logits/chosen": -0.058693576604127884, + "logits/rejected": 0.02043284848332405, + "logps/chosen": -1.3020961284637451, + "logps/rejected": -1.544008493423462, + "loss": 2.0443, + "rewards/accuracies": 0.53125, + "rewards/chosen": -1.3020961284637451, + "rewards/margins": 0.24191224575042725, + "rewards/rejected": -1.544008493423462, + "semantic_entropy": 0.8002818822860718, + "step": 2850 + }, + { + "epoch": 1.5280147181802977, + "grad_norm": 7.099331223706971, + "learning_rate": 5.706248424007545e-07, + "logits/chosen": -0.04099093750119209, + "logits/rejected": 0.13332320749759674, + "logps/chosen": -1.361289620399475, + "logps/rejected": -1.6211292743682861, + "loss": 2.0504, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.361289620399475, + "rewards/margins": 0.2598397433757782, + "rewards/rejected": -1.6211292743682861, + "semantic_entropy": 0.7661951184272766, + "step": 2855 + }, + { + "epoch": 1.5306907509616994, + "grad_norm": 10.367612289081793, + "learning_rate": 5.690827141431699e-07, + "logits/chosen": -0.1250082552433014, + "logits/rejected": 0.06297969073057175, + "logps/chosen": -1.253442406654358, + "logps/rejected": -1.446865439414978, + "loss": 2.0052, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.253442406654358, + "rewards/margins": 0.19342297315597534, + "rewards/rejected": -1.446865439414978, + "semantic_entropy": 0.8168342709541321, + "step": 2860 + }, + { + "epoch": 1.5333667837431009, + "grad_norm": 8.184876215714546, + "learning_rate": 5.675399156447897e-07, + "logits/chosen": -0.18241655826568604, + "logits/rejected": -0.04299313947558403, + "logps/chosen": -1.258354902267456, + "logps/rejected": -1.5374661684036255, + "loss": 2.0191, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.258354902267456, + "rewards/margins": 0.279111385345459, + "rewards/rejected": -1.5374661684036255, + "semantic_entropy": 0.8082086443901062, + "step": 2865 + }, + { + "epoch": 1.5360428165245024, + "grad_norm": 10.230428213029905, + "learning_rate": 5.659964618738515e-07, + "logits/chosen": -0.06168074160814285, + "logits/rejected": 0.06839561462402344, + "logps/chosen": -1.3128324747085571, + "logps/rejected": -1.4371354579925537, + "loss": 2.0895, + "rewards/accuracies": 0.4937500059604645, + "rewards/chosen": -1.3128324747085571, + "rewards/margins": 0.12430305778980255, + "rewards/rejected": -1.4371354579925537, + "semantic_entropy": 0.7939896583557129, + "step": 2870 + }, + { + "epoch": 1.538718849305904, + "grad_norm": 8.389267495481054, + "learning_rate": 5.644523678049509e-07, + "logits/chosen": -0.04819143936038017, + "logits/rejected": 0.055794280022382736, + "logps/chosen": -1.2973073720932007, + "logps/rejected": -1.4822477102279663, + "loss": 2.0293, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.2973073720932007, + "rewards/margins": 0.184940367937088, + "rewards/rejected": -1.4822477102279663, + "semantic_entropy": 0.7976966500282288, + "step": 2875 + }, + { + "epoch": 1.5413948820873056, + "grad_norm": 10.857288352616921, + "learning_rate": 5.629076484188952e-07, + "logits/chosen": 0.04812438413500786, + "logits/rejected": 0.16079124808311462, + "logps/chosen": -1.2496439218521118, + "logps/rejected": -1.5407159328460693, + "loss": 2.0209, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.2496439218521118, + "rewards/margins": 0.2910721004009247, + "rewards/rejected": -1.5407159328460693, + "semantic_entropy": 0.8148852586746216, + "step": 2880 + }, + { + "epoch": 1.544070914868707, + "grad_norm": 8.342141881463323, + "learning_rate": 5.613623187025587e-07, + "logits/chosen": -0.038243114948272705, + "logits/rejected": 0.08904510736465454, + "logps/chosen": -1.2702000141143799, + "logps/rejected": -1.5126235485076904, + "loss": 2.0171, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2702000141143799, + "rewards/margins": 0.242423415184021, + "rewards/rejected": -1.5126235485076904, + "semantic_entropy": 0.8042590022087097, + "step": 2885 + }, + { + "epoch": 1.5467469476501088, + "grad_norm": 9.564009717517427, + "learning_rate": 5.598163936487369e-07, + "logits/chosen": -0.1139940619468689, + "logits/rejected": 0.07968755066394806, + "logps/chosen": -1.2677125930786133, + "logps/rejected": -1.4977737665176392, + "loss": 2.0176, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2677125930786133, + "rewards/margins": 0.23006126284599304, + "rewards/rejected": -1.4977737665176392, + "semantic_entropy": 0.8163504600524902, + "step": 2890 + }, + { + "epoch": 1.5494229804315103, + "grad_norm": 8.897773500157538, + "learning_rate": 5.582698882560017e-07, + "logits/chosen": -0.08605276048183441, + "logits/rejected": 0.06514769792556763, + "logps/chosen": -1.2082021236419678, + "logps/rejected": -1.4648030996322632, + "loss": 1.9763, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.2082021236419678, + "rewards/margins": 0.2566010057926178, + "rewards/rejected": -1.4648030996322632, + "semantic_entropy": 0.8213735818862915, + "step": 2895 + }, + { + "epoch": 1.5520990132129118, + "grad_norm": 9.072552859671193, + "learning_rate": 5.567228175285549e-07, + "logits/chosen": -0.020847894251346588, + "logits/rejected": 0.07970386743545532, + "logps/chosen": -1.2829740047454834, + "logps/rejected": -1.522769570350647, + "loss": 2.0128, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2829740047454834, + "rewards/margins": 0.2397955358028412, + "rewards/rejected": -1.522769570350647, + "semantic_entropy": 0.7907953262329102, + "step": 2900 + }, + { + "epoch": 1.5547750459943135, + "grad_norm": 7.7152752664459605, + "learning_rate": 5.551751964760838e-07, + "logits/chosen": 0.05560041591525078, + "logits/rejected": 0.08189483731985092, + "logps/chosen": -1.2490696907043457, + "logps/rejected": -1.4978667497634888, + "loss": 1.9936, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.2490696907043457, + "rewards/margins": 0.24879701435565948, + "rewards/rejected": -1.4978667497634888, + "semantic_entropy": 0.8143652081489563, + "step": 2905 + }, + { + "epoch": 1.557451078775715, + "grad_norm": 8.452751247504208, + "learning_rate": 5.536270401136145e-07, + "logits/chosen": -0.009603270329535007, + "logits/rejected": 0.08335626870393753, + "logps/chosen": -1.22438645362854, + "logps/rejected": -1.4349185228347778, + "loss": 1.9937, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.22438645362854, + "rewards/margins": 0.2105320245027542, + "rewards/rejected": -1.4349185228347778, + "semantic_entropy": 0.8151274919509888, + "step": 2910 + }, + { + "epoch": 1.5601271115571165, + "grad_norm": 13.291563098455539, + "learning_rate": 5.520783634613667e-07, + "logits/chosen": 0.028323788195848465, + "logits/rejected": 0.19542863965034485, + "logps/chosen": -1.3303496837615967, + "logps/rejected": -1.5417304039001465, + "loss": 2.0507, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.3303496837615967, + "rewards/margins": 0.21138079464435577, + "rewards/rejected": -1.5417304039001465, + "semantic_entropy": 0.7754170298576355, + "step": 2915 + }, + { + "epoch": 1.5628031443385182, + "grad_norm": 8.197490163591677, + "learning_rate": 5.505291815446082e-07, + "logits/chosen": 0.02938341535627842, + "logits/rejected": 0.15413501858711243, + "logps/chosen": -1.3015943765640259, + "logps/rejected": -1.5449639558792114, + "loss": 2.0232, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.3015943765640259, + "rewards/margins": 0.24336960911750793, + "rewards/rejected": -1.5449639558792114, + "semantic_entropy": 0.7976529002189636, + "step": 2920 + }, + { + "epoch": 1.5654791771199197, + "grad_norm": 10.372056783410427, + "learning_rate": 5.489795093935089e-07, + "logits/chosen": 0.04805382713675499, + "logits/rejected": 0.11252939701080322, + "logps/chosen": -1.22806978225708, + "logps/rejected": -1.5528632402420044, + "loss": 1.9853, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.22806978225708, + "rewards/margins": 0.3247934877872467, + "rewards/rejected": -1.5528632402420044, + "semantic_entropy": 0.8175485730171204, + "step": 2925 + }, + { + "epoch": 1.5681552099013212, + "grad_norm": 10.419268390877221, + "learning_rate": 5.474293620429946e-07, + "logits/chosen": -0.13174846768379211, + "logits/rejected": 0.04244610667228699, + "logps/chosen": -1.2425198554992676, + "logps/rejected": -1.6323487758636475, + "loss": 1.9664, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.2425198554992676, + "rewards/margins": 0.3898290991783142, + "rewards/rejected": -1.6323487758636475, + "semantic_entropy": 0.798608660697937, + "step": 2930 + }, + { + "epoch": 1.570831242682723, + "grad_norm": 7.620399450088161, + "learning_rate": 5.458787545326018e-07, + "logits/chosen": -0.0838308110833168, + "logits/rejected": 0.06031849980354309, + "logps/chosen": -1.288436770439148, + "logps/rejected": -1.513409972190857, + "loss": 2.0247, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": -1.288436770439148, + "rewards/margins": 0.2249731719493866, + "rewards/rejected": -1.513409972190857, + "semantic_entropy": 0.7873957753181458, + "step": 2935 + }, + { + "epoch": 1.5735072754641244, + "grad_norm": 6.330710597157233, + "learning_rate": 5.443277019063311e-07, + "logits/chosen": -0.06933264434337616, + "logits/rejected": 0.10066770017147064, + "logps/chosen": -1.277112603187561, + "logps/rejected": -1.6146923303604126, + "loss": 2.0084, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.277112603187561, + "rewards/margins": 0.3375798761844635, + "rewards/rejected": -1.6146923303604126, + "semantic_entropy": 0.790891706943512, + "step": 2940 + }, + { + "epoch": 1.5761833082455259, + "grad_norm": 10.827020128865453, + "learning_rate": 5.427762192125023e-07, + "logits/chosen": -0.08359242975711823, + "logits/rejected": 0.06046230345964432, + "logps/chosen": -1.2609506845474243, + "logps/rejected": -1.4343267679214478, + "loss": 2.0267, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.2609506845474243, + "rewards/margins": 0.1733759641647339, + "rewards/rejected": -1.4343267679214478, + "semantic_entropy": 0.8085500597953796, + "step": 2945 + }, + { + "epoch": 1.5788593410269276, + "grad_norm": 11.942780852463356, + "learning_rate": 5.41224321503607e-07, + "logits/chosen": 0.03151168301701546, + "logits/rejected": 0.28194573521614075, + "logps/chosen": -1.2235701084136963, + "logps/rejected": -1.5240552425384521, + "loss": 1.9511, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2235701084136963, + "rewards/margins": 0.3004850745201111, + "rewards/rejected": -1.5240552425384521, + "semantic_entropy": 0.8099506497383118, + "step": 2950 + }, + { + "epoch": 1.5815353738083293, + "grad_norm": 8.354754182965555, + "learning_rate": 5.396720238361637e-07, + "logits/chosen": 0.023025449365377426, + "logits/rejected": 0.12840992212295532, + "logps/chosen": -1.242348313331604, + "logps/rejected": -1.498448133468628, + "loss": 1.9863, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.242348313331604, + "rewards/margins": 0.2560999393463135, + "rewards/rejected": -1.498448133468628, + "semantic_entropy": 0.8004533052444458, + "step": 2955 + }, + { + "epoch": 1.5842114065897306, + "grad_norm": 9.470837718257423, + "learning_rate": 5.381193412705711e-07, + "logits/chosen": -0.07888027280569077, + "logits/rejected": 0.05634929984807968, + "logps/chosen": -1.2643178701400757, + "logps/rejected": -1.4858763217926025, + "loss": 1.9969, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2643178701400757, + "rewards/margins": 0.2215585708618164, + "rewards/rejected": -1.4858763217926025, + "semantic_entropy": 0.8052790760993958, + "step": 2960 + }, + { + "epoch": 1.5868874393711323, + "grad_norm": 11.719959752917669, + "learning_rate": 5.365662888709622e-07, + "logits/chosen": -0.02363435924053192, + "logits/rejected": 0.08364065736532211, + "logps/chosen": -1.2288060188293457, + "logps/rejected": -1.4702298641204834, + "loss": 1.9937, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2288060188293457, + "rewards/margins": 0.24142387509346008, + "rewards/rejected": -1.4702298641204834, + "semantic_entropy": 0.8168516159057617, + "step": 2965 + }, + { + "epoch": 1.589563472152534, + "grad_norm": 13.767804991891948, + "learning_rate": 5.350128817050585e-07, + "logits/chosen": -0.04800247400999069, + "logits/rejected": 0.14096921682357788, + "logps/chosen": -1.3195701837539673, + "logps/rejected": -1.5222375392913818, + "loss": 2.0751, + "rewards/accuracies": 0.53125, + "rewards/chosen": -1.3195701837539673, + "rewards/margins": 0.20266716182231903, + "rewards/rejected": -1.5222375392913818, + "semantic_entropy": 0.7949842810630798, + "step": 2970 + }, + { + "epoch": 1.5922395049339353, + "grad_norm": 9.10072268997601, + "learning_rate": 5.334591348440229e-07, + "logits/chosen": 0.010277917608618736, + "logits/rejected": 0.16392071545124054, + "logps/chosen": -1.2994472980499268, + "logps/rejected": -1.6308130025863647, + "loss": 2.0031, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2994472980499268, + "rewards/margins": 0.3313658535480499, + "rewards/rejected": -1.6308130025863647, + "semantic_entropy": 0.7773244380950928, + "step": 2975 + }, + { + "epoch": 1.594915537715337, + "grad_norm": 11.464691228439937, + "learning_rate": 5.319050633623141e-07, + "logits/chosen": -0.10767938196659088, + "logits/rejected": 0.06060751527547836, + "logps/chosen": -1.339794397354126, + "logps/rejected": -1.5700126886367798, + "loss": 2.0697, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.339794397354126, + "rewards/margins": 0.2302185595035553, + "rewards/rejected": -1.5700126886367798, + "semantic_entropy": 0.7720788717269897, + "step": 2980 + }, + { + "epoch": 1.5975915704967387, + "grad_norm": 6.312446403147627, + "learning_rate": 5.303506823375409e-07, + "logits/chosen": -0.07872191816568375, + "logits/rejected": 0.11133910715579987, + "logps/chosen": -1.3662869930267334, + "logps/rejected": -1.5414142608642578, + "loss": 2.079, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.3662869930267334, + "rewards/margins": 0.17512717843055725, + "rewards/rejected": -1.5414142608642578, + "semantic_entropy": 0.7808175086975098, + "step": 2985 + }, + { + "epoch": 1.60026760327814, + "grad_norm": 8.842807601753613, + "learning_rate": 5.287960068503143e-07, + "logits/chosen": -0.05272887274622917, + "logits/rejected": 0.13576534390449524, + "logps/chosen": -1.2263929843902588, + "logps/rejected": -1.5741432905197144, + "loss": 1.9769, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -1.2263929843902588, + "rewards/margins": 0.34775030612945557, + "rewards/rejected": -1.5741432905197144, + "semantic_entropy": 0.8109342455863953, + "step": 2990 + }, + { + "epoch": 1.6029436360595417, + "grad_norm": 9.090297915482498, + "learning_rate": 5.272410519841032e-07, + "logits/chosen": -0.02721552550792694, + "logits/rejected": 0.0816565752029419, + "logps/chosen": -1.3226561546325684, + "logps/rejected": -1.6543006896972656, + "loss": 2.0125, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.3226561546325684, + "rewards/margins": 0.33164435625076294, + "rewards/rejected": -1.6543006896972656, + "semantic_entropy": 0.7670749425888062, + "step": 2995 + }, + { + "epoch": 1.6056196688409434, + "grad_norm": 7.089363880096743, + "learning_rate": 5.256858328250861e-07, + "logits/chosen": -0.07572749257087708, + "logits/rejected": 0.0714503601193428, + "logps/chosen": -1.3386659622192383, + "logps/rejected": -1.5658314228057861, + "loss": 2.0894, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.3386659622192383, + "rewards/margins": 0.22716538608074188, + "rewards/rejected": -1.5658314228057861, + "semantic_entropy": 0.7847882509231567, + "step": 3000 + }, + { + "epoch": 1.608295701622345, + "grad_norm": 8.59220488525354, + "learning_rate": 5.241303644620063e-07, + "logits/chosen": -0.14671023190021515, + "logits/rejected": -0.00781182711943984, + "logps/chosen": -1.2366340160369873, + "logps/rejected": -1.4559341669082642, + "loss": 2.0106, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2366340160369873, + "rewards/margins": 0.21930010616779327, + "rewards/rejected": -1.4559341669082642, + "semantic_entropy": 0.8152686357498169, + "step": 3005 + }, + { + "epoch": 1.6109717344037464, + "grad_norm": 10.880437653964645, + "learning_rate": 5.225746619860248e-07, + "logits/chosen": -0.09534215927124023, + "logits/rejected": 0.04673149809241295, + "logps/chosen": -1.2736543416976929, + "logps/rejected": -1.4805123805999756, + "loss": 2.0545, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.2736543416976929, + "rewards/margins": 0.20685787498950958, + "rewards/rejected": -1.4805123805999756, + "semantic_entropy": 0.8125940561294556, + "step": 3010 + }, + { + "epoch": 1.6136477671851481, + "grad_norm": 9.259488437555573, + "learning_rate": 5.210187404905735e-07, + "logits/chosen": 0.06914304196834564, + "logits/rejected": 0.14941935241222382, + "logps/chosen": -1.2673017978668213, + "logps/rejected": -1.542967438697815, + "loss": 1.9873, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.2673017978668213, + "rewards/margins": 0.2756657004356384, + "rewards/rejected": -1.542967438697815, + "semantic_entropy": 0.7979117631912231, + "step": 3015 + }, + { + "epoch": 1.6163237999665496, + "grad_norm": 8.235738095463073, + "learning_rate": 5.194626150712098e-07, + "logits/chosen": -0.13079389929771423, + "logits/rejected": 0.02636866271495819, + "logps/chosen": -1.2667205333709717, + "logps/rejected": -1.4473745822906494, + "loss": 2.0273, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.2667205333709717, + "rewards/margins": 0.18065424263477325, + "rewards/rejected": -1.4473745822906494, + "semantic_entropy": 0.8098528981208801, + "step": 3020 + }, + { + "epoch": 1.6189998327479511, + "grad_norm": 8.377388523055298, + "learning_rate": 5.179063008254695e-07, + "logits/chosen": -0.06490382552146912, + "logits/rejected": 0.08870285749435425, + "logps/chosen": -1.232604742050171, + "logps/rejected": -1.4257268905639648, + "loss": 2.011, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.232604742050171, + "rewards/margins": 0.19312207400798798, + "rewards/rejected": -1.4257268905639648, + "semantic_entropy": 0.8080936670303345, + "step": 3025 + }, + { + "epoch": 1.6216758655293528, + "grad_norm": 6.092592887944284, + "learning_rate": 5.163498128527199e-07, + "logits/chosen": -0.02501394972205162, + "logits/rejected": 0.12650498747825623, + "logps/chosen": -1.321905493736267, + "logps/rejected": -1.5000231266021729, + "loss": 2.0363, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.321905493736267, + "rewards/margins": 0.17811742424964905, + "rewards/rejected": -1.5000231266021729, + "semantic_entropy": 0.7947280406951904, + "step": 3030 + }, + { + "epoch": 1.6243518983107543, + "grad_norm": 8.418555595272252, + "learning_rate": 5.147931662540144e-07, + "logits/chosen": 0.05391792207956314, + "logits/rejected": 0.17616690695285797, + "logps/chosen": -1.2905899286270142, + "logps/rejected": -1.4090840816497803, + "loss": 2.0586, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.2905899286270142, + "rewards/margins": 0.11849413812160492, + "rewards/rejected": -1.4090840816497803, + "semantic_entropy": 0.8115051984786987, + "step": 3035 + }, + { + "epoch": 1.6270279310921558, + "grad_norm": 12.220354986532154, + "learning_rate": 5.132363761319449e-07, + "logits/chosen": -0.05254555493593216, + "logits/rejected": 0.010517707094550133, + "logps/chosen": -1.2007555961608887, + "logps/rejected": -1.5174579620361328, + "loss": 1.9429, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.2007555961608887, + "rewards/margins": 0.31670236587524414, + "rewards/rejected": -1.5174579620361328, + "semantic_entropy": 0.809045135974884, + "step": 3040 + }, + { + "epoch": 1.6297039638735575, + "grad_norm": 15.788139287566645, + "learning_rate": 5.116794575904962e-07, + "logits/chosen": -0.06810857355594635, + "logits/rejected": 0.0314619354903698, + "logps/chosen": -1.2489879131317139, + "logps/rejected": -1.3966195583343506, + "loss": 2.037, + "rewards/accuracies": 0.53125, + "rewards/chosen": -1.2489879131317139, + "rewards/margins": 0.14763149619102478, + "rewards/rejected": -1.3966195583343506, + "semantic_entropy": 0.8127719163894653, + "step": 3045 + }, + { + "epoch": 1.632379996654959, + "grad_norm": 9.279317596842828, + "learning_rate": 5.101224257348987e-07, + "logits/chosen": -0.10826786607503891, + "logits/rejected": 0.04793906211853027, + "logps/chosen": -1.3108704090118408, + "logps/rejected": -1.6074644327163696, + "loss": 2.0063, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.3108704090118408, + "rewards/margins": 0.2965940833091736, + "rewards/rejected": -1.6074644327163696, + "semantic_entropy": 0.7802410125732422, + "step": 3050 + }, + { + "epoch": 1.6350560294363605, + "grad_norm": 7.285636339025404, + "learning_rate": 5.085652956714823e-07, + "logits/chosen": -0.11437921226024628, + "logits/rejected": 0.03067811205983162, + "logps/chosen": -1.2466702461242676, + "logps/rejected": -1.534493327140808, + "loss": 1.9999, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2466702461242676, + "rewards/margins": 0.28782323002815247, + "rewards/rejected": -1.534493327140808, + "semantic_entropy": 0.8040468096733093, + "step": 3055 + }, + { + "epoch": 1.6377320622177622, + "grad_norm": 7.061985397071907, + "learning_rate": 5.070080825075298e-07, + "logits/chosen": -0.11119749397039413, + "logits/rejected": 0.06831619143486023, + "logps/chosen": -1.2897229194641113, + "logps/rejected": -1.5156347751617432, + "loss": 2.0244, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2897229194641113, + "rewards/margins": 0.22591181099414825, + "rewards/rejected": -1.5156347751617432, + "semantic_entropy": 0.7982439398765564, + "step": 3060 + }, + { + "epoch": 1.6404080949991637, + "grad_norm": 8.674746275808152, + "learning_rate": 5.0545080135113e-07, + "logits/chosen": -0.006783929653465748, + "logits/rejected": 0.03824920952320099, + "logps/chosen": -1.2787463665008545, + "logps/rejected": -1.598751187324524, + "loss": 1.9926, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.2787463665008545, + "rewards/margins": 0.3200048804283142, + "rewards/rejected": -1.598751187324524, + "semantic_entropy": 0.7831565737724304, + "step": 3065 + }, + { + "epoch": 1.6430841277805652, + "grad_norm": 7.231939797662285, + "learning_rate": 5.038934673110316e-07, + "logits/chosen": -0.12250302731990814, + "logits/rejected": -0.00720958411693573, + "logps/chosen": -1.2766475677490234, + "logps/rejected": -1.5290186405181885, + "loss": 2.0108, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2766475677490234, + "rewards/margins": 0.2523711621761322, + "rewards/rejected": -1.5290186405181885, + "semantic_entropy": 0.7957003116607666, + "step": 3070 + }, + { + "epoch": 1.645760160561967, + "grad_norm": 7.122051956828495, + "learning_rate": 5.023360954964963e-07, + "logits/chosen": -0.13707543909549713, + "logits/rejected": -0.06735747307538986, + "logps/chosen": -1.2285687923431396, + "logps/rejected": -1.4890555143356323, + "loss": 1.9769, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2285687923431396, + "rewards/margins": 0.2604869306087494, + "rewards/rejected": -1.4890555143356323, + "semantic_entropy": 0.8113986253738403, + "step": 3075 + }, + { + "epoch": 1.6484361933433684, + "grad_norm": 9.025536229209312, + "learning_rate": 5.007787010171524e-07, + "logits/chosen": -0.1931755542755127, + "logits/rejected": -0.0045111337676644325, + "logps/chosen": -1.18930983543396, + "logps/rejected": -1.4476261138916016, + "loss": 1.9691, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.18930983543396, + "rewards/margins": 0.25831639766693115, + "rewards/rejected": -1.4476261138916016, + "semantic_entropy": 0.8228625059127808, + "step": 3080 + }, + { + "epoch": 1.65111222612477, + "grad_norm": 9.40683918898441, + "learning_rate": 4.992212989828477e-07, + "logits/chosen": 0.007282339967787266, + "logits/rejected": 0.015585768036544323, + "logps/chosen": -1.2093725204467773, + "logps/rejected": -1.4898974895477295, + "loss": 1.9669, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.2093725204467773, + "rewards/margins": 0.2805247902870178, + "rewards/rejected": -1.4898974895477295, + "semantic_entropy": 0.8125426173210144, + "step": 3085 + }, + { + "epoch": 1.6537882589061716, + "grad_norm": 8.620227561484512, + "learning_rate": 4.976639045035036e-07, + "logits/chosen": 0.028462231159210205, + "logits/rejected": 0.10271923243999481, + "logps/chosen": -1.2567675113677979, + "logps/rejected": -1.4431809186935425, + "loss": 2.0372, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2567675113677979, + "rewards/margins": 0.18641360104084015, + "rewards/rejected": -1.4431809186935425, + "semantic_entropy": 0.8090243339538574, + "step": 3090 + }, + { + "epoch": 1.6564642916875731, + "grad_norm": 9.715546359495004, + "learning_rate": 4.961065326889683e-07, + "logits/chosen": -0.03923254460096359, + "logits/rejected": 0.10990728437900543, + "logps/chosen": -1.282456398010254, + "logps/rejected": -1.5208022594451904, + "loss": 2.0179, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.282456398010254, + "rewards/margins": 0.23834581673145294, + "rewards/rejected": -1.5208022594451904, + "semantic_entropy": 0.8012547492980957, + "step": 3095 + }, + { + "epoch": 1.6591403244689746, + "grad_norm": 10.009918519263168, + "learning_rate": 4.9454919864887e-07, + "logits/chosen": -0.16790278255939484, + "logits/rejected": -0.028138387948274612, + "logps/chosen": -1.3211452960968018, + "logps/rejected": -1.5278136730194092, + "loss": 2.0624, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.3211452960968018, + "rewards/margins": 0.20666833221912384, + "rewards/rejected": -1.5278136730194092, + "semantic_entropy": 0.7871449589729309, + "step": 3100 + }, + { + "epoch": 1.6618163572503764, + "grad_norm": 10.855910976334592, + "learning_rate": 4.929919174924701e-07, + "logits/chosen": -0.10906453430652618, + "logits/rejected": 0.08404166996479034, + "logps/chosen": -1.3063828945159912, + "logps/rejected": -1.53128182888031, + "loss": 2.0223, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.3063828945159912, + "rewards/margins": 0.2248990535736084, + "rewards/rejected": -1.53128182888031, + "semantic_entropy": 0.7838009595870972, + "step": 3105 + }, + { + "epoch": 1.6644923900317778, + "grad_norm": 7.404272070267876, + "learning_rate": 4.914347043285177e-07, + "logits/chosen": -0.04257003217935562, + "logits/rejected": 0.07210078090429306, + "logps/chosen": -1.2872350215911865, + "logps/rejected": -1.5439434051513672, + "loss": 2.0154, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.2872350215911865, + "rewards/margins": 0.2567083239555359, + "rewards/rejected": -1.5439434051513672, + "semantic_entropy": 0.7958934903144836, + "step": 3110 + }, + { + "epoch": 1.6671684228131793, + "grad_norm": 7.985100484489514, + "learning_rate": 4.898775742651013e-07, + "logits/chosen": 0.02354573830962181, + "logits/rejected": 0.1122712641954422, + "logps/chosen": -1.291019082069397, + "logps/rejected": -1.5914843082427979, + "loss": 1.9917, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.291019082069397, + "rewards/margins": 0.30046507716178894, + "rewards/rejected": -1.5914843082427979, + "semantic_entropy": 0.777329683303833, + "step": 3115 + }, + { + "epoch": 1.669844455594581, + "grad_norm": 6.377919696230422, + "learning_rate": 4.883205424095037e-07, + "logits/chosen": -0.10408266633749008, + "logits/rejected": 0.05661571025848389, + "logps/chosen": -1.3499923944473267, + "logps/rejected": -1.600716233253479, + "loss": 2.0779, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.3499923944473267, + "rewards/margins": 0.25072377920150757, + "rewards/rejected": -1.600716233253479, + "semantic_entropy": 0.7723182439804077, + "step": 3120 + }, + { + "epoch": 1.6725204883759828, + "grad_norm": 7.489392193604876, + "learning_rate": 4.86763623868055e-07, + "logits/chosen": 0.002272368874400854, + "logits/rejected": 0.12083474546670914, + "logps/chosen": -1.3575466871261597, + "logps/rejected": -1.654810905456543, + "loss": 2.0625, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.3575466871261597, + "rewards/margins": 0.297264039516449, + "rewards/rejected": -1.654810905456543, + "semantic_entropy": 0.7716215252876282, + "step": 3125 + }, + { + "epoch": 1.675196521157384, + "grad_norm": 7.096207569657256, + "learning_rate": 4.852068337459856e-07, + "logits/chosen": 0.01183260791003704, + "logits/rejected": 0.16132783889770508, + "logps/chosen": -1.3407800197601318, + "logps/rejected": -1.5702488422393799, + "loss": 2.0401, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.3407800197601318, + "rewards/margins": 0.22946885228157043, + "rewards/rejected": -1.5702488422393799, + "semantic_entropy": 0.7798231244087219, + "step": 3130 + }, + { + "epoch": 1.6778725539387858, + "grad_norm": 7.565280374118657, + "learning_rate": 4.8365018714728e-07, + "logits/chosen": 0.05541212111711502, + "logits/rejected": 0.11548095941543579, + "logps/chosen": -1.3500267267227173, + "logps/rejected": -1.5410051345825195, + "loss": 2.0785, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.3500267267227173, + "rewards/margins": 0.19097848236560822, + "rewards/rejected": -1.5410051345825195, + "semantic_entropy": 0.776996910572052, + "step": 3135 + }, + { + "epoch": 1.6805485867201875, + "grad_norm": 6.038134474525186, + "learning_rate": 4.820936991745304e-07, + "logits/chosen": -0.20805136859416962, + "logits/rejected": -0.044590163975954056, + "logps/chosen": -1.2141269445419312, + "logps/rejected": -1.3900539875030518, + "loss": 1.9992, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.2141269445419312, + "rewards/margins": 0.17592690885066986, + "rewards/rejected": -1.3900539875030518, + "semantic_entropy": 0.8216248750686646, + "step": 3140 + }, + { + "epoch": 1.6832246195015887, + "grad_norm": 8.710741282482623, + "learning_rate": 4.8053738492879e-07, + "logits/chosen": 0.0013846077490597963, + "logits/rejected": 0.147234708070755, + "logps/chosen": -1.2627509832382202, + "logps/rejected": -1.4728825092315674, + "loss": 2.0147, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.2627509832382202, + "rewards/margins": 0.2101314812898636, + "rewards/rejected": -1.4728825092315674, + "semantic_entropy": 0.8021093606948853, + "step": 3145 + }, + { + "epoch": 1.6859006522829905, + "grad_norm": 8.204856396695087, + "learning_rate": 4.789812595094265e-07, + "logits/chosen": -0.14415964484214783, + "logits/rejected": -0.010510803200304508, + "logps/chosen": -1.3389784097671509, + "logps/rejected": -1.546367883682251, + "loss": 2.0382, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.3389784097671509, + "rewards/margins": 0.20738950371742249, + "rewards/rejected": -1.546367883682251, + "semantic_entropy": 0.7847117781639099, + "step": 3150 + }, + { + "epoch": 1.6885766850643922, + "grad_norm": 14.18752375431332, + "learning_rate": 4.774253380139752e-07, + "logits/chosen": -0.1542137712240219, + "logits/rejected": -0.03455689921975136, + "logps/chosen": -1.2210431098937988, + "logps/rejected": -1.4889132976531982, + "loss": 1.977, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.2210431098937988, + "rewards/margins": 0.2678702473640442, + "rewards/rejected": -1.4889132976531982, + "semantic_entropy": 0.8089672923088074, + "step": 3155 + }, + { + "epoch": 1.6912527178457935, + "grad_norm": 8.584676492997804, + "learning_rate": 4.758696355379936e-07, + "logits/chosen": -0.11684201657772064, + "logits/rejected": -0.09880466759204865, + "logps/chosen": -1.2682487964630127, + "logps/rejected": -1.5760184526443481, + "loss": 2.0007, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2682487964630127, + "rewards/margins": 0.3077697455883026, + "rewards/rejected": -1.5760184526443481, + "semantic_entropy": 0.797630250453949, + "step": 3160 + }, + { + "epoch": 1.6939287506271952, + "grad_norm": 6.252793796460734, + "learning_rate": 4.743141671749138e-07, + "logits/chosen": -0.18824736773967743, + "logits/rejected": -0.098078653216362, + "logps/chosen": -1.3107153177261353, + "logps/rejected": -1.4777319431304932, + "loss": 2.0477, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.3107153177261353, + "rewards/margins": 0.16701671481132507, + "rewards/rejected": -1.4777319431304932, + "semantic_entropy": 0.7871649861335754, + "step": 3165 + }, + { + "epoch": 1.6966047834085969, + "grad_norm": 6.388763474026948, + "learning_rate": 4.727589480158968e-07, + "logits/chosen": -0.13319739699363708, + "logits/rejected": -0.029325807467103004, + "logps/chosen": -1.2875155210494995, + "logps/rejected": -1.5287894010543823, + "loss": 2.0377, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.2875155210494995, + "rewards/margins": 0.24127385020256042, + "rewards/rejected": -1.5287894010543823, + "semantic_entropy": 0.7841957807540894, + "step": 3170 + }, + { + "epoch": 1.6992808161899984, + "grad_norm": 11.07062376146642, + "learning_rate": 4.712039931496855e-07, + "logits/chosen": -0.15786674618721008, + "logits/rejected": -0.05507850646972656, + "logps/chosen": -1.2572047710418701, + "logps/rejected": -1.4527876377105713, + "loss": 2.0407, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2572047710418701, + "rewards/margins": 0.19558288156986237, + "rewards/rejected": -1.4527876377105713, + "semantic_entropy": 0.804765522480011, + "step": 3175 + }, + { + "epoch": 1.7019568489713999, + "grad_norm": 5.6188414262791495, + "learning_rate": 4.6964931766245905e-07, + "logits/chosen": -0.01330545824021101, + "logits/rejected": 0.041144389659166336, + "logps/chosen": -1.3052301406860352, + "logps/rejected": -1.6139024496078491, + "loss": 2.0025, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.3052301406860352, + "rewards/margins": 0.3086722791194916, + "rewards/rejected": -1.6139024496078491, + "semantic_entropy": 0.7841047048568726, + "step": 3180 + }, + { + "epoch": 1.7046328817528016, + "grad_norm": 8.862736744649089, + "learning_rate": 4.6809493663768575e-07, + "logits/chosen": -0.06787719577550888, + "logits/rejected": -0.045422784984111786, + "logps/chosen": -1.2365736961364746, + "logps/rejected": -1.436856985092163, + "loss": 1.9958, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2365736961364746, + "rewards/margins": 0.2002832591533661, + "rewards/rejected": -1.436856985092163, + "semantic_entropy": 0.8167963027954102, + "step": 3185 + }, + { + "epoch": 1.707308914534203, + "grad_norm": 9.181454482123595, + "learning_rate": 4.6654086515597716e-07, + "logits/chosen": -0.1357351839542389, + "logits/rejected": 0.02635277807712555, + "logps/chosen": -1.2460029125213623, + "logps/rejected": -1.5762498378753662, + "loss": 1.9446, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2460029125213623, + "rewards/margins": 0.3302469253540039, + "rewards/rejected": -1.5762498378753662, + "semantic_entropy": 0.8032658696174622, + "step": 3190 + }, + { + "epoch": 1.7099849473156046, + "grad_norm": 6.31497143037071, + "learning_rate": 4.6498711829494154e-07, + "logits/chosen": -0.1393432319164276, + "logits/rejected": -0.032652225345373154, + "logps/chosen": -1.2492889165878296, + "logps/rejected": -1.5665004253387451, + "loss": 1.9693, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.2492889165878296, + "rewards/margins": 0.31721144914627075, + "rewards/rejected": -1.5665004253387451, + "semantic_entropy": 0.8023662567138672, + "step": 3195 + }, + { + "epoch": 1.7126609800970063, + "grad_norm": 8.943664344540371, + "learning_rate": 4.6343371112903777e-07, + "logits/chosen": -0.03442186489701271, + "logits/rejected": 0.11945761740207672, + "logps/chosen": -1.3083088397979736, + "logps/rejected": -1.6861575841903687, + "loss": 2.0057, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.3083088397979736, + "rewards/margins": 0.3778485655784607, + "rewards/rejected": -1.6861575841903687, + "semantic_entropy": 0.7777668833732605, + "step": 3200 + }, + { + "epoch": 1.7126609800970063, + "eval_logits/chosen": 0.2875590920448303, + "eval_logits/rejected": 0.3803773522377014, + "eval_logps/chosen": -1.3269991874694824, + "eval_logps/rejected": -1.575361967086792, + "eval_loss": 2.0459630489349365, + "eval_rewards/accuracies": 0.5882789492607117, + "eval_rewards/chosen": -1.3269991874694824, + "eval_rewards/margins": 0.24836291372776031, + "eval_rewards/rejected": -1.575361967086792, + "eval_runtime": 34.5538, + "eval_samples_per_second": 38.925, + "eval_semantic_entropy": 0.7827669978141785, + "eval_steps_per_second": 9.753, + "step": 3200 + }, + { + "epoch": 1.7153370128784078, + "grad_norm": 6.181562002484526, + "learning_rate": 4.618806587294291e-07, + "logits/chosen": -0.19150307774543762, + "logits/rejected": -0.07083725929260254, + "logps/chosen": -1.3303239345550537, + "logps/rejected": -1.5726451873779297, + "loss": 2.0342, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.3303239345550537, + "rewards/margins": 0.24232113361358643, + "rewards/rejected": -1.5726451873779297, + "semantic_entropy": 0.7856124639511108, + "step": 3205 + }, + { + "epoch": 1.7180130456598093, + "grad_norm": 12.92251198732787, + "learning_rate": 4.603279761638365e-07, + "logits/chosen": -0.15394911170005798, + "logits/rejected": -0.04621434956789017, + "logps/chosen": -1.293766975402832, + "logps/rejected": -1.5395978689193726, + "loss": 2.0374, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.293766975402832, + "rewards/margins": 0.24583086371421814, + "rewards/rejected": -1.5395978689193726, + "semantic_entropy": 0.7910820245742798, + "step": 3210 + }, + { + "epoch": 1.720689078441211, + "grad_norm": 10.95795391128249, + "learning_rate": 4.5877567849639315e-07, + "logits/chosen": -0.14600083231925964, + "logits/rejected": -0.021480122581124306, + "logps/chosen": -1.2595031261444092, + "logps/rejected": -1.4939130544662476, + "loss": 2.0118, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2595031261444092, + "rewards/margins": 0.23441000282764435, + "rewards/rejected": -1.4939130544662476, + "semantic_entropy": 0.8064897656440735, + "step": 3215 + }, + { + "epoch": 1.7233651112226125, + "grad_norm": 8.336084511981387, + "learning_rate": 4.572237807874979e-07, + "logits/chosen": -0.1490456759929657, + "logits/rejected": 0.06070408970117569, + "logps/chosen": -1.3603156805038452, + "logps/rejected": -1.6003338098526, + "loss": 2.0626, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.3603156805038452, + "rewards/margins": 0.24001812934875488, + "rewards/rejected": -1.6003338098526, + "semantic_entropy": 0.7771097421646118, + "step": 3220 + }, + { + "epoch": 1.726041144004014, + "grad_norm": 10.327496360033788, + "learning_rate": 4.5567229809366895e-07, + "logits/chosen": -0.1425078958272934, + "logits/rejected": -0.012681936845183372, + "logps/chosen": -1.2058963775634766, + "logps/rejected": -1.4731707572937012, + "loss": 1.9638, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2058963775634766, + "rewards/margins": 0.2672743499279022, + "rewards/rejected": -1.4731707572937012, + "semantic_entropy": 0.8253902196884155, + "step": 3225 + }, + { + "epoch": 1.7287171767854157, + "grad_norm": 7.275516801530044, + "learning_rate": 4.541212454673984e-07, + "logits/chosen": -0.14230267703533173, + "logits/rejected": 0.01516336016356945, + "logps/chosen": -1.2601486444473267, + "logps/rejected": -1.6092818975448608, + "loss": 1.9666, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2601486444473267, + "rewards/margins": 0.34913307428359985, + "rewards/rejected": -1.6092818975448608, + "semantic_entropy": 0.7881345152854919, + "step": 3230 + }, + { + "epoch": 1.7313932095668172, + "grad_norm": 10.07712270276737, + "learning_rate": 4.525706379570055e-07, + "logits/chosen": -0.09800378978252411, + "logits/rejected": -0.029662657529115677, + "logps/chosen": -1.2746299505233765, + "logps/rejected": -1.5383646488189697, + "loss": 2.0011, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.2746299505233765, + "rewards/margins": 0.26373451948165894, + "rewards/rejected": -1.5383646488189697, + "semantic_entropy": 0.8014398813247681, + "step": 3235 + }, + { + "epoch": 1.7340692423482187, + "grad_norm": 6.246134723181422, + "learning_rate": 4.510204906064911e-07, + "logits/chosen": 0.0009855165844783187, + "logits/rejected": 0.11134722083806992, + "logps/chosen": -1.2328336238861084, + "logps/rejected": -1.5697522163391113, + "loss": 1.9354, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -1.2328336238861084, + "rewards/margins": 0.33691850304603577, + "rewards/rejected": -1.5697522163391113, + "semantic_entropy": 0.8106444478034973, + "step": 3240 + }, + { + "epoch": 1.7367452751296204, + "grad_norm": 9.727248624352367, + "learning_rate": 4.4947081845539177e-07, + "logits/chosen": -0.20650625228881836, + "logits/rejected": -0.06619389355182648, + "logps/chosen": -1.2484396696090698, + "logps/rejected": -1.5200622081756592, + "loss": 2.0096, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.2484396696090698, + "rewards/margins": 0.2716224789619446, + "rewards/rejected": -1.5200622081756592, + "semantic_entropy": 0.8025262951850891, + "step": 3245 + }, + { + "epoch": 1.739421307911022, + "grad_norm": 9.681377075377828, + "learning_rate": 4.479216365386333e-07, + "logits/chosen": 0.02583617903292179, + "logits/rejected": 0.17587433755397797, + "logps/chosen": -1.28328537940979, + "logps/rejected": -1.5489881038665771, + "loss": 2.0047, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.28328537940979, + "rewards/margins": 0.26570266485214233, + "rewards/rejected": -1.5489881038665771, + "semantic_entropy": 0.7944211959838867, + "step": 3250 + }, + { + "epoch": 1.7420973406924234, + "grad_norm": 7.1162691309907995, + "learning_rate": 4.4637295988638555e-07, + "logits/chosen": -0.0334712490439415, + "logits/rejected": 0.05123140290379524, + "logps/chosen": -1.3655494451522827, + "logps/rejected": -1.4881891012191772, + "loss": 2.0721, + "rewards/accuracies": 0.5375000238418579, + "rewards/chosen": -1.3655494451522827, + "rewards/margins": 0.12263967096805573, + "rewards/rejected": -1.4881891012191772, + "semantic_entropy": 0.7760681509971619, + "step": 3255 + }, + { + "epoch": 1.744773373473825, + "grad_norm": 8.40556859712584, + "learning_rate": 4.4482480352391623e-07, + "logits/chosen": -0.1488790214061737, + "logits/rejected": -0.009104812517762184, + "logps/chosen": -1.3160731792449951, + "logps/rejected": -1.4561564922332764, + "loss": 2.0384, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.3160731792449951, + "rewards/margins": 0.1400834321975708, + "rewards/rejected": -1.4561564922332764, + "semantic_entropy": 0.7991342544555664, + "step": 3260 + }, + { + "epoch": 1.7474494062552266, + "grad_norm": 11.769905710707198, + "learning_rate": 4.4327718247144507e-07, + "logits/chosen": -0.0454300232231617, + "logits/rejected": 0.06899180263280869, + "logps/chosen": -1.2196002006530762, + "logps/rejected": -1.5009334087371826, + "loss": 1.9635, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2196002006530762, + "rewards/margins": 0.28133314847946167, + "rewards/rejected": -1.5009334087371826, + "semantic_entropy": 0.8047575950622559, + "step": 3265 + }, + { + "epoch": 1.750125439036628, + "grad_norm": 9.860790687033859, + "learning_rate": 4.417301117439984e-07, + "logits/chosen": -0.04798585921525955, + "logits/rejected": 0.09852831065654755, + "logps/chosen": -1.1797542572021484, + "logps/rejected": -1.489864706993103, + "loss": 1.9504, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.1797542572021484, + "rewards/margins": 0.31011033058166504, + "rewards/rejected": -1.489864706993103, + "semantic_entropy": 0.8194267153739929, + "step": 3270 + }, + { + "epoch": 1.7528014718180298, + "grad_norm": 11.278419301684329, + "learning_rate": 4.401836063512631e-07, + "logits/chosen": -0.09553112089633942, + "logits/rejected": 0.22362038493156433, + "logps/chosen": -1.2884795665740967, + "logps/rejected": -1.4595869779586792, + "loss": 2.0391, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.2884795665740967, + "rewards/margins": 0.17110735177993774, + "rewards/rejected": -1.4595869779586792, + "semantic_entropy": 0.8058779835700989, + "step": 3275 + }, + { + "epoch": 1.7554775045994313, + "grad_norm": 9.495267927421752, + "learning_rate": 4.386376812974413e-07, + "logits/chosen": -0.09709614515304565, + "logits/rejected": -0.005060785915702581, + "logps/chosen": -1.235475778579712, + "logps/rejected": -1.4756667613983154, + "loss": 1.998, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.235475778579712, + "rewards/margins": 0.24019083380699158, + "rewards/rejected": -1.4756667613983154, + "semantic_entropy": 0.8163919448852539, + "step": 3280 + }, + { + "epoch": 1.7581535373808328, + "grad_norm": 7.859136954214772, + "learning_rate": 4.370923515811048e-07, + "logits/chosen": -0.1316256821155548, + "logits/rejected": 0.08077608048915863, + "logps/chosen": -1.2407208681106567, + "logps/rejected": -1.5216323137283325, + "loss": 2.0041, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2407208681106567, + "rewards/margins": 0.28091129660606384, + "rewards/rejected": -1.5216323137283325, + "semantic_entropy": 0.8013314008712769, + "step": 3285 + }, + { + "epoch": 1.7608295701622345, + "grad_norm": 8.930092324257826, + "learning_rate": 4.35547632195049e-07, + "logits/chosen": -0.04723476618528366, + "logits/rejected": 0.06438259780406952, + "logps/chosen": -1.2634159326553345, + "logps/rejected": -1.4355380535125732, + "loss": 2.0441, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -1.2634159326553345, + "rewards/margins": 0.17212224006652832, + "rewards/rejected": -1.4355380535125732, + "semantic_entropy": 0.8089887499809265, + "step": 3290 + }, + { + "epoch": 1.763505602943636, + "grad_norm": 10.272633292110191, + "learning_rate": 4.340035381261484e-07, + "logits/chosen": -0.08970440179109573, + "logits/rejected": -0.01908710040152073, + "logps/chosen": -1.3528639078140259, + "logps/rejected": -1.507359266281128, + "loss": 2.1138, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.3528639078140259, + "rewards/margins": 0.15449512004852295, + "rewards/rejected": -1.507359266281128, + "semantic_entropy": 0.7817720174789429, + "step": 3295 + }, + { + "epoch": 1.7661816357250375, + "grad_norm": 8.125587964992828, + "learning_rate": 4.324600843552104e-07, + "logits/chosen": -0.18519294261932373, + "logits/rejected": -0.04560722038149834, + "logps/chosen": -1.34574294090271, + "logps/rejected": -1.5829145908355713, + "loss": 2.0564, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.34574294090271, + "rewards/margins": 0.2371715009212494, + "rewards/rejected": -1.5829145908355713, + "semantic_entropy": 0.7710360288619995, + "step": 3300 + }, + { + "epoch": 1.7688576685064392, + "grad_norm": 9.374436536799694, + "learning_rate": 4.309172858568302e-07, + "logits/chosen": -0.165949285030365, + "logits/rejected": -0.02660273388028145, + "logps/chosen": -1.3073817491531372, + "logps/rejected": -1.4832111597061157, + "loss": 2.0696, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.3073817491531372, + "rewards/margins": 0.1758294552564621, + "rewards/rejected": -1.4832111597061157, + "semantic_entropy": 0.7953299283981323, + "step": 3305 + }, + { + "epoch": 1.771533701287841, + "grad_norm": 8.087464752020232, + "learning_rate": 4.293751575992455e-07, + "logits/chosen": 0.013452662155032158, + "logits/rejected": 0.05870788171887398, + "logps/chosen": -1.2773231267929077, + "logps/rejected": -1.4914847612380981, + "loss": 2.0228, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2773231267929077, + "rewards/margins": 0.2141617238521576, + "rewards/rejected": -1.4914847612380981, + "semantic_entropy": 0.8066979646682739, + "step": 3310 + }, + { + "epoch": 1.7742097340692422, + "grad_norm": 10.725463234406757, + "learning_rate": 4.278337145441916e-07, + "logits/chosen": -0.1867462694644928, + "logits/rejected": -0.037748754024505615, + "logps/chosen": -1.255563497543335, + "logps/rejected": -1.5021106004714966, + "loss": 2.0126, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.255563497543335, + "rewards/margins": 0.2465471774339676, + "rewards/rejected": -1.5021106004714966, + "semantic_entropy": 0.8064797520637512, + "step": 3315 + }, + { + "epoch": 1.776885766850644, + "grad_norm": 7.58508784876293, + "learning_rate": 4.262929716467556e-07, + "logits/chosen": -0.11277034133672714, + "logits/rejected": 0.07743777334690094, + "logps/chosen": -1.2668403387069702, + "logps/rejected": -1.6261193752288818, + "loss": 1.9878, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2668403387069702, + "rewards/margins": 0.35927897691726685, + "rewards/rejected": -1.6261193752288818, + "semantic_entropy": 0.8014122247695923, + "step": 3320 + }, + { + "epoch": 1.7795617996320456, + "grad_norm": 17.453866294688677, + "learning_rate": 4.247529438552321e-07, + "logits/chosen": -0.19220510125160217, + "logits/rejected": -0.0137777179479599, + "logps/chosen": -1.2939108610153198, + "logps/rejected": -1.5628669261932373, + "loss": 2.0125, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2939108610153198, + "rewards/margins": 0.2689562141895294, + "rewards/rejected": -1.5628669261932373, + "semantic_entropy": 0.8005807995796204, + "step": 3325 + }, + { + "epoch": 1.782237832413447, + "grad_norm": 12.090886939609968, + "learning_rate": 4.232136461109773e-07, + "logits/chosen": -0.06670709699392319, + "logits/rejected": 0.040992237627506256, + "logps/chosen": -1.1921557188034058, + "logps/rejected": -1.5382206439971924, + "loss": 1.9347, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.1921557188034058, + "rewards/margins": 0.346064954996109, + "rewards/rejected": -1.5382206439971924, + "semantic_entropy": 0.8233155012130737, + "step": 3330 + }, + { + "epoch": 1.7849138651948486, + "grad_norm": 12.17353452249854, + "learning_rate": 4.216750933482646e-07, + "logits/chosen": -0.11370841413736343, + "logits/rejected": 0.046696338802576065, + "logps/chosen": -1.3182452917099, + "logps/rejected": -1.5295355319976807, + "loss": 2.029, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.3182452917099, + "rewards/margins": 0.2112903892993927, + "rewards/rejected": -1.5295355319976807, + "semantic_entropy": 0.7953765392303467, + "step": 3335 + }, + { + "epoch": 1.7875898979762503, + "grad_norm": 7.707961560103677, + "learning_rate": 4.2013730049413986e-07, + "logits/chosen": -0.0673111230134964, + "logits/rejected": 0.0833623856306076, + "logps/chosen": -1.2442004680633545, + "logps/rejected": -1.5510399341583252, + "loss": 1.9881, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.2442004680633545, + "rewards/margins": 0.3068394660949707, + "rewards/rejected": -1.5510399341583252, + "semantic_entropy": 0.8056608438491821, + "step": 3340 + }, + { + "epoch": 1.7902659307576518, + "grad_norm": 9.860843976130422, + "learning_rate": 4.1860028246827594e-07, + "logits/chosen": -0.10094896703958511, + "logits/rejected": 0.0693550854921341, + "logps/chosen": -1.1767338514328003, + "logps/rejected": -1.4444658756256104, + "loss": 1.9528, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.1767338514328003, + "rewards/margins": 0.26773205399513245, + "rewards/rejected": -1.4444658756256104, + "semantic_entropy": 0.8237002491950989, + "step": 3345 + }, + { + "epoch": 1.7929419635390533, + "grad_norm": 8.425056726555647, + "learning_rate": 4.170640541828285e-07, + "logits/chosen": -0.1958092898130417, + "logits/rejected": -0.04710087925195694, + "logps/chosen": -1.3505299091339111, + "logps/rejected": -1.549831748008728, + "loss": 2.068, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.3505299091339111, + "rewards/margins": 0.19930198788642883, + "rewards/rejected": -1.549831748008728, + "semantic_entropy": 0.782744824886322, + "step": 3350 + }, + { + "epoch": 1.795617996320455, + "grad_norm": 11.576807594386622, + "learning_rate": 4.1552863054229116e-07, + "logits/chosen": 0.04093014448881149, + "logits/rejected": 0.08852342516183853, + "logps/chosen": -1.3552569150924683, + "logps/rejected": -1.504966139793396, + "loss": 2.1187, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -1.3552569150924683, + "rewards/margins": 0.14970925450325012, + "rewards/rejected": -1.504966139793396, + "semantic_entropy": 0.7735754251480103, + "step": 3355 + }, + { + "epoch": 1.7982940291018565, + "grad_norm": 8.288746454534513, + "learning_rate": 4.139940264433508e-07, + "logits/chosen": -0.09469692409038544, + "logits/rejected": 0.12611272931098938, + "logps/chosen": -1.2306644916534424, + "logps/rejected": -1.4716196060180664, + "loss": 1.9888, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2306644916534424, + "rewards/margins": 0.24095502495765686, + "rewards/rejected": -1.4716196060180664, + "semantic_entropy": 0.8139133453369141, + "step": 3360 + }, + { + "epoch": 1.800970061883258, + "grad_norm": 7.1777727486770955, + "learning_rate": 4.1246025677474303e-07, + "logits/chosen": -0.1426037847995758, + "logits/rejected": 0.014146551489830017, + "logps/chosen": -1.2692500352859497, + "logps/rejected": -1.5440677404403687, + "loss": 1.9848, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2692500352859497, + "rewards/margins": 0.2748177647590637, + "rewards/rejected": -1.5440677404403687, + "semantic_entropy": 0.7960511445999146, + "step": 3365 + }, + { + "epoch": 1.8036460946646597, + "grad_norm": 7.206340297679535, + "learning_rate": 4.10927336417108e-07, + "logits/chosen": -0.11211264133453369, + "logits/rejected": 0.04343719407916069, + "logps/chosen": -1.2716386318206787, + "logps/rejected": -1.4695169925689697, + "loss": 2.0223, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.2716386318206787, + "rewards/margins": 0.19787819683551788, + "rewards/rejected": -1.4695169925689697, + "semantic_entropy": 0.8089770078659058, + "step": 3370 + }, + { + "epoch": 1.8063221274460612, + "grad_norm": 8.240207415115314, + "learning_rate": 4.093952802428457e-07, + "logits/chosen": 0.055115751922130585, + "logits/rejected": 0.10022290796041489, + "logps/chosen": -1.3147608041763306, + "logps/rejected": -1.4589980840682983, + "loss": 2.0785, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.3147608041763306, + "rewards/margins": 0.1442374736070633, + "rewards/rejected": -1.4589980840682983, + "semantic_entropy": 0.8038409948348999, + "step": 3375 + }, + { + "epoch": 1.8089981602274627, + "grad_norm": 6.219293393123405, + "learning_rate": 4.0786410311597184e-07, + "logits/chosen": -0.14277827739715576, + "logits/rejected": 0.0073592751286923885, + "logps/chosen": -1.2585190534591675, + "logps/rejected": -1.5473378896713257, + "loss": 1.9828, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.2585190534591675, + "rewards/margins": 0.28881901502609253, + "rewards/rejected": -1.5473378896713257, + "semantic_entropy": 0.8017240762710571, + "step": 3380 + }, + { + "epoch": 1.8116741930088645, + "grad_norm": 8.042866914467831, + "learning_rate": 4.063338198919737e-07, + "logits/chosen": -0.12222157418727875, + "logits/rejected": -0.0931001529097557, + "logps/chosen": -1.3203542232513428, + "logps/rejected": -1.5036855936050415, + "loss": 2.0601, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.3203542232513428, + "rewards/margins": 0.18333129584789276, + "rewards/rejected": -1.5036855936050415, + "semantic_entropy": 0.7957580089569092, + "step": 3385 + }, + { + "epoch": 1.814350225790266, + "grad_norm": 11.085977240555279, + "learning_rate": 4.0480444541766575e-07, + "logits/chosen": -0.09060386568307877, + "logits/rejected": 0.03485582396388054, + "logps/chosen": -1.3461965322494507, + "logps/rejected": -1.4985482692718506, + "loss": 2.0926, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.3461965322494507, + "rewards/margins": 0.1523514986038208, + "rewards/rejected": -1.4985482692718506, + "semantic_entropy": 0.788882851600647, + "step": 3390 + }, + { + "epoch": 1.8170262585716674, + "grad_norm": 10.154637330551065, + "learning_rate": 4.0327599453104606e-07, + "logits/chosen": -0.13672223687171936, + "logits/rejected": -0.03219519183039665, + "logps/chosen": -1.21634840965271, + "logps/rejected": -1.5037835836410522, + "loss": 1.9777, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.21634840965271, + "rewards/margins": 0.28743523359298706, + "rewards/rejected": -1.5037835836410522, + "semantic_entropy": 0.8166402578353882, + "step": 3395 + }, + { + "epoch": 1.8197022913530692, + "grad_norm": 9.548085302685983, + "learning_rate": 4.017484820611514e-07, + "logits/chosen": -0.09091535955667496, + "logits/rejected": 0.03177911415696144, + "logps/chosen": -1.287781000137329, + "logps/rejected": -1.5104598999023438, + "loss": 2.016, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.287781000137329, + "rewards/margins": 0.2226787507534027, + "rewards/rejected": -1.5104598999023438, + "semantic_entropy": 0.7931040525436401, + "step": 3400 + }, + { + "epoch": 1.8223783241344707, + "grad_norm": 12.039861832870322, + "learning_rate": 4.002219228279148e-07, + "logits/chosen": -0.09477577358484268, + "logits/rejected": 0.061077456921339035, + "logps/chosen": -1.275453805923462, + "logps/rejected": -1.4921751022338867, + "loss": 2.0209, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.275453805923462, + "rewards/margins": 0.2167212963104248, + "rewards/rejected": -1.4921751022338867, + "semantic_entropy": 0.808013916015625, + "step": 3405 + }, + { + "epoch": 1.8250543569158721, + "grad_norm": 10.043505087001625, + "learning_rate": 3.9869633164202045e-07, + "logits/chosen": -0.10172738879919052, + "logits/rejected": 0.12015529721975327, + "logps/chosen": -1.4054529666900635, + "logps/rejected": -1.5626072883605957, + "loss": 2.1194, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -1.4054529666900635, + "rewards/margins": 0.15715420246124268, + "rewards/rejected": -1.5626072883605957, + "semantic_entropy": 0.7611129879951477, + "step": 3410 + }, + { + "epoch": 1.8277303896972739, + "grad_norm": 9.554825491563795, + "learning_rate": 3.9717172330476077e-07, + "logits/chosen": -0.1088135689496994, + "logits/rejected": -0.0048567550256848335, + "logps/chosen": -1.2670540809631348, + "logps/rejected": -1.5354701280593872, + "loss": 2.0084, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.2670540809631348, + "rewards/margins": 0.2684159576892853, + "rewards/rejected": -1.5354701280593872, + "semantic_entropy": 0.7980339527130127, + "step": 3415 + }, + { + "epoch": 1.8304064224786754, + "grad_norm": 10.833270444734117, + "learning_rate": 3.956481126078927e-07, + "logits/chosen": -0.07312284409999847, + "logits/rejected": 0.03952528536319733, + "logps/chosen": -1.3103262186050415, + "logps/rejected": -1.6255826950073242, + "loss": 2.0335, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.3103262186050415, + "rewards/margins": 0.3152565062046051, + "rewards/rejected": -1.6255826950073242, + "semantic_entropy": 0.779475212097168, + "step": 3420 + }, + { + "epoch": 1.8330824552600768, + "grad_norm": 6.387688251221902, + "learning_rate": 3.941255143334937e-07, + "logits/chosen": -0.143729105591774, + "logits/rejected": -0.09878051280975342, + "logps/chosen": -1.2660369873046875, + "logps/rejected": -1.5195882320404053, + "loss": 2.0111, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.2660369873046875, + "rewards/margins": 0.2535511553287506, + "rewards/rejected": -1.5195882320404053, + "semantic_entropy": 0.8040241003036499, + "step": 3425 + }, + { + "epoch": 1.8357584880414786, + "grad_norm": 12.017230340151167, + "learning_rate": 3.9260394325381895e-07, + "logits/chosen": -0.1061253771185875, + "logits/rejected": 0.02162790857255459, + "logps/chosen": -1.3311865329742432, + "logps/rejected": -1.6028121709823608, + "loss": 2.0403, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.3311865329742432, + "rewards/margins": 0.2716255187988281, + "rewards/rejected": -1.6028121709823608, + "semantic_entropy": 0.7831372022628784, + "step": 3430 + }, + { + "epoch": 1.83843452082288, + "grad_norm": 11.260166805719937, + "learning_rate": 3.9108341413115784e-07, + "logits/chosen": -0.12128078937530518, + "logits/rejected": -0.029097210615873337, + "logps/chosen": -1.274632453918457, + "logps/rejected": -1.5507726669311523, + "loss": 1.9776, + "rewards/accuracies": 0.65625, + "rewards/chosen": -1.274632453918457, + "rewards/margins": 0.2761402130126953, + "rewards/rejected": -1.5507726669311523, + "semantic_entropy": 0.790766716003418, + "step": 3435 + }, + { + "epoch": 1.8411105536042816, + "grad_norm": 11.838134033346803, + "learning_rate": 3.895639417176905e-07, + "logits/chosen": -0.1734875738620758, + "logits/rejected": -0.10030355304479599, + "logps/chosen": -1.2006644010543823, + "logps/rejected": -1.5158345699310303, + "loss": 1.9848, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2006644010543823, + "rewards/margins": 0.31517019867897034, + "rewards/rejected": -1.5158345699310303, + "semantic_entropy": 0.8104988932609558, + "step": 3440 + }, + { + "epoch": 1.8437865863856833, + "grad_norm": 9.095694299153367, + "learning_rate": 3.8804554075534497e-07, + "logits/chosen": -0.18821772933006287, + "logits/rejected": 0.03512702137231827, + "logps/chosen": -1.2557734251022339, + "logps/rejected": -1.4832450151443481, + "loss": 1.9862, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.2557734251022339, + "rewards/margins": 0.2274715006351471, + "rewards/rejected": -1.4832450151443481, + "semantic_entropy": 0.8089550137519836, + "step": 3445 + }, + { + "epoch": 1.8464626191670848, + "grad_norm": 13.087986143054922, + "learning_rate": 3.8652822597565403e-07, + "logits/chosen": -0.2668977975845337, + "logits/rejected": -0.08437497913837433, + "logps/chosen": -1.2757649421691895, + "logps/rejected": -1.5986865758895874, + "loss": 1.9793, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.2757649421691895, + "rewards/margins": 0.3229215145111084, + "rewards/rejected": -1.5986865758895874, + "semantic_entropy": 0.7837361097335815, + "step": 3450 + }, + { + "epoch": 1.8491386519484863, + "grad_norm": 7.107161760470273, + "learning_rate": 3.850120120996123e-07, + "logits/chosen": -0.1137063130736351, + "logits/rejected": 0.04673473909497261, + "logps/chosen": -1.4318660497665405, + "logps/rejected": -1.6971549987792969, + "loss": 2.1086, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.4318660497665405, + "rewards/margins": 0.2652890682220459, + "rewards/rejected": -1.6971549987792969, + "semantic_entropy": 0.7388371229171753, + "step": 3455 + }, + { + "epoch": 1.851814684729888, + "grad_norm": 11.203647624565647, + "learning_rate": 3.8349691383753356e-07, + "logits/chosen": -0.014518792741000652, + "logits/rejected": 0.11191823333501816, + "logps/chosen": -1.2621428966522217, + "logps/rejected": -1.5209171772003174, + "loss": 2.0493, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.2621428966522217, + "rewards/margins": 0.258774071931839, + "rewards/rejected": -1.5209171772003174, + "semantic_entropy": 0.799616813659668, + "step": 3460 + }, + { + "epoch": 1.8544907175112895, + "grad_norm": 6.988737589871721, + "learning_rate": 3.819829458889078e-07, + "logits/chosen": -0.16742148995399475, + "logits/rejected": -0.038034576922655106, + "logps/chosen": -1.2240421772003174, + "logps/rejected": -1.4272968769073486, + "loss": 1.9947, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2240421772003174, + "rewards/margins": 0.2032547891139984, + "rewards/rejected": -1.4272968769073486, + "semantic_entropy": 0.8272393345832825, + "step": 3465 + }, + { + "epoch": 1.857166750292691, + "grad_norm": 7.58424123902832, + "learning_rate": 3.804701229422585e-07, + "logits/chosen": -0.16845372319221497, + "logits/rejected": -0.07218648493289948, + "logps/chosen": -1.351628065109253, + "logps/rejected": -1.55996572971344, + "loss": 2.0535, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.351628065109253, + "rewards/margins": 0.20833781361579895, + "rewards/rejected": -1.55996572971344, + "semantic_entropy": 0.7803600430488586, + "step": 3470 + }, + { + "epoch": 1.8598427830740927, + "grad_norm": 9.02069349372374, + "learning_rate": 3.789584596750007e-07, + "logits/chosen": -0.1916225254535675, + "logits/rejected": -0.12627634406089783, + "logps/chosen": -1.2798500061035156, + "logps/rejected": -1.5317795276641846, + "loss": 2.0162, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2798500061035156, + "rewards/margins": 0.2519295811653137, + "rewards/rejected": -1.5317795276641846, + "semantic_entropy": 0.7949556112289429, + "step": 3475 + }, + { + "epoch": 1.8625188158554944, + "grad_norm": 8.745520346577289, + "learning_rate": 3.77447970753298e-07, + "logits/chosen": -0.054103873670101166, + "logits/rejected": -0.01997731253504753, + "logps/chosen": -1.3081482648849487, + "logps/rejected": -1.5802189111709595, + "loss": 2.0131, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.3081482648849487, + "rewards/margins": 0.2720705270767212, + "rewards/rejected": -1.5802189111709595, + "semantic_entropy": 0.7743436098098755, + "step": 3480 + }, + { + "epoch": 1.8651948486368957, + "grad_norm": 10.422824586512155, + "learning_rate": 3.7593867083192057e-07, + "logits/chosen": -0.11146446317434311, + "logits/rejected": -0.003896909300237894, + "logps/chosen": -1.266123652458191, + "logps/rejected": -1.5006070137023926, + "loss": 2.0187, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.266123652458191, + "rewards/margins": 0.2344832867383957, + "rewards/rejected": -1.5006070137023926, + "semantic_entropy": 0.8031982183456421, + "step": 3485 + }, + { + "epoch": 1.8678708814182974, + "grad_norm": 7.17462834869493, + "learning_rate": 3.7443057455410276e-07, + "logits/chosen": -0.09207798540592194, + "logits/rejected": 0.03373739868402481, + "logps/chosen": -1.3165172338485718, + "logps/rejected": -1.4651727676391602, + "loss": 2.0569, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.3165172338485718, + "rewards/margins": 0.14865554869174957, + "rewards/rejected": -1.4651727676391602, + "semantic_entropy": 0.7978287935256958, + "step": 3490 + }, + { + "epoch": 1.870546914199699, + "grad_norm": 8.608209174018716, + "learning_rate": 3.7292369655140145e-07, + "logits/chosen": -0.19836124777793884, + "logits/rejected": -0.036930233240127563, + "logps/chosen": -1.2759727239608765, + "logps/rejected": -1.4997532367706299, + "loss": 2.0038, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2759727239608765, + "rewards/margins": 0.223780557513237, + "rewards/rejected": -1.4997532367706299, + "semantic_entropy": 0.808540940284729, + "step": 3495 + }, + { + "epoch": 1.8732229469811004, + "grad_norm": 8.037145528269233, + "learning_rate": 3.714180514435534e-07, + "logits/chosen": -0.1019342690706253, + "logits/rejected": 0.04769862815737724, + "logps/chosen": -1.3215068578720093, + "logps/rejected": -1.6422048807144165, + "loss": 2.006, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.3215068578720093, + "rewards/margins": 0.3206980526447296, + "rewards/rejected": -1.6422048807144165, + "semantic_entropy": 0.7793091535568237, + "step": 3500 + }, + { + "epoch": 1.875898979762502, + "grad_norm": 10.400013771141799, + "learning_rate": 3.6991365383833426e-07, + "logits/chosen": -0.10635554790496826, + "logits/rejected": 0.01562192477285862, + "logps/chosen": -1.321709156036377, + "logps/rejected": -1.6019134521484375, + "loss": 2.0371, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.321709156036377, + "rewards/margins": 0.2802041471004486, + "rewards/rejected": -1.6019134521484375, + "semantic_entropy": 0.7732099294662476, + "step": 3505 + }, + { + "epoch": 1.8785750125439038, + "grad_norm": 10.206269336294888, + "learning_rate": 3.684105183314162e-07, + "logits/chosen": -0.13000985980033875, + "logits/rejected": -0.04903121665120125, + "logps/chosen": -1.265295147895813, + "logps/rejected": -1.4725327491760254, + "loss": 1.9937, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.265295147895813, + "rewards/margins": 0.20723757147789001, + "rewards/rejected": -1.4725327491760254, + "semantic_entropy": 0.8022018671035767, + "step": 3510 + }, + { + "epoch": 1.881251045325305, + "grad_norm": 11.736347149498757, + "learning_rate": 3.669086595062263e-07, + "logits/chosen": -0.13198086619377136, + "logits/rejected": 0.06340041011571884, + "logps/chosen": -1.3111042976379395, + "logps/rejected": -1.5591113567352295, + "loss": 2.0347, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.3111042976379395, + "rewards/margins": 0.24800701439380646, + "rewards/rejected": -1.5591113567352295, + "semantic_entropy": 0.8016371726989746, + "step": 3515 + }, + { + "epoch": 1.8839270781067068, + "grad_norm": 10.291606681517248, + "learning_rate": 3.654080919338056e-07, + "logits/chosen": -0.17716601490974426, + "logits/rejected": -0.03891471400856972, + "logps/chosen": -1.317662000656128, + "logps/rejected": -1.593109130859375, + "loss": 2.0252, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.317662000656128, + "rewards/margins": 0.27544716000556946, + "rewards/rejected": -1.593109130859375, + "semantic_entropy": 0.7637777328491211, + "step": 3520 + }, + { + "epoch": 1.8866031108881085, + "grad_norm": 7.861563362786371, + "learning_rate": 3.639088301726673e-07, + "logits/chosen": -0.0826728343963623, + "logits/rejected": 0.11588224023580551, + "logps/chosen": -1.2835619449615479, + "logps/rejected": -1.5396344661712646, + "loss": 2.0151, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.2835619449615479, + "rewards/margins": 0.2560725212097168, + "rewards/rejected": -1.5396344661712646, + "semantic_entropy": 0.795852541923523, + "step": 3525 + }, + { + "epoch": 1.88927914366951, + "grad_norm": 13.126218986628478, + "learning_rate": 3.624108887686556e-07, + "logits/chosen": -0.09712855517864227, + "logits/rejected": -0.02328089438378811, + "logps/chosen": -1.260948896408081, + "logps/rejected": -1.507131814956665, + "loss": 2.0035, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.260948896408081, + "rewards/margins": 0.24618300795555115, + "rewards/rejected": -1.507131814956665, + "semantic_entropy": 0.8005443811416626, + "step": 3530 + }, + { + "epoch": 1.8919551764509115, + "grad_norm": 6.546292643866425, + "learning_rate": 3.6091428225480433e-07, + "logits/chosen": -0.18229694664478302, + "logits/rejected": -0.04712440446019173, + "logps/chosen": -1.227616548538208, + "logps/rejected": -1.4500181674957275, + "loss": 2.0085, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.227616548538208, + "rewards/margins": 0.22240164875984192, + "rewards/rejected": -1.4500181674957275, + "semantic_entropy": 0.8142200708389282, + "step": 3535 + }, + { + "epoch": 1.8946312092323132, + "grad_norm": 14.05217500248357, + "learning_rate": 3.5941902515119674e-07, + "logits/chosen": -0.13271181285381317, + "logits/rejected": 0.09598705172538757, + "logps/chosen": -1.2617511749267578, + "logps/rejected": -1.4930734634399414, + "loss": 2.0113, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.2617511749267578, + "rewards/margins": 0.23132216930389404, + "rewards/rejected": -1.4930734634399414, + "semantic_entropy": 0.8063961863517761, + "step": 3540 + }, + { + "epoch": 1.8973072420137147, + "grad_norm": 9.515261930521739, + "learning_rate": 3.5792513196482373e-07, + "logits/chosen": -0.2581722140312195, + "logits/rejected": 0.015956703573465347, + "logps/chosen": -1.2583338022232056, + "logps/rejected": -1.442116141319275, + "loss": 2.0185, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.2583338022232056, + "rewards/margins": 0.18378230929374695, + "rewards/rejected": -1.442116141319275, + "semantic_entropy": 0.8180877566337585, + "step": 3545 + }, + { + "epoch": 1.8999832747951162, + "grad_norm": 7.148565276722444, + "learning_rate": 3.5643261718944346e-07, + "logits/chosen": -0.08364517986774445, + "logits/rejected": 0.008080209605395794, + "logps/chosen": -1.2652015686035156, + "logps/rejected": -1.4693939685821533, + "loss": 2.0108, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.2652015686035156, + "rewards/margins": 0.20419220626354218, + "rewards/rejected": -1.4693939685821533, + "semantic_entropy": 0.823981761932373, + "step": 3550 + }, + { + "epoch": 1.902659307576518, + "grad_norm": 6.059750779835962, + "learning_rate": 3.5494149530544087e-07, + "logits/chosen": -0.2066015899181366, + "logits/rejected": -0.067063108086586, + "logps/chosen": -1.2199000120162964, + "logps/rejected": -1.479786992073059, + "loss": 2.0179, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2199000120162964, + "rewards/margins": 0.2598869204521179, + "rewards/rejected": -1.479786992073059, + "semantic_entropy": 0.8159680366516113, + "step": 3555 + }, + { + "epoch": 1.9053353403579194, + "grad_norm": 12.349981712337039, + "learning_rate": 3.534517807796871e-07, + "logits/chosen": -0.10540161281824112, + "logits/rejected": -0.01878190226852894, + "logps/chosen": -1.3090038299560547, + "logps/rejected": -1.5431599617004395, + "loss": 2.0363, + "rewards/accuracies": 0.53125, + "rewards/chosen": -1.3090038299560547, + "rewards/margins": 0.2341562807559967, + "rewards/rejected": -1.5431599617004395, + "semantic_entropy": 0.7972686290740967, + "step": 3560 + }, + { + "epoch": 1.908011373139321, + "grad_norm": 6.601573179170184, + "learning_rate": 3.519634880653988e-07, + "logits/chosen": -0.12914696335792542, + "logits/rejected": -0.0617067813873291, + "logps/chosen": -1.2342662811279297, + "logps/rejected": -1.5450482368469238, + "loss": 1.9972, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.2342662811279297, + "rewards/margins": 0.3107820749282837, + "rewards/rejected": -1.5450482368469238, + "semantic_entropy": 0.8117408752441406, + "step": 3565 + }, + { + "epoch": 1.9106874059207226, + "grad_norm": 7.578795754764497, + "learning_rate": 3.504766316019987e-07, + "logits/chosen": -0.15820898115634918, + "logits/rejected": -0.017529401928186417, + "logps/chosen": -1.262640118598938, + "logps/rejected": -1.5338207483291626, + "loss": 2.0033, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.262640118598938, + "rewards/margins": 0.27118054032325745, + "rewards/rejected": -1.5338207483291626, + "semantic_entropy": 0.8046427965164185, + "step": 3570 + }, + { + "epoch": 1.913363438702124, + "grad_norm": 8.152077198375881, + "learning_rate": 3.489912258149745e-07, + "logits/chosen": -0.05192985013127327, + "logits/rejected": 0.0704207569360733, + "logps/chosen": -1.2362616062164307, + "logps/rejected": -1.5028289556503296, + "loss": 2.0064, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.2362616062164307, + "rewards/margins": 0.2665674090385437, + "rewards/rejected": -1.5028289556503296, + "semantic_entropy": 0.8162727355957031, + "step": 3575 + }, + { + "epoch": 1.9160394714835256, + "grad_norm": 10.272991612886464, + "learning_rate": 3.475072851157397e-07, + "logits/chosen": -0.11488697677850723, + "logits/rejected": -0.06389477103948593, + "logps/chosen": -1.2549583911895752, + "logps/rejected": -1.5735355615615845, + "loss": 1.9829, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2549583911895752, + "rewards/margins": 0.3185771107673645, + "rewards/rejected": -1.5735355615615845, + "semantic_entropy": 0.7964209914207458, + "step": 3580 + }, + { + "epoch": 1.9187155042649273, + "grad_norm": 8.11158263217016, + "learning_rate": 3.460248239014936e-07, + "logits/chosen": -0.015757273882627487, + "logits/rejected": 0.03946710377931595, + "logps/chosen": -1.3438389301300049, + "logps/rejected": -1.534001350402832, + "loss": 2.0518, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.3438389301300049, + "rewards/margins": 0.19016249477863312, + "rewards/rejected": -1.534001350402832, + "semantic_entropy": 0.7529127597808838, + "step": 3585 + }, + { + "epoch": 1.9213915370463288, + "grad_norm": 8.210026560238209, + "learning_rate": 3.4454385655508134e-07, + "logits/chosen": -0.04379934445023537, + "logits/rejected": 0.029793402180075645, + "logps/chosen": -1.3078745603561401, + "logps/rejected": -1.49537992477417, + "loss": 2.0368, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.3078745603561401, + "rewards/margins": 0.18750540912151337, + "rewards/rejected": -1.49537992477417, + "semantic_entropy": 0.7906752824783325, + "step": 3590 + }, + { + "epoch": 1.9240675698277303, + "grad_norm": 6.953433919929532, + "learning_rate": 3.4306439744485447e-07, + "logits/chosen": -0.21919319033622742, + "logits/rejected": -0.014783772639930248, + "logps/chosen": -1.2707698345184326, + "logps/rejected": -1.529934287071228, + "loss": 2.0018, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.2707698345184326, + "rewards/margins": 0.25916433334350586, + "rewards/rejected": -1.529934287071228, + "semantic_entropy": 0.7992693185806274, + "step": 3595 + }, + { + "epoch": 1.926743602609132, + "grad_norm": 10.877825850627211, + "learning_rate": 3.415864609245322e-07, + "logits/chosen": -0.059394340962171555, + "logits/rejected": 0.12252438068389893, + "logps/chosen": -1.2442500591278076, + "logps/rejected": -1.5825135707855225, + "loss": 1.971, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2442500591278076, + "rewards/margins": 0.33826351165771484, + "rewards/rejected": -1.5825135707855225, + "semantic_entropy": 0.7945801019668579, + "step": 3600 + }, + { + "epoch": 1.926743602609132, + "eval_logits/chosen": 0.21609361469745636, + "eval_logits/rejected": 0.3034639358520508, + "eval_logps/chosen": -1.3219430446624756, + "eval_logps/rejected": -1.569968342781067, + "eval_loss": 2.04423189163208, + "eval_rewards/accuracies": 0.5860534310340881, + "eval_rewards/chosen": -1.3219430446624756, + "eval_rewards/margins": 0.24802519381046295, + "eval_rewards/rejected": -1.569968342781067, + "eval_runtime": 34.5858, + "eval_samples_per_second": 38.889, + "eval_semantic_entropy": 0.7832201719284058, + "eval_steps_per_second": 9.744, + "step": 3600 + }, + { + "epoch": 1.9294196353905335, + "grad_norm": 8.144866054072718, + "learning_rate": 3.401100613330605e-07, + "logits/chosen": -0.158551424741745, + "logits/rejected": -0.11706791073083878, + "logps/chosen": -1.2888761758804321, + "logps/rejected": -1.4633435010910034, + "loss": 2.0513, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.2888761758804321, + "rewards/margins": 0.17446734011173248, + "rewards/rejected": -1.4633435010910034, + "semantic_entropy": 0.8004593849182129, + "step": 3605 + }, + { + "epoch": 1.932095668171935, + "grad_norm": 7.073985429506359, + "learning_rate": 3.3863521299447514e-07, + "logits/chosen": -0.08732198923826218, + "logits/rejected": 0.03606638312339783, + "logps/chosen": -1.2564102411270142, + "logps/rejected": -1.526855707168579, + "loss": 1.9798, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.2564102411270142, + "rewards/margins": 0.2704453766345978, + "rewards/rejected": -1.526855707168579, + "semantic_entropy": 0.7902366518974304, + "step": 3610 + }, + { + "epoch": 1.9347717009533367, + "grad_norm": 6.3886970871943065, + "learning_rate": 3.371619302177609e-07, + "logits/chosen": -0.023822737857699394, + "logits/rejected": 0.09075307846069336, + "logps/chosen": -1.3473241329193115, + "logps/rejected": -1.5236204862594604, + "loss": 2.0573, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.3473241329193115, + "rewards/margins": 0.1762964427471161, + "rewards/rejected": -1.5236204862594604, + "semantic_entropy": 0.7837681770324707, + "step": 3615 + }, + { + "epoch": 1.9374477337347382, + "grad_norm": 10.305278907533184, + "learning_rate": 3.3569022729671393e-07, + "logits/chosen": -0.07354442030191422, + "logits/rejected": -0.0033059536945074797, + "logps/chosen": -1.299324631690979, + "logps/rejected": -1.4606642723083496, + "loss": 2.065, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.299324631690979, + "rewards/margins": 0.1613396406173706, + "rewards/rejected": -1.4606642723083496, + "semantic_entropy": 0.7894943356513977, + "step": 3620 + }, + { + "epoch": 1.9401237665161397, + "grad_norm": 8.565777965624173, + "learning_rate": 3.342201185098024e-07, + "logits/chosen": -0.0113115468993783, + "logits/rejected": 0.005684341304004192, + "logps/chosen": -1.2715879678726196, + "logps/rejected": -1.5271837711334229, + "loss": 2.0008, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2715879678726196, + "rewards/margins": 0.25559574365615845, + "rewards/rejected": -1.5271837711334229, + "semantic_entropy": 0.7982123494148254, + "step": 3625 + }, + { + "epoch": 1.9427997992975414, + "grad_norm": 8.985052902489157, + "learning_rate": 3.3275161812002807e-07, + "logits/chosen": -0.11439623683691025, + "logits/rejected": -0.057005755603313446, + "logps/chosen": -1.2966344356536865, + "logps/rejected": -1.54485285282135, + "loss": 2.0394, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.2966344356536865, + "rewards/margins": 0.24821829795837402, + "rewards/rejected": -1.54485285282135, + "semantic_entropy": 0.7945786714553833, + "step": 3630 + }, + { + "epoch": 1.945475832078943, + "grad_norm": 10.35010463149898, + "learning_rate": 3.312847403747883e-07, + "logits/chosen": -0.15070085227489471, + "logits/rejected": -0.06268687546253204, + "logps/chosen": -1.310336709022522, + "logps/rejected": -1.533013105392456, + "loss": 2.0457, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.310336709022522, + "rewards/margins": 0.22267630696296692, + "rewards/rejected": -1.533013105392456, + "semantic_entropy": 0.7924118041992188, + "step": 3635 + }, + { + "epoch": 1.9481518648603444, + "grad_norm": 8.069338833575255, + "learning_rate": 3.2981949950573733e-07, + "logits/chosen": -0.07498464733362198, + "logits/rejected": 0.039922118186950684, + "logps/chosen": -1.3872663974761963, + "logps/rejected": -1.5021774768829346, + "loss": 2.0958, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.3872663974761963, + "rewards/margins": 0.1149112731218338, + "rewards/rejected": -1.5021774768829346, + "semantic_entropy": 0.76435387134552, + "step": 3640 + }, + { + "epoch": 1.9508278976417461, + "grad_norm": 7.908362354066705, + "learning_rate": 3.283559097286486e-07, + "logits/chosen": -0.11206521838903427, + "logits/rejected": 0.020204050466418266, + "logps/chosen": -1.3849326372146606, + "logps/rejected": -1.5381712913513184, + "loss": 2.0937, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.3849326372146606, + "rewards/margins": 0.15323874354362488, + "rewards/rejected": -1.5381712913513184, + "semantic_entropy": 0.7717481255531311, + "step": 3645 + }, + { + "epoch": 1.9535039304231478, + "grad_norm": 9.712785475979707, + "learning_rate": 3.268939852432765e-07, + "logits/chosen": -0.13621452450752258, + "logits/rejected": -0.029971089214086533, + "logps/chosen": -1.2723300457000732, + "logps/rejected": -1.471142053604126, + "loss": 2.017, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2723300457000732, + "rewards/margins": 0.1988120973110199, + "rewards/rejected": -1.471142053604126, + "semantic_entropy": 0.8029723167419434, + "step": 3650 + }, + { + "epoch": 1.9561799632045491, + "grad_norm": 12.481648173594774, + "learning_rate": 3.254337402332187e-07, + "logits/chosen": -0.11881868541240692, + "logits/rejected": 0.017414908856153488, + "logps/chosen": -1.3414294719696045, + "logps/rejected": -1.5426610708236694, + "loss": 2.08, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.3414294719696045, + "rewards/margins": 0.20123150944709778, + "rewards/rejected": -1.5426610708236694, + "semantic_entropy": 0.7877740859985352, + "step": 3655 + }, + { + "epoch": 1.9588559959859508, + "grad_norm": 11.346605430241318, + "learning_rate": 3.239751888657788e-07, + "logits/chosen": -0.14560244977474213, + "logits/rejected": -0.0107874795794487, + "logps/chosen": -1.2256336212158203, + "logps/rejected": -1.4671902656555176, + "loss": 2.0069, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2256336212158203, + "rewards/margins": 0.24155676364898682, + "rewards/rejected": -1.4671902656555176, + "semantic_entropy": 0.8126087188720703, + "step": 3660 + }, + { + "epoch": 1.9615320287673526, + "grad_norm": 10.395984398742891, + "learning_rate": 3.2251834529182856e-07, + "logits/chosen": -0.08570713549852371, + "logits/rejected": 0.031803928315639496, + "logps/chosen": -1.2469929456710815, + "logps/rejected": -1.4951390027999878, + "loss": 2.017, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2469929456710815, + "rewards/margins": 0.2481461763381958, + "rewards/rejected": -1.4951390027999878, + "semantic_entropy": 0.8090551495552063, + "step": 3665 + }, + { + "epoch": 1.9642080615487538, + "grad_norm": 6.370737640712571, + "learning_rate": 3.2106322364567075e-07, + "logits/chosen": -0.1544579565525055, + "logits/rejected": -0.005169686861336231, + "logps/chosen": -1.2779276371002197, + "logps/rejected": -1.5672134160995483, + "loss": 1.9949, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.2779276371002197, + "rewards/margins": 0.2892858386039734, + "rewards/rejected": -1.5672134160995483, + "semantic_entropy": 0.7989880442619324, + "step": 3670 + }, + { + "epoch": 1.9668840943301555, + "grad_norm": 7.798494451995573, + "learning_rate": 3.1960983804490183e-07, + "logits/chosen": -0.11066894233226776, + "logits/rejected": 0.02928655780851841, + "logps/chosen": -1.312739372253418, + "logps/rejected": -1.6923391819000244, + "loss": 1.9945, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -1.312739372253418, + "rewards/margins": 0.3795998692512512, + "rewards/rejected": -1.6923391819000244, + "semantic_entropy": 0.7777242660522461, + "step": 3675 + }, + { + "epoch": 1.9695601271115573, + "grad_norm": 8.440153896776039, + "learning_rate": 3.1815820259027537e-07, + "logits/chosen": -0.11401332914829254, + "logits/rejected": 0.00045472680358216166, + "logps/chosen": -1.1583054065704346, + "logps/rejected": -1.4422087669372559, + "loss": 1.9335, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -1.1583054065704346, + "rewards/margins": 0.28390341997146606, + "rewards/rejected": -1.4422087669372559, + "semantic_entropy": 0.8331707715988159, + "step": 3680 + }, + { + "epoch": 1.9722361598929585, + "grad_norm": 10.959153779640832, + "learning_rate": 3.16708331365565e-07, + "logits/chosen": -0.11247573047876358, + "logits/rejected": -0.026667693629860878, + "logps/chosen": -1.268367052078247, + "logps/rejected": -1.5691543817520142, + "loss": 1.9707, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.268367052078247, + "rewards/margins": 0.3007873296737671, + "rewards/rejected": -1.5691543817520142, + "semantic_entropy": 0.7896307706832886, + "step": 3685 + }, + { + "epoch": 1.9749121926743602, + "grad_norm": 7.408577878714491, + "learning_rate": 3.152602384374275e-07, + "logits/chosen": -0.09716648608446121, + "logits/rejected": 0.06562429666519165, + "logps/chosen": -1.3082753419876099, + "logps/rejected": -1.5784547328948975, + "loss": 2.0316, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.3082753419876099, + "rewards/margins": 0.2701794505119324, + "rewards/rejected": -1.5784547328948975, + "semantic_entropy": 0.7799612283706665, + "step": 3690 + }, + { + "epoch": 1.977588225455762, + "grad_norm": 8.179466642910503, + "learning_rate": 3.1381393785526697e-07, + "logits/chosen": -0.05400281026959419, + "logits/rejected": -0.006647360511124134, + "logps/chosen": -1.3509399890899658, + "logps/rejected": -1.6333929300308228, + "loss": 2.0255, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.3509399890899658, + "rewards/margins": 0.28245288133621216, + "rewards/rejected": -1.6333929300308228, + "semantic_entropy": 0.7614681124687195, + "step": 3695 + }, + { + "epoch": 1.9802642582371635, + "grad_norm": 12.428902913590797, + "learning_rate": 3.123694436510979e-07, + "logits/chosen": -0.029851287603378296, + "logits/rejected": 0.08511605858802795, + "logps/chosen": -1.2690141201019287, + "logps/rejected": -1.543554425239563, + "loss": 2.0004, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2690141201019287, + "rewards/margins": 0.2745402455329895, + "rewards/rejected": -1.543554425239563, + "semantic_entropy": 0.7996218800544739, + "step": 3700 + }, + { + "epoch": 1.982940291018565, + "grad_norm": 9.828271639098743, + "learning_rate": 3.1092676983940946e-07, + "logits/chosen": -0.11319668591022491, + "logits/rejected": -0.033414699137210846, + "logps/chosen": -1.3103493452072144, + "logps/rejected": -1.594153642654419, + "loss": 2.0304, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.3103493452072144, + "rewards/margins": 0.2838039994239807, + "rewards/rejected": -1.594153642654419, + "semantic_entropy": 0.7914609909057617, + "step": 3705 + }, + { + "epoch": 1.9856163237999667, + "grad_norm": 9.075896180830405, + "learning_rate": 3.094859304170293e-07, + "logits/chosen": 0.06817227602005005, + "logits/rejected": 0.12981468439102173, + "logps/chosen": -1.3226096630096436, + "logps/rejected": -1.5663455724716187, + "loss": 2.0396, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -1.3226096630096436, + "rewards/margins": 0.24373598396778107, + "rewards/rejected": -1.5663455724716187, + "semantic_entropy": 0.781419575214386, + "step": 3710 + }, + { + "epoch": 1.9882923565813682, + "grad_norm": 5.926526406892064, + "learning_rate": 3.0804693936298795e-07, + "logits/chosen": -0.021849263459444046, + "logits/rejected": 0.05796981602907181, + "logps/chosen": -1.3069286346435547, + "logps/rejected": -1.5945860147476196, + "loss": 2.0158, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.3069286346435547, + "rewards/margins": 0.2876574993133545, + "rewards/rejected": -1.5945860147476196, + "semantic_entropy": 0.7784181237220764, + "step": 3715 + }, + { + "epoch": 1.9909683893627697, + "grad_norm": 7.061494171242169, + "learning_rate": 3.066098106383826e-07, + "logits/chosen": -0.08047592639923096, + "logits/rejected": 0.008429741486907005, + "logps/chosen": -1.2842880487442017, + "logps/rejected": -1.4925073385238647, + "loss": 2.0469, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.2842880487442017, + "rewards/margins": 0.20821920037269592, + "rewards/rejected": -1.4925073385238647, + "semantic_entropy": 0.7969603538513184, + "step": 3720 + }, + { + "epoch": 1.9936444221441714, + "grad_norm": 7.610782704184788, + "learning_rate": 3.0517455818624263e-07, + "logits/chosen": -0.13162103295326233, + "logits/rejected": -0.02476480044424534, + "logps/chosen": -1.282560110092163, + "logps/rejected": -1.5355018377304077, + "loss": 2.0228, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.282560110092163, + "rewards/margins": 0.25294169783592224, + "rewards/rejected": -1.5355018377304077, + "semantic_entropy": 0.8069452047348022, + "step": 3725 + }, + { + "epoch": 1.9963204549255729, + "grad_norm": 7.758490750103918, + "learning_rate": 3.037411959313936e-07, + "logits/chosen": -0.027306068688631058, + "logits/rejected": 0.10662344843149185, + "logps/chosen": -1.2372483015060425, + "logps/rejected": -1.51559579372406, + "loss": 1.9763, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2372483015060425, + "rewards/margins": 0.2783472537994385, + "rewards/rejected": -1.51559579372406, + "semantic_entropy": 0.8002141714096069, + "step": 3730 + }, + { + "epoch": 1.9989964877069744, + "grad_norm": 9.839994365343232, + "learning_rate": 3.023097377803224e-07, + "logits/chosen": 0.0056734951213002205, + "logits/rejected": 0.08714814484119415, + "logps/chosen": -1.3742187023162842, + "logps/rejected": -1.5301988124847412, + "loss": 2.1023, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.3742187023162842, + "rewards/margins": 0.155980184674263, + "rewards/rejected": -1.5301988124847412, + "semantic_entropy": 0.7796865105628967, + "step": 3735 + }, + { + "epoch": 2.001672520488376, + "grad_norm": 6.9179271221848095, + "learning_rate": 3.008801976210423e-07, + "logits/chosen": 0.021409938111901283, + "logits/rejected": 0.07372380793094635, + "logps/chosen": -1.3586982488632202, + "logps/rejected": -1.5227470397949219, + "loss": 2.0708, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.3586982488632202, + "rewards/margins": 0.16404885053634644, + "rewards/rejected": -1.5227470397949219, + "semantic_entropy": 0.7766925692558289, + "step": 3740 + }, + { + "epoch": 2.0043485532697773, + "grad_norm": 7.620126843705302, + "learning_rate": 2.994525893229581e-07, + "logits/chosen": -0.058956682682037354, + "logits/rejected": 0.04316861927509308, + "logps/chosen": -1.3086483478546143, + "logps/rejected": -1.5185747146606445, + "loss": 2.0332, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.3086483478546143, + "rewards/margins": 0.20992644131183624, + "rewards/rejected": -1.5185747146606445, + "semantic_entropy": 0.7986747622489929, + "step": 3745 + }, + { + "epoch": 2.007024586051179, + "grad_norm": 6.914423972752786, + "learning_rate": 2.98026926736732e-07, + "logits/chosen": -0.1123870238661766, + "logits/rejected": -0.023719770833849907, + "logps/chosen": -1.2111319303512573, + "logps/rejected": -1.542511224746704, + "loss": 1.9347, + "rewards/accuracies": 0.65625, + "rewards/chosen": -1.2111319303512573, + "rewards/margins": 0.33137932419776917, + "rewards/rejected": -1.542511224746704, + "semantic_entropy": 0.8043048977851868, + "step": 3750 + }, + { + "epoch": 2.0097006188325808, + "grad_norm": 8.314845242126639, + "learning_rate": 2.9660322369414846e-07, + "logits/chosen": -0.06645806133747101, + "logits/rejected": 0.04517248272895813, + "logps/chosen": -1.222695231437683, + "logps/rejected": -1.6410176753997803, + "loss": 1.936, + "rewards/accuracies": 0.65625, + "rewards/chosen": -1.222695231437683, + "rewards/margins": 0.41832247376441956, + "rewards/rejected": -1.6410176753997803, + "semantic_entropy": 0.7972557544708252, + "step": 3755 + }, + { + "epoch": 2.0123766516139825, + "grad_norm": 7.978805802446787, + "learning_rate": 2.9518149400798063e-07, + "logits/chosen": -0.14016160368919373, + "logits/rejected": -0.11200448125600815, + "logps/chosen": -1.2655471563339233, + "logps/rejected": -1.6075023412704468, + "loss": 1.9572, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2655471563339233, + "rewards/margins": 0.34195518493652344, + "rewards/rejected": -1.6075023412704468, + "semantic_entropy": 0.7767778635025024, + "step": 3760 + }, + { + "epoch": 2.0150526843953838, + "grad_norm": 10.85433114680117, + "learning_rate": 2.9376175147185633e-07, + "logits/chosen": -0.005012214183807373, + "logits/rejected": 0.1798551231622696, + "logps/chosen": -1.259853720664978, + "logps/rejected": -1.5986053943634033, + "loss": 1.9709, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -1.259853720664978, + "rewards/margins": 0.3387514650821686, + "rewards/rejected": -1.5986053943634033, + "semantic_entropy": 0.8001511693000793, + "step": 3765 + }, + { + "epoch": 2.0177287171767855, + "grad_norm": 10.519710493335946, + "learning_rate": 2.9234400986012376e-07, + "logits/chosen": -0.14667272567749023, + "logits/rejected": 0.024977799504995346, + "logps/chosen": -1.2061131000518799, + "logps/rejected": -1.6993818283081055, + "loss": 1.9044, + "rewards/accuracies": 0.71875, + "rewards/chosen": -1.2061131000518799, + "rewards/margins": 0.4932686388492584, + "rewards/rejected": -1.6993818283081055, + "semantic_entropy": 0.7931141257286072, + "step": 3770 + }, + { + "epoch": 2.020404749958187, + "grad_norm": 8.439243831534606, + "learning_rate": 2.9092828292771817e-07, + "logits/chosen": -0.08728514611721039, + "logits/rejected": -0.02273547649383545, + "logps/chosen": -1.2604572772979736, + "logps/rejected": -1.5512092113494873, + "loss": 1.9991, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.2604572772979736, + "rewards/margins": 0.2907518744468689, + "rewards/rejected": -1.5512092113494873, + "semantic_entropy": 0.8103083372116089, + "step": 3775 + }, + { + "epoch": 2.0230807827395885, + "grad_norm": 7.982311356152901, + "learning_rate": 2.8951458441002875e-07, + "logits/chosen": -0.032814525067806244, + "logits/rejected": 0.005210143513977528, + "logps/chosen": -1.273353099822998, + "logps/rejected": -1.5685460567474365, + "loss": 2.0184, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.273353099822998, + "rewards/margins": 0.2951931059360504, + "rewards/rejected": -1.5685460567474365, + "semantic_entropy": 0.7983887195587158, + "step": 3780 + }, + { + "epoch": 2.02575681552099, + "grad_norm": 7.081907610905893, + "learning_rate": 2.881029280227643e-07, + "logits/chosen": -0.09091036021709442, + "logits/rejected": 0.035645000636577606, + "logps/chosen": -1.296726942062378, + "logps/rejected": -1.6826658248901367, + "loss": 1.9666, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.296726942062378, + "rewards/margins": 0.3859389126300812, + "rewards/rejected": -1.6826658248901367, + "semantic_entropy": 0.772459864616394, + "step": 3785 + }, + { + "epoch": 2.028432848302392, + "grad_norm": 6.553036872618776, + "learning_rate": 2.8669332746182177e-07, + "logits/chosen": -0.1433906853199005, + "logits/rejected": 0.03814217075705528, + "logps/chosen": -1.2487014532089233, + "logps/rejected": -1.595126748085022, + "loss": 1.9387, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.2487014532089233, + "rewards/margins": 0.34642526507377625, + "rewards/rejected": -1.595126748085022, + "semantic_entropy": 0.7828398942947388, + "step": 3790 + }, + { + "epoch": 2.031108881083793, + "grad_norm": 6.515967009686399, + "learning_rate": 2.8528579640315156e-07, + "logits/chosen": -0.08073130995035172, + "logits/rejected": -0.052123237401247025, + "logps/chosen": -1.2313802242279053, + "logps/rejected": -1.4814410209655762, + "loss": 1.9817, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.2313802242279053, + "rewards/margins": 0.2500608563423157, + "rewards/rejected": -1.4814410209655762, + "semantic_entropy": 0.8185272216796875, + "step": 3795 + }, + { + "epoch": 2.033784913865195, + "grad_norm": 9.658581177515622, + "learning_rate": 2.8388034850262646e-07, + "logits/chosen": -0.048771053552627563, + "logits/rejected": 0.08041323721408844, + "logps/chosen": -1.3159829378128052, + "logps/rejected": -1.6725927591323853, + "loss": 2.0205, + "rewards/accuracies": 0.65625, + "rewards/chosen": -1.3159829378128052, + "rewards/margins": 0.35660985112190247, + "rewards/rejected": -1.6725927591323853, + "semantic_entropy": 0.7742539644241333, + "step": 3800 + }, + { + "epoch": 2.0364609466465966, + "grad_norm": 13.080769815083926, + "learning_rate": 2.824769973959079e-07, + "logits/chosen": -0.02432123012840748, + "logits/rejected": 0.1006104126572609, + "logps/chosen": -1.2159243822097778, + "logps/rejected": -1.528954267501831, + "loss": 1.9372, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2159243822097778, + "rewards/margins": 0.31303003430366516, + "rewards/rejected": -1.528954267501831, + "semantic_entropy": 0.8056100010871887, + "step": 3805 + }, + { + "epoch": 2.039136979427998, + "grad_norm": 8.574320370782875, + "learning_rate": 2.81075756698315e-07, + "logits/chosen": 0.03976669907569885, + "logits/rejected": 0.1289629340171814, + "logps/chosen": -1.2397609949111938, + "logps/rejected": -1.5856821537017822, + "loss": 1.9553, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2397609949111938, + "rewards/margins": 0.34592103958129883, + "rewards/rejected": -1.5856821537017822, + "semantic_entropy": 0.7904757261276245, + "step": 3810 + }, + { + "epoch": 2.0418130122093996, + "grad_norm": 7.864942736987948, + "learning_rate": 2.7967664000469035e-07, + "logits/chosen": -0.17100025713443756, + "logits/rejected": -0.03756319358944893, + "logps/chosen": -1.292011022567749, + "logps/rejected": -1.5119432210922241, + "loss": 2.0294, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.292011022567749, + "rewards/margins": 0.21993222832679749, + "rewards/rejected": -1.5119432210922241, + "semantic_entropy": 0.7964569926261902, + "step": 3815 + }, + { + "epoch": 2.0444890449908013, + "grad_norm": 8.272492234979593, + "learning_rate": 2.7827966088927095e-07, + "logits/chosen": -0.17397232353687286, + "logits/rejected": 0.03040078654885292, + "logps/chosen": -1.3632009029388428, + "logps/rejected": -1.5390888452529907, + "loss": 2.0824, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.3632009029388428, + "rewards/margins": 0.1758880466222763, + "rewards/rejected": -1.5390888452529907, + "semantic_entropy": 0.7751861214637756, + "step": 3820 + }, + { + "epoch": 2.0471650777722026, + "grad_norm": 9.290254425431332, + "learning_rate": 2.768848329055538e-07, + "logits/chosen": -0.12546177208423615, + "logits/rejected": -0.009615510702133179, + "logps/chosen": -1.2498613595962524, + "logps/rejected": -1.4855058193206787, + "loss": 1.9852, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.2498613595962524, + "rewards/margins": 0.2356446087360382, + "rewards/rejected": -1.4855058193206787, + "semantic_entropy": 0.8016610145568848, + "step": 3825 + }, + { + "epoch": 2.0498411105536043, + "grad_norm": 8.589930136904933, + "learning_rate": 2.7549216958616657e-07, + "logits/chosen": -0.17075549066066742, + "logits/rejected": -0.013943374156951904, + "logps/chosen": -1.316281795501709, + "logps/rejected": -1.6756961345672607, + "loss": 1.9949, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.316281795501709, + "rewards/margins": 0.3594144880771637, + "rewards/rejected": -1.6756961345672607, + "semantic_entropy": 0.7636662721633911, + "step": 3830 + }, + { + "epoch": 2.052517143335006, + "grad_norm": 8.141804117372102, + "learning_rate": 2.741016844427344e-07, + "logits/chosen": -0.07137881219387054, + "logits/rejected": 0.0819416269659996, + "logps/chosen": -1.3067326545715332, + "logps/rejected": -1.5788406133651733, + "loss": 2.0157, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.3067326545715332, + "rewards/margins": 0.27210795879364014, + "rewards/rejected": -1.5788406133651733, + "semantic_entropy": 0.7795847654342651, + "step": 3835 + }, + { + "epoch": 2.0551931761164073, + "grad_norm": 8.785325832990798, + "learning_rate": 2.7271339096575073e-07, + "logits/chosen": -0.03475064039230347, + "logits/rejected": 0.07072875648736954, + "logps/chosen": -1.2127859592437744, + "logps/rejected": -1.6222158670425415, + "loss": 1.9293, + "rewards/accuracies": 0.65625, + "rewards/chosen": -1.2127859592437744, + "rewards/margins": 0.4094300866127014, + "rewards/rejected": -1.6222158670425415, + "semantic_entropy": 0.8095159530639648, + "step": 3840 + }, + { + "epoch": 2.057869208897809, + "grad_norm": 7.770262670893247, + "learning_rate": 2.713273026244446e-07, + "logits/chosen": -0.2322789430618286, + "logits/rejected": 0.011026580817997456, + "logps/chosen": -1.3297617435455322, + "logps/rejected": -1.628687858581543, + "loss": 2.0274, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.3297617435455322, + "rewards/margins": 0.29892611503601074, + "rewards/rejected": -1.628687858581543, + "semantic_entropy": 0.7731846570968628, + "step": 3845 + }, + { + "epoch": 2.0605452416792107, + "grad_norm": 8.270676971050367, + "learning_rate": 2.6994343286665156e-07, + "logits/chosen": -0.08706966042518616, + "logits/rejected": 0.0894269123673439, + "logps/chosen": -1.3270047903060913, + "logps/rejected": -1.5941104888916016, + "loss": 2.0329, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.3270047903060913, + "rewards/margins": 0.26710575819015503, + "rewards/rejected": -1.5941104888916016, + "semantic_entropy": 0.7839833498001099, + "step": 3850 + }, + { + "epoch": 2.063221274460612, + "grad_norm": 9.677987341117074, + "learning_rate": 2.6856179511868156e-07, + "logits/chosen": -0.05726304650306702, + "logits/rejected": 0.13342034816741943, + "logps/chosen": -1.2690484523773193, + "logps/rejected": -1.6873533725738525, + "loss": 1.9796, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.2690484523773193, + "rewards/margins": 0.4183047413825989, + "rewards/rejected": -1.6873533725738525, + "semantic_entropy": 0.7959326505661011, + "step": 3855 + }, + { + "epoch": 2.0658973072420137, + "grad_norm": 6.222363162895002, + "learning_rate": 2.6718240278519056e-07, + "logits/chosen": -0.08423185348510742, + "logits/rejected": 0.08383387327194214, + "logps/chosen": -1.2890822887420654, + "logps/rejected": -1.6258800029754639, + "loss": 2.0136, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.2890822887420654, + "rewards/margins": 0.3367977738380432, + "rewards/rejected": -1.6258800029754639, + "semantic_entropy": 0.7936300039291382, + "step": 3860 + }, + { + "epoch": 2.0685733400234154, + "grad_norm": 11.215212145108538, + "learning_rate": 2.6580526924904866e-07, + "logits/chosen": -0.1955564320087433, + "logits/rejected": 0.0005769982817582786, + "logps/chosen": -1.325304388999939, + "logps/rejected": -1.5285425186157227, + "loss": 2.0362, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.325304388999939, + "rewards/margins": 0.2032381296157837, + "rewards/rejected": -1.5285425186157227, + "semantic_entropy": 0.7790963649749756, + "step": 3865 + }, + { + "epoch": 2.0712493728048167, + "grad_norm": 9.725281583079685, + "learning_rate": 2.6443040787121186e-07, + "logits/chosen": -0.10238544642925262, + "logits/rejected": -0.04848577454686165, + "logps/chosen": -1.1635031700134277, + "logps/rejected": -1.4549726247787476, + "loss": 1.9292, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.1635031700134277, + "rewards/margins": 0.2914695143699646, + "rewards/rejected": -1.4549726247787476, + "semantic_entropy": 0.8310944437980652, + "step": 3870 + }, + { + "epoch": 2.0739254055862184, + "grad_norm": 7.543625625162968, + "learning_rate": 2.6305783199059084e-07, + "logits/chosen": -0.10998735576868057, + "logits/rejected": 0.00942229200154543, + "logps/chosen": -1.263076901435852, + "logps/rejected": -1.5884946584701538, + "loss": 1.9641, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -1.263076901435852, + "rewards/margins": 0.3254178464412689, + "rewards/rejected": -1.5884946584701538, + "semantic_entropy": 0.7829529047012329, + "step": 3875 + }, + { + "epoch": 2.07660143836762, + "grad_norm": 7.455548456156887, + "learning_rate": 2.6168755492392324e-07, + "logits/chosen": -0.11983096599578857, + "logits/rejected": 0.05408283323049545, + "logps/chosen": -1.1768678426742554, + "logps/rejected": -1.528571367263794, + "loss": 1.911, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.1768678426742554, + "rewards/margins": 0.3517035245895386, + "rewards/rejected": -1.528571367263794, + "semantic_entropy": 0.8298446536064148, + "step": 3880 + }, + { + "epoch": 2.0792774711490214, + "grad_norm": 8.413218220585827, + "learning_rate": 2.6031958996564274e-07, + "logits/chosen": -0.15878412127494812, + "logits/rejected": -0.00622421782463789, + "logps/chosen": -1.2211363315582275, + "logps/rejected": -1.6070476770401, + "loss": 1.9309, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2211363315582275, + "rewards/margins": 0.3859114646911621, + "rewards/rejected": -1.6070476770401, + "semantic_entropy": 0.8035030364990234, + "step": 3885 + }, + { + "epoch": 2.081953503930423, + "grad_norm": 11.64726487557045, + "learning_rate": 2.589539503877518e-07, + "logits/chosen": -0.03869933634996414, + "logits/rejected": 0.05498753860592842, + "logps/chosen": -1.2614524364471436, + "logps/rejected": -1.5560925006866455, + "loss": 2.0024, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.2614524364471436, + "rewards/margins": 0.29464003443717957, + "rewards/rejected": -1.5560925006866455, + "semantic_entropy": 0.7954789400100708, + "step": 3890 + }, + { + "epoch": 2.084629536711825, + "grad_norm": 10.412752082048966, + "learning_rate": 2.5759064943969125e-07, + "logits/chosen": -0.11886237561702728, + "logits/rejected": 0.12702760100364685, + "logps/chosen": -1.2422895431518555, + "logps/rejected": -1.584733247756958, + "loss": 1.9709, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2422895431518555, + "rewards/margins": 0.3424435257911682, + "rewards/rejected": -1.584733247756958, + "semantic_entropy": 0.8005902171134949, + "step": 3895 + }, + { + "epoch": 2.087305569493226, + "grad_norm": 8.46319246124346, + "learning_rate": 2.562297003482131e-07, + "logits/chosen": 0.023574665188789368, + "logits/rejected": 0.03584318235516548, + "logps/chosen": -1.2451503276824951, + "logps/rejected": -1.533006191253662, + "loss": 1.9799, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.2451503276824951, + "rewards/margins": 0.28785592317581177, + "rewards/rejected": -1.533006191253662, + "semantic_entropy": 0.8111165165901184, + "step": 3900 + }, + { + "epoch": 2.089981602274628, + "grad_norm": 8.304832510053231, + "learning_rate": 2.548711163172512e-07, + "logits/chosen": -0.05365335941314697, + "logits/rejected": 0.048491910099983215, + "logps/chosen": -1.2720180749893188, + "logps/rejected": -1.5300616025924683, + "loss": 1.9934, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2720180749893188, + "rewards/margins": 0.2580435872077942, + "rewards/rejected": -1.5300616025924683, + "semantic_entropy": 0.7993375062942505, + "step": 3905 + }, + { + "epoch": 2.0926576350560295, + "grad_norm": 7.208604172092762, + "learning_rate": 2.53514910527794e-07, + "logits/chosen": -0.007045459933578968, + "logits/rejected": 0.12363393604755402, + "logps/chosen": -1.1774392127990723, + "logps/rejected": -1.5029500722885132, + "loss": 1.9469, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.1774392127990723, + "rewards/margins": 0.3255109488964081, + "rewards/rejected": -1.5029500722885132, + "semantic_entropy": 0.8183084726333618, + "step": 3910 + }, + { + "epoch": 2.095333667837431, + "grad_norm": 6.02671830959697, + "learning_rate": 2.5216109613775573e-07, + "logits/chosen": -0.0878453403711319, + "logits/rejected": 0.0802905336022377, + "logps/chosen": -1.295917272567749, + "logps/rejected": -1.6467256546020508, + "loss": 1.9905, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.295917272567749, + "rewards/margins": 0.3508082628250122, + "rewards/rejected": -1.6467256546020508, + "semantic_entropy": 0.792289137840271, + "step": 3915 + }, + { + "epoch": 2.0980097006188325, + "grad_norm": 8.246722913305083, + "learning_rate": 2.5080968628184993e-07, + "logits/chosen": -0.08657468855381012, + "logits/rejected": 0.0812126100063324, + "logps/chosen": -1.300894856452942, + "logps/rejected": -1.7418267726898193, + "loss": 1.9738, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.300894856452942, + "rewards/margins": 0.4409319758415222, + "rewards/rejected": -1.7418267726898193, + "semantic_entropy": 0.7787320017814636, + "step": 3920 + }, + { + "epoch": 2.1006857334002342, + "grad_norm": 9.365923612794083, + "learning_rate": 2.494606940714605e-07, + "logits/chosen": -0.08720335364341736, + "logits/rejected": 0.03497013822197914, + "logps/chosen": -1.2213890552520752, + "logps/rejected": -1.5773426294326782, + "loss": 1.9608, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.2213890552520752, + "rewards/margins": 0.3559534549713135, + "rewards/rejected": -1.5773426294326782, + "semantic_entropy": 0.787868320941925, + "step": 3925 + }, + { + "epoch": 2.103361766181636, + "grad_norm": 6.485820842635789, + "learning_rate": 2.4811413259451625e-07, + "logits/chosen": -0.1779341697692871, + "logits/rejected": -0.029109278693795204, + "logps/chosen": -1.3027786016464233, + "logps/rejected": -1.5649641752243042, + "loss": 2.0159, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.3027786016464233, + "rewards/margins": 0.2621855139732361, + "rewards/rejected": -1.5649641752243042, + "semantic_entropy": 0.784981369972229, + "step": 3930 + }, + { + "epoch": 2.106037798963037, + "grad_norm": 7.57817047099227, + "learning_rate": 2.46770014915362e-07, + "logits/chosen": -0.023013921454548836, + "logits/rejected": 0.04001970216631889, + "logps/chosen": -1.2584831714630127, + "logps/rejected": -1.6443601846694946, + "loss": 1.9534, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2584831714630127, + "rewards/margins": 0.38587701320648193, + "rewards/rejected": -1.6443601846694946, + "semantic_entropy": 0.7898402810096741, + "step": 3935 + }, + { + "epoch": 2.108713831744439, + "grad_norm": 11.18516150542784, + "learning_rate": 2.45428354074634e-07, + "logits/chosen": -0.04282800853252411, + "logits/rejected": 0.014397243969142437, + "logps/chosen": -1.1761744022369385, + "logps/rejected": -1.6283648014068604, + "loss": 1.8839, + "rewards/accuracies": 0.65625, + "rewards/chosen": -1.1761744022369385, + "rewards/margins": 0.45219022035598755, + "rewards/rejected": -1.6283648014068604, + "semantic_entropy": 0.8094109296798706, + "step": 3940 + }, + { + "epoch": 2.1113898645258407, + "grad_norm": 9.02202970388384, + "learning_rate": 2.4408916308913105e-07, + "logits/chosen": -0.06121363118290901, + "logits/rejected": 0.11865203082561493, + "logps/chosen": -1.308318018913269, + "logps/rejected": -1.5495134592056274, + "loss": 2.0296, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.308318018913269, + "rewards/margins": 0.2411954700946808, + "rewards/rejected": -1.5495134592056274, + "semantic_entropy": 0.7808352112770081, + "step": 3945 + }, + { + "epoch": 2.114065897307242, + "grad_norm": 10.43072144089605, + "learning_rate": 2.4275245495169025e-07, + "logits/chosen": 0.00885589700192213, + "logits/rejected": 0.17541439831256866, + "logps/chosen": -1.2490864992141724, + "logps/rejected": -1.4956276416778564, + "loss": 2.0013, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.2490864992141724, + "rewards/margins": 0.2465410977602005, + "rewards/rejected": -1.4956276416778564, + "semantic_entropy": 0.806951642036438, + "step": 3950 + }, + { + "epoch": 2.1167419300886436, + "grad_norm": 8.73901614305497, + "learning_rate": 2.414182426310597e-07, + "logits/chosen": -0.1357141137123108, + "logits/rejected": -0.04953531548380852, + "logps/chosen": -1.2262976169586182, + "logps/rejected": -1.5870901346206665, + "loss": 1.9462, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2262976169586182, + "rewards/margins": 0.36079269647598267, + "rewards/rejected": -1.5870901346206665, + "semantic_entropy": 0.7998301982879639, + "step": 3955 + }, + { + "epoch": 2.1194179628700454, + "grad_norm": 13.96331969015264, + "learning_rate": 2.400865390717734e-07, + "logits/chosen": -0.04178817197680473, + "logits/rejected": 0.07974360138177872, + "logps/chosen": -1.2890770435333252, + "logps/rejected": -1.7201932668685913, + "loss": 1.9744, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.2890770435333252, + "rewards/margins": 0.4311162531375885, + "rewards/rejected": -1.7201932668685913, + "semantic_entropy": 0.7699192762374878, + "step": 3960 + }, + { + "epoch": 2.1220939956514466, + "grad_norm": 8.690446841300258, + "learning_rate": 2.3875735719402475e-07, + "logits/chosen": 0.002413132693618536, + "logits/rejected": 0.11878538131713867, + "logps/chosen": -1.197887897491455, + "logps/rejected": -1.5999175310134888, + "loss": 1.9194, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -1.197887897491455, + "rewards/margins": 0.40202951431274414, + "rewards/rejected": -1.5999175310134888, + "semantic_entropy": 0.8078168630599976, + "step": 3965 + }, + { + "epoch": 2.1247700284328483, + "grad_norm": 7.546364801753625, + "learning_rate": 2.3743070989354258e-07, + "logits/chosen": -0.05478797107934952, + "logits/rejected": 0.040621694177389145, + "logps/chosen": -1.2546120882034302, + "logps/rejected": -1.6230491399765015, + "loss": 1.9688, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2546120882034302, + "rewards/margins": 0.3684369921684265, + "rewards/rejected": -1.6230491399765015, + "semantic_entropy": 0.7908979058265686, + "step": 3970 + }, + { + "epoch": 2.12744606121425, + "grad_norm": 7.943490277866662, + "learning_rate": 2.3610661004146454e-07, + "logits/chosen": -0.035158295184373856, + "logits/rejected": 0.0653868168592453, + "logps/chosen": -1.1693830490112305, + "logps/rejected": -1.4864637851715088, + "loss": 1.9245, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -1.1693830490112305, + "rewards/margins": 0.3170807361602783, + "rewards/rejected": -1.4864637851715088, + "semantic_entropy": 0.8256049156188965, + "step": 3975 + }, + { + "epoch": 2.1301220939956513, + "grad_norm": 9.99264274060387, + "learning_rate": 2.3478507048421314e-07, + "logits/chosen": -0.11048316955566406, + "logits/rejected": -0.030838940292596817, + "logps/chosen": -1.1681926250457764, + "logps/rejected": -1.6157619953155518, + "loss": 1.892, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -1.1681926250457764, + "rewards/margins": 0.4475693702697754, + "rewards/rejected": -1.6157619953155518, + "semantic_entropy": 0.8003697395324707, + "step": 3980 + }, + { + "epoch": 2.132798126777053, + "grad_norm": 14.451761814862564, + "learning_rate": 2.334661040433713e-07, + "logits/chosen": -0.1646009385585785, + "logits/rejected": -0.05796453356742859, + "logps/chosen": -1.2593486309051514, + "logps/rejected": -1.6181026697158813, + "loss": 1.9615, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -1.2593486309051514, + "rewards/margins": 0.3587539792060852, + "rewards/rejected": -1.6181026697158813, + "semantic_entropy": 0.7864263653755188, + "step": 3985 + }, + { + "epoch": 2.1354741595584548, + "grad_norm": 8.209541807724527, + "learning_rate": 2.321497235155568e-07, + "logits/chosen": -0.18386468291282654, + "logits/rejected": -0.04220528155565262, + "logps/chosen": -1.205427885055542, + "logps/rejected": -1.552451252937317, + "loss": 1.9238, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.205427885055542, + "rewards/margins": 0.3470233678817749, + "rewards/rejected": -1.552451252937317, + "semantic_entropy": 0.8057225942611694, + "step": 3990 + }, + { + "epoch": 2.138150192339856, + "grad_norm": 13.990666954895628, + "learning_rate": 2.3083594167229965e-07, + "logits/chosen": -0.22056451439857483, + "logits/rejected": 0.03705238923430443, + "logps/chosen": -1.2569921016693115, + "logps/rejected": -1.6521209478378296, + "loss": 1.9669, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2569921016693115, + "rewards/margins": 0.3951290249824524, + "rewards/rejected": -1.6521209478378296, + "semantic_entropy": 0.7817844152450562, + "step": 3995 + }, + { + "epoch": 2.1408262251212578, + "grad_norm": 15.435024760859893, + "learning_rate": 2.295247712599167e-07, + "logits/chosen": -0.0924561619758606, + "logits/rejected": 0.022307047620415688, + "logps/chosen": -1.2224881649017334, + "logps/rejected": -1.615614652633667, + "loss": 1.9276, + "rewards/accuracies": 0.65625, + "rewards/chosen": -1.2224881649017334, + "rewards/margins": 0.39312633872032166, + "rewards/rejected": -1.615614652633667, + "semantic_entropy": 0.7890762090682983, + "step": 4000 + }, + { + "epoch": 2.1408262251212578, + "eval_logits/chosen": 0.28233543038368225, + "eval_logits/rejected": 0.37708133459091187, + "eval_logps/chosen": -1.3312329053878784, + "eval_logps/rejected": -1.6023647785186768, + "eval_loss": 2.0445003509521484, + "eval_rewards/accuracies": 0.5905044674873352, + "eval_rewards/chosen": -1.3312329053878784, + "eval_rewards/margins": 0.2711319625377655, + "eval_rewards/rejected": -1.6023647785186768, + "eval_runtime": 34.171, + "eval_samples_per_second": 39.361, + "eval_semantic_entropy": 0.7784487009048462, + "eval_steps_per_second": 9.862, + "step": 4000 + }, + { + "epoch": 2.1435022579026595, + "grad_norm": 6.777658647158, + "learning_rate": 2.2821622499938948e-07, + "logits/chosen": -0.10082963854074478, + "logits/rejected": 0.12032053619623184, + "logps/chosen": -1.3868082761764526, + "logps/rejected": -1.6199979782104492, + "loss": 2.0674, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.3868082761764526, + "rewards/margins": 0.23318979144096375, + "rewards/rejected": -1.6199979782104492, + "semantic_entropy": 0.7673584222793579, + "step": 4005 + }, + { + "epoch": 2.1461782906840607, + "grad_norm": 13.218879283641312, + "learning_rate": 2.269103155862391e-07, + "logits/chosen": -0.09654682874679565, + "logits/rejected": 0.003932853229343891, + "logps/chosen": -1.2881536483764648, + "logps/rejected": -1.5453517436981201, + "loss": 1.9957, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2881536483764648, + "rewards/margins": 0.2571980357170105, + "rewards/rejected": -1.5453517436981201, + "semantic_entropy": 0.7780659198760986, + "step": 4010 + }, + { + "epoch": 2.1488543234654625, + "grad_norm": 8.059694189229388, + "learning_rate": 2.2560705569040483e-07, + "logits/chosen": -0.10584896802902222, + "logits/rejected": 0.1518443524837494, + "logps/chosen": -1.2734086513519287, + "logps/rejected": -1.5498818159103394, + "loss": 1.9992, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.2734086513519287, + "rewards/margins": 0.27647334337234497, + "rewards/rejected": -1.5498818159103394, + "semantic_entropy": 0.7976066470146179, + "step": 4015 + }, + { + "epoch": 2.151530356246864, + "grad_norm": 7.686166913386505, + "learning_rate": 2.2430645795611963e-07, + "logits/chosen": -0.18073877692222595, + "logits/rejected": -0.03493395447731018, + "logps/chosen": -1.301017165184021, + "logps/rejected": -1.5369887351989746, + "loss": 2.0196, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.301017165184021, + "rewards/margins": 0.23597149550914764, + "rewards/rejected": -1.5369887351989746, + "semantic_entropy": 0.7954763174057007, + "step": 4020 + }, + { + "epoch": 2.1542063890282654, + "grad_norm": 8.340165419877227, + "learning_rate": 2.230085350017884e-07, + "logits/chosen": -0.11597704887390137, + "logits/rejected": -0.012895789928734303, + "logps/chosen": -1.20939040184021, + "logps/rejected": -1.5225939750671387, + "loss": 1.9657, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.20939040184021, + "rewards/margins": 0.3132036328315735, + "rewards/rejected": -1.5225939750671387, + "semantic_entropy": 0.8167934417724609, + "step": 4025 + }, + { + "epoch": 2.156882421809667, + "grad_norm": 10.629523832293694, + "learning_rate": 2.2171329941986554e-07, + "logits/chosen": -0.1574414223432541, + "logits/rejected": -0.07325611263513565, + "logps/chosen": -1.2151435613632202, + "logps/rejected": -1.5735080242156982, + "loss": 1.9421, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2151435613632202, + "rewards/margins": 0.358364462852478, + "rewards/rejected": -1.5735080242156982, + "semantic_entropy": 0.7974958419799805, + "step": 4030 + }, + { + "epoch": 2.159558454591069, + "grad_norm": 14.813281848954883, + "learning_rate": 2.2042076377673202e-07, + "logits/chosen": -0.10038616508245468, + "logits/rejected": -0.06362534314393997, + "logps/chosen": -1.2242885828018188, + "logps/rejected": -1.3778797388076782, + "loss": 2.0183, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2242885828018188, + "rewards/margins": 0.153591126203537, + "rewards/rejected": -1.3778797388076782, + "semantic_entropy": 0.8234738111495972, + "step": 4035 + }, + { + "epoch": 2.16223448737247, + "grad_norm": 7.133608092036389, + "learning_rate": 2.1913094061257476e-07, + "logits/chosen": -0.11537346988916397, + "logits/rejected": -0.07308430969715118, + "logps/chosen": -1.1973235607147217, + "logps/rejected": -1.4738342761993408, + "loss": 1.9631, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.1973235607147217, + "rewards/margins": 0.2765108346939087, + "rewards/rejected": -1.4738342761993408, + "semantic_entropy": 0.8326338529586792, + "step": 4040 + }, + { + "epoch": 2.164910520153872, + "grad_norm": 9.65466716777848, + "learning_rate": 2.178438424412633e-07, + "logits/chosen": -0.07614657282829285, + "logits/rejected": 0.052105795592069626, + "logps/chosen": -1.2976658344268799, + "logps/rejected": -1.5549640655517578, + "loss": 2.0149, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.2976658344268799, + "rewards/margins": 0.25729843974113464, + "rewards/rejected": -1.5549640655517578, + "semantic_entropy": 0.7852495312690735, + "step": 4045 + }, + { + "epoch": 2.1675865529352736, + "grad_norm": 5.3450360348739006, + "learning_rate": 2.165594817502302e-07, + "logits/chosen": -0.18720772862434387, + "logits/rejected": -0.08179843425750732, + "logps/chosen": -1.2786951065063477, + "logps/rejected": -1.4834058284759521, + "loss": 2.0237, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2786951065063477, + "rewards/margins": 0.2047106921672821, + "rewards/rejected": -1.4834058284759521, + "semantic_entropy": 0.7854777574539185, + "step": 4050 + }, + { + "epoch": 2.170262585716675, + "grad_norm": 7.106303803593254, + "learning_rate": 2.1527787100034806e-07, + "logits/chosen": -0.05425887182354927, + "logits/rejected": 0.00020194947137497365, + "logps/chosen": -1.2666957378387451, + "logps/rejected": -1.4778165817260742, + "loss": 1.9981, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.2666957378387451, + "rewards/margins": 0.21112075448036194, + "rewards/rejected": -1.4778165817260742, + "semantic_entropy": 0.808153510093689, + "step": 4055 + }, + { + "epoch": 2.1729386184980766, + "grad_norm": 11.034265016869863, + "learning_rate": 2.1399902262581037e-07, + "logits/chosen": 0.0023912787437438965, + "logits/rejected": 0.10449610650539398, + "logps/chosen": -1.1998052597045898, + "logps/rejected": -1.4737387895584106, + "loss": 1.9525, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.1998052597045898, + "rewards/margins": 0.2739335000514984, + "rewards/rejected": -1.4737387895584106, + "semantic_entropy": 0.8287736773490906, + "step": 4060 + }, + { + "epoch": 2.1756146512794783, + "grad_norm": 8.768829635920307, + "learning_rate": 2.127229490340094e-07, + "logits/chosen": -0.1924888640642166, + "logits/rejected": -0.09673380851745605, + "logps/chosen": -1.2586601972579956, + "logps/rejected": -1.641990303993225, + "loss": 1.9554, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2586601972579956, + "rewards/margins": 0.38332998752593994, + "rewards/rejected": -1.641990303993225, + "semantic_entropy": 0.7948935627937317, + "step": 4065 + }, + { + "epoch": 2.1782906840608796, + "grad_norm": 12.865765144912011, + "learning_rate": 2.1144966260541698e-07, + "logits/chosen": -0.09876732528209686, + "logits/rejected": 0.1042884811758995, + "logps/chosen": -1.2214984893798828, + "logps/rejected": -1.5891913175582886, + "loss": 1.9443, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2214984893798828, + "rewards/margins": 0.3676929175853729, + "rewards/rejected": -1.5891913175582886, + "semantic_entropy": 0.8093553781509399, + "step": 4070 + }, + { + "epoch": 2.1809667168422813, + "grad_norm": 10.135158048631018, + "learning_rate": 2.1017917569346332e-07, + "logits/chosen": -0.14737705886363983, + "logits/rejected": 0.036610908806324005, + "logps/chosen": -1.311643362045288, + "logps/rejected": -1.5534847974777222, + "loss": 2.0146, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.311643362045288, + "rewards/margins": 0.24184155464172363, + "rewards/rejected": -1.5534847974777222, + "semantic_entropy": 0.7842815518379211, + "step": 4075 + }, + { + "epoch": 2.183642749623683, + "grad_norm": 5.280114652940724, + "learning_rate": 2.0891150062441837e-07, + "logits/chosen": -0.14050161838531494, + "logits/rejected": -0.004181402735412121, + "logps/chosen": -1.2973473072052002, + "logps/rejected": -1.6200370788574219, + "loss": 1.9934, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2973473072052002, + "rewards/margins": 0.32268989086151123, + "rewards/rejected": -1.6200370788574219, + "semantic_entropy": 0.7707660794258118, + "step": 4080 + }, + { + "epoch": 2.1863187824050843, + "grad_norm": 7.507466722589746, + "learning_rate": 2.0764664969727086e-07, + "logits/chosen": -0.10917153209447861, + "logits/rejected": -0.014076301828026772, + "logps/chosen": -1.2789490222930908, + "logps/rejected": -1.4958078861236572, + "loss": 2.0392, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.2789490222930908, + "rewards/margins": 0.2168588638305664, + "rewards/rejected": -1.4958078861236572, + "semantic_entropy": 0.7913199067115784, + "step": 4085 + }, + { + "epoch": 2.188994815186486, + "grad_norm": 8.443772130300719, + "learning_rate": 2.0638463518361033e-07, + "logits/chosen": -0.19746491312980652, + "logits/rejected": -0.0061895460821688175, + "logps/chosen": -1.2484588623046875, + "logps/rejected": -1.5168395042419434, + "loss": 1.9785, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2484588623046875, + "rewards/margins": 0.26838067173957825, + "rewards/rejected": -1.5168395042419434, + "semantic_entropy": 0.796512246131897, + "step": 4090 + }, + { + "epoch": 2.1916708479678877, + "grad_norm": 10.320844957676552, + "learning_rate": 2.0512546932750702e-07, + "logits/chosen": -0.15037508308887482, + "logits/rejected": -0.06879515200853348, + "logps/chosen": -1.3391551971435547, + "logps/rejected": -1.5366318225860596, + "loss": 2.0446, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.3391551971435547, + "rewards/margins": 0.19747667014598846, + "rewards/rejected": -1.5366318225860596, + "semantic_entropy": 0.7697279453277588, + "step": 4095 + }, + { + "epoch": 2.194346880749289, + "grad_norm": 8.874663144842327, + "learning_rate": 2.0386916434539343e-07, + "logits/chosen": -0.10112161934375763, + "logits/rejected": 0.03384857252240181, + "logps/chosen": -1.1738734245300293, + "logps/rejected": -1.5382874011993408, + "loss": 1.925, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.1738734245300293, + "rewards/margins": 0.36441394686698914, + "rewards/rejected": -1.5382874011993408, + "semantic_entropy": 0.8074756860733032, + "step": 4100 + }, + { + "epoch": 2.1970229135306907, + "grad_norm": 11.524650233894052, + "learning_rate": 2.0261573242594627e-07, + "logits/chosen": -0.11960120499134064, + "logits/rejected": 0.07483614981174469, + "logps/chosen": -1.3214560747146606, + "logps/rejected": -1.556175947189331, + "loss": 2.0245, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.3214560747146606, + "rewards/margins": 0.23472002148628235, + "rewards/rejected": -1.556175947189331, + "semantic_entropy": 0.783279299736023, + "step": 4105 + }, + { + "epoch": 2.1996989463120924, + "grad_norm": 10.565157270268786, + "learning_rate": 2.0136518572996724e-07, + "logits/chosen": -0.08820009231567383, + "logits/rejected": 0.07982397079467773, + "logps/chosen": -1.2397620677947998, + "logps/rejected": -1.518204689025879, + "loss": 1.9902, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2397620677947998, + "rewards/margins": 0.2784424424171448, + "rewards/rejected": -1.518204689025879, + "semantic_entropy": 0.8072455525398254, + "step": 4110 + }, + { + "epoch": 2.202374979093494, + "grad_norm": 8.282720482886361, + "learning_rate": 2.0011753639026617e-07, + "logits/chosen": -0.07058636844158173, + "logits/rejected": -0.027735024690628052, + "logps/chosen": -1.2462873458862305, + "logps/rejected": -1.6336066722869873, + "loss": 1.9538, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -1.2462873458862305, + "rewards/margins": 0.38731926679611206, + "rewards/rejected": -1.6336066722869873, + "semantic_entropy": 0.7994504570960999, + "step": 4115 + }, + { + "epoch": 2.2050510118748954, + "grad_norm": 6.648714961392694, + "learning_rate": 1.988727965115421e-07, + "logits/chosen": -0.11287019401788712, + "logits/rejected": -0.035372935235500336, + "logps/chosen": -1.1937533617019653, + "logps/rejected": -1.5723878145217896, + "loss": 1.9056, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -1.1937533617019653, + "rewards/margins": 0.37863463163375854, + "rewards/rejected": -1.5723878145217896, + "semantic_entropy": 0.7976509928703308, + "step": 4120 + }, + { + "epoch": 2.207727044656297, + "grad_norm": 11.00571287521211, + "learning_rate": 1.9763097817026713e-07, + "logits/chosen": -0.16754977405071259, + "logits/rejected": 0.03706652671098709, + "logps/chosen": -1.2285130023956299, + "logps/rejected": -1.5773698091506958, + "loss": 1.9546, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2285130023956299, + "rewards/margins": 0.3488568663597107, + "rewards/rejected": -1.5773698091506958, + "semantic_entropy": 0.8013304471969604, + "step": 4125 + }, + { + "epoch": 2.210403077437699, + "grad_norm": 8.80659775953107, + "learning_rate": 1.9639209341456796e-07, + "logits/chosen": -0.08111747354269028, + "logits/rejected": -0.0008040537941269577, + "logps/chosen": -1.2347103357315063, + "logps/rejected": -1.581979751586914, + "loss": 1.9478, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.2347103357315063, + "rewards/margins": 0.3472694456577301, + "rewards/rejected": -1.581979751586914, + "semantic_entropy": 0.8024226427078247, + "step": 4130 + }, + { + "epoch": 2.2130791102191, + "grad_norm": 17.10761812582053, + "learning_rate": 1.951561542641102e-07, + "logits/chosen": -0.07774971425533295, + "logits/rejected": -0.08791448175907135, + "logps/chosen": -1.2992351055145264, + "logps/rejected": -1.6637672185897827, + "loss": 2.0077, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.2992351055145264, + "rewards/margins": 0.36453211307525635, + "rewards/rejected": -1.6637672185897827, + "semantic_entropy": 0.7671725749969482, + "step": 4135 + }, + { + "epoch": 2.215755143000502, + "grad_norm": 9.270540567718726, + "learning_rate": 1.939231727099806e-07, + "logits/chosen": -0.25526660680770874, + "logits/rejected": -0.1716885268688202, + "logps/chosen": -1.2138149738311768, + "logps/rejected": -1.5514978170394897, + "loss": 1.9477, + "rewards/accuracies": 0.65625, + "rewards/chosen": -1.2138149738311768, + "rewards/margins": 0.337682843208313, + "rewards/rejected": -1.5514978170394897, + "semantic_entropy": 0.8171249628067017, + "step": 4140 + }, + { + "epoch": 2.2184311757819035, + "grad_norm": 8.253484382591838, + "learning_rate": 1.926931607145719e-07, + "logits/chosen": -0.06769241392612457, + "logits/rejected": 0.06590889394283295, + "logps/chosen": -1.3090394735336304, + "logps/rejected": -1.6794878244400024, + "loss": 2.002, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.3090394735336304, + "rewards/margins": 0.3704483211040497, + "rewards/rejected": -1.6794878244400024, + "semantic_entropy": 0.7758277654647827, + "step": 4145 + }, + { + "epoch": 2.221107208563305, + "grad_norm": 11.290111124403301, + "learning_rate": 1.9146613021146564e-07, + "logits/chosen": -0.0959344133734703, + "logits/rejected": -0.003268678905442357, + "logps/chosen": -1.203951358795166, + "logps/rejected": -1.5836807489395142, + "loss": 1.9386, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.203951358795166, + "rewards/margins": 0.37972959876060486, + "rewards/rejected": -1.5836807489395142, + "semantic_entropy": 0.7990487217903137, + "step": 4150 + }, + { + "epoch": 2.2237832413447065, + "grad_norm": 10.982154674202642, + "learning_rate": 1.9024209310531736e-07, + "logits/chosen": -0.03283507376909256, + "logits/rejected": -0.05256788060069084, + "logps/chosen": -1.2322564125061035, + "logps/rejected": -1.5404446125030518, + "loss": 1.9473, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -1.2322564125061035, + "rewards/margins": 0.30818822979927063, + "rewards/rejected": -1.5404446125030518, + "semantic_entropy": 0.794259250164032, + "step": 4155 + }, + { + "epoch": 2.2264592741261082, + "grad_norm": 11.44996223685397, + "learning_rate": 1.890210612717401e-07, + "logits/chosen": -0.1292242854833603, + "logits/rejected": 0.018556052818894386, + "logps/chosen": -1.3012385368347168, + "logps/rejected": -1.582773208618164, + "loss": 2.01, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.3012385368347168, + "rewards/margins": 0.28153473138809204, + "rewards/rejected": -1.582773208618164, + "semantic_entropy": 0.7758509516716003, + "step": 4160 + }, + { + "epoch": 2.2291353069075095, + "grad_norm": 7.439751836298014, + "learning_rate": 1.8780304655719054e-07, + "logits/chosen": -0.13371321558952332, + "logits/rejected": -0.025500833988189697, + "logps/chosen": -1.2886149883270264, + "logps/rejected": -1.6176551580429077, + "loss": 1.9978, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.2886149883270264, + "rewards/margins": 0.329039990901947, + "rewards/rejected": -1.6176551580429077, + "semantic_entropy": 0.7811014652252197, + "step": 4165 + }, + { + "epoch": 2.231811339688911, + "grad_norm": 9.912674788694538, + "learning_rate": 1.865880607788523e-07, + "logits/chosen": -0.012343516573309898, + "logits/rejected": 0.0449819378554821, + "logps/chosen": -1.273308277130127, + "logps/rejected": -1.610945701599121, + "loss": 1.971, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.273308277130127, + "rewards/margins": 0.33763742446899414, + "rewards/rejected": -1.610945701599121, + "semantic_entropy": 0.7837294340133667, + "step": 4170 + }, + { + "epoch": 2.234487372470313, + "grad_norm": 8.108254067117462, + "learning_rate": 1.8537611572452316e-07, + "logits/chosen": -0.14038607478141785, + "logits/rejected": -0.039006926119327545, + "logps/chosen": -1.2530543804168701, + "logps/rejected": -1.4794385433197021, + "loss": 1.9945, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.2530543804168701, + "rewards/margins": 0.22638408839702606, + "rewards/rejected": -1.4794385433197021, + "semantic_entropy": 0.8008133172988892, + "step": 4175 + }, + { + "epoch": 2.237163405251714, + "grad_norm": 11.866315689855641, + "learning_rate": 1.84167223152499e-07, + "logits/chosen": -0.1662590205669403, + "logits/rejected": 0.048979468643665314, + "logps/chosen": -1.199005365371704, + "logps/rejected": -1.56072199344635, + "loss": 1.9336, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": -1.199005365371704, + "rewards/margins": 0.36171668767929077, + "rewards/rejected": -1.56072199344635, + "semantic_entropy": 0.8317713737487793, + "step": 4180 + }, + { + "epoch": 2.239839438033116, + "grad_norm": 11.019023925593954, + "learning_rate": 1.8296139479146112e-07, + "logits/chosen": -0.19250182807445526, + "logits/rejected": -0.18602564930915833, + "logps/chosen": -1.1668365001678467, + "logps/rejected": -1.524772047996521, + "loss": 1.9172, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.1668365001678467, + "rewards/margins": 0.3579355776309967, + "rewards/rejected": -1.524772047996521, + "semantic_entropy": 0.8143719434738159, + "step": 4185 + }, + { + "epoch": 2.2425154708145176, + "grad_norm": 11.552918601934442, + "learning_rate": 1.8175864234036132e-07, + "logits/chosen": -0.032421402633190155, + "logits/rejected": 0.07119415700435638, + "logps/chosen": -1.248134970664978, + "logps/rejected": -1.5825941562652588, + "loss": 1.9738, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.248134970664978, + "rewards/margins": 0.334459125995636, + "rewards/rejected": -1.5825941562652588, + "semantic_entropy": 0.8027989268302917, + "step": 4190 + }, + { + "epoch": 2.245191503595919, + "grad_norm": 5.834167395205214, + "learning_rate": 1.805589774683094e-07, + "logits/chosen": -0.243893101811409, + "logits/rejected": -0.08822005242109299, + "logps/chosen": -1.2915481328964233, + "logps/rejected": -1.5085653066635132, + "loss": 2.0228, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2915481328964233, + "rewards/margins": 0.2170170247554779, + "rewards/rejected": -1.5085653066635132, + "semantic_entropy": 0.7923386096954346, + "step": 4195 + }, + { + "epoch": 2.2478675363773206, + "grad_norm": 10.490816788351006, + "learning_rate": 1.79362411814459e-07, + "logits/chosen": 0.012433795258402824, + "logits/rejected": -0.003731709672138095, + "logps/chosen": -1.3018280267715454, + "logps/rejected": -1.6060158014297485, + "loss": 1.9941, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.3018280267715454, + "rewards/margins": 0.3041878342628479, + "rewards/rejected": -1.6060158014297485, + "semantic_entropy": 0.7790161967277527, + "step": 4200 + }, + { + "epoch": 2.2505435691587223, + "grad_norm": 10.451097828510381, + "learning_rate": 1.7816895698789552e-07, + "logits/chosen": -0.18953083455562592, + "logits/rejected": -0.11586049944162369, + "logps/chosen": -1.2320678234100342, + "logps/rejected": -1.557760238647461, + "loss": 1.9327, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.2320678234100342, + "rewards/margins": 0.32569244503974915, + "rewards/rejected": -1.557760238647461, + "semantic_entropy": 0.801048755645752, + "step": 4205 + }, + { + "epoch": 2.2532196019401236, + "grad_norm": 9.394165279246357, + "learning_rate": 1.7697862456752271e-07, + "logits/chosen": -0.14001984894275665, + "logits/rejected": -0.02964564599096775, + "logps/chosen": -1.2370903491973877, + "logps/rejected": -1.7004258632659912, + "loss": 1.9495, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.2370903491973877, + "rewards/margins": 0.4633353352546692, + "rewards/rejected": -1.7004258632659912, + "semantic_entropy": 0.7974084615707397, + "step": 4210 + }, + { + "epoch": 2.2558956347215253, + "grad_norm": 7.41481622790433, + "learning_rate": 1.7579142610195124e-07, + "logits/chosen": -0.11725671589374542, + "logits/rejected": 0.018145056441426277, + "logps/chosen": -1.265994906425476, + "logps/rejected": -1.5646344423294067, + "loss": 1.9885, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.265994906425476, + "rewards/margins": 0.2986395061016083, + "rewards/rejected": -1.5646344423294067, + "semantic_entropy": 0.787617027759552, + "step": 4215 + }, + { + "epoch": 2.258571667502927, + "grad_norm": 8.497384684424222, + "learning_rate": 1.7460737310938568e-07, + "logits/chosen": -0.1787947118282318, + "logits/rejected": 0.02110464498400688, + "logps/chosen": -1.2498914003372192, + "logps/rejected": -1.543668508529663, + "loss": 1.9893, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.2498914003372192, + "rewards/margins": 0.2937771677970886, + "rewards/rejected": -1.543668508529663, + "semantic_entropy": 0.7971317768096924, + "step": 4220 + }, + { + "epoch": 2.2612477002843283, + "grad_norm": 10.470912497403106, + "learning_rate": 1.734264770775133e-07, + "logits/chosen": -0.16025003790855408, + "logits/rejected": 0.01915592886507511, + "logps/chosen": -1.2886840105056763, + "logps/rejected": -1.6545966863632202, + "loss": 1.9668, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.2886840105056763, + "rewards/margins": 0.3659127950668335, + "rewards/rejected": -1.6545966863632202, + "semantic_entropy": 0.7657202482223511, + "step": 4225 + }, + { + "epoch": 2.26392373306573, + "grad_norm": 8.935706182234096, + "learning_rate": 1.7224874946339241e-07, + "logits/chosen": -0.16925734281539917, + "logits/rejected": -0.08762809634208679, + "logps/chosen": -1.322160005569458, + "logps/rejected": -1.6796150207519531, + "loss": 2.0271, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.322160005569458, + "rewards/margins": 0.35745516419410706, + "rewards/rejected": -1.6796150207519531, + "semantic_entropy": 0.781524121761322, + "step": 4230 + }, + { + "epoch": 2.2665997658471317, + "grad_norm": 8.343351194328045, + "learning_rate": 1.7107420169334186e-07, + "logits/chosen": -0.12380006164312363, + "logits/rejected": -0.005908069666475058, + "logps/chosen": -1.3306140899658203, + "logps/rejected": -1.4885650873184204, + "loss": 2.0753, + "rewards/accuracies": 0.5062500238418579, + "rewards/chosen": -1.3306140899658203, + "rewards/margins": 0.1579509973526001, + "rewards/rejected": -1.4885650873184204, + "semantic_entropy": 0.7841423749923706, + "step": 4235 + }, + { + "epoch": 2.269275798628533, + "grad_norm": 9.747809456731876, + "learning_rate": 1.6990284516282893e-07, + "logits/chosen": -0.12111307680606842, + "logits/rejected": -0.0011890288442373276, + "logps/chosen": -1.257455825805664, + "logps/rejected": -1.4483282566070557, + "loss": 2.0125, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.257455825805664, + "rewards/margins": 0.1908724457025528, + "rewards/rejected": -1.4483282566070557, + "semantic_entropy": 0.8028818368911743, + "step": 4240 + }, + { + "epoch": 2.2719518314099347, + "grad_norm": 7.533871407979239, + "learning_rate": 1.687346912363602e-07, + "logits/chosen": -0.2015412300825119, + "logits/rejected": -0.07601834833621979, + "logps/chosen": -1.2797132730484009, + "logps/rejected": -1.4889013767242432, + "loss": 2.0075, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2797132730484009, + "rewards/margins": 0.20918801426887512, + "rewards/rejected": -1.4889013767242432, + "semantic_entropy": 0.8059288263320923, + "step": 4245 + }, + { + "epoch": 2.2746278641913364, + "grad_norm": 5.385013656266259, + "learning_rate": 1.675697512473697e-07, + "logits/chosen": -0.10228011757135391, + "logits/rejected": 0.07375967502593994, + "logps/chosen": -1.2664943933486938, + "logps/rejected": -1.5417674779891968, + "loss": 2.0087, + "rewards/accuracies": 0.5562499761581421, + "rewards/chosen": -1.2664943933486938, + "rewards/margins": 0.2752731740474701, + "rewards/rejected": -1.5417674779891968, + "semantic_entropy": 0.7941025495529175, + "step": 4250 + }, + { + "epoch": 2.2773038969727377, + "grad_norm": 12.063702544187802, + "learning_rate": 1.6640803649811087e-07, + "logits/chosen": -0.1658763885498047, + "logits/rejected": 0.06614699214696884, + "logps/chosen": -1.286105990409851, + "logps/rejected": -1.6867536306381226, + "loss": 1.9867, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.286105990409851, + "rewards/margins": 0.4006476402282715, + "rewards/rejected": -1.6867536306381226, + "semantic_entropy": 0.7811282277107239, + "step": 4255 + }, + { + "epoch": 2.2799799297541394, + "grad_norm": 10.859671266641389, + "learning_rate": 1.6524955825954472e-07, + "logits/chosen": -0.14719471335411072, + "logits/rejected": -0.04040377587080002, + "logps/chosen": -1.2015544176101685, + "logps/rejected": -1.5611904859542847, + "loss": 1.9292, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.2015544176101685, + "rewards/margins": 0.35963618755340576, + "rewards/rejected": -1.5611904859542847, + "semantic_entropy": 0.808398425579071, + "step": 4260 + }, + { + "epoch": 2.282655962535541, + "grad_norm": 7.040659457503802, + "learning_rate": 1.6409432777123277e-07, + "logits/chosen": -0.25105080008506775, + "logits/rejected": -0.08539889752864838, + "logps/chosen": -1.2529833316802979, + "logps/rejected": -1.5797755718231201, + "loss": 1.9642, + "rewards/accuracies": 0.65625, + "rewards/chosen": -1.2529833316802979, + "rewards/margins": 0.3267921507358551, + "rewards/rejected": -1.5797755718231201, + "semantic_entropy": 0.7997857332229614, + "step": 4265 + }, + { + "epoch": 2.285331995316943, + "grad_norm": 6.433767783550345, + "learning_rate": 1.6294235624122577e-07, + "logits/chosen": -0.055332403630018234, + "logits/rejected": 0.1894211620092392, + "logps/chosen": -1.2655553817749023, + "logps/rejected": -1.7325023412704468, + "loss": 1.9706, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -1.2655553817749023, + "rewards/margins": 0.466947078704834, + "rewards/rejected": -1.7325023412704468, + "semantic_entropy": 0.7825048565864563, + "step": 4270 + }, + { + "epoch": 2.288008028098344, + "grad_norm": 12.153539061712653, + "learning_rate": 1.6179365484595697e-07, + "logits/chosen": -0.09241259098052979, + "logits/rejected": -0.00836194772273302, + "logps/chosen": -1.3286699056625366, + "logps/rejected": -1.6374403238296509, + "loss": 2.0464, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.3286699056625366, + "rewards/margins": 0.3087703585624695, + "rewards/rejected": -1.6374403238296509, + "semantic_entropy": 0.7748314142227173, + "step": 4275 + }, + { + "epoch": 2.290684060879746, + "grad_norm": 7.697929758201799, + "learning_rate": 1.60648234730132e-07, + "logits/chosen": -0.13402453064918518, + "logits/rejected": -0.043066900223493576, + "logps/chosen": -1.2240922451019287, + "logps/rejected": -1.5100966691970825, + "loss": 1.9501, + "rewards/accuracies": 0.65625, + "rewards/chosen": -1.2240922451019287, + "rewards/margins": 0.28600430488586426, + "rewards/rejected": -1.5100966691970825, + "semantic_entropy": 0.8081742525100708, + "step": 4280 + }, + { + "epoch": 2.293360093661147, + "grad_norm": 14.290716459143816, + "learning_rate": 1.595061070066222e-07, + "logits/chosen": -0.09124065935611725, + "logits/rejected": -0.09486423432826996, + "logps/chosen": -1.1951037645339966, + "logps/rejected": -1.5133121013641357, + "loss": 1.9142, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": -1.1951037645339966, + "rewards/margins": 0.318208247423172, + "rewards/rejected": -1.5133121013641357, + "semantic_entropy": 0.818766713142395, + "step": 4285 + }, + { + "epoch": 2.296036126442549, + "grad_norm": 15.187423626425694, + "learning_rate": 1.5836728275635542e-07, + "logits/chosen": -0.2084455043077469, + "logits/rejected": -0.03481137752532959, + "logps/chosen": -1.2978460788726807, + "logps/rejected": -1.568542242050171, + "loss": 2.0049, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.2978460788726807, + "rewards/margins": 0.2706960141658783, + "rewards/rejected": -1.568542242050171, + "semantic_entropy": 0.7863389253616333, + "step": 4290 + }, + { + "epoch": 2.2987121592239506, + "grad_norm": 9.04534945918805, + "learning_rate": 1.5723177302820984e-07, + "logits/chosen": -0.18676696717739105, + "logits/rejected": -0.1372944712638855, + "logps/chosen": -1.2712112665176392, + "logps/rejected": -1.5649312734603882, + "loss": 1.9921, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.2712112665176392, + "rewards/margins": 0.2937201261520386, + "rewards/rejected": -1.5649312734603882, + "semantic_entropy": 0.7963592410087585, + "step": 4295 + }, + { + "epoch": 2.3013881920053523, + "grad_norm": 13.090168405149521, + "learning_rate": 1.5609958883890544e-07, + "logits/chosen": -0.13981503248214722, + "logits/rejected": -0.026316970586776733, + "logps/chosen": -1.2637109756469727, + "logps/rejected": -1.489816427230835, + "loss": 1.9939, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2637109756469727, + "rewards/margins": 0.22610552608966827, + "rewards/rejected": -1.489816427230835, + "semantic_entropy": 0.7971670031547546, + "step": 4300 + }, + { + "epoch": 2.3040642247867535, + "grad_norm": 33.64487463400989, + "learning_rate": 1.5497074117289865e-07, + "logits/chosen": -0.20040778815746307, + "logits/rejected": -0.0799594298005104, + "logps/chosen": -1.2583087682724, + "logps/rejected": -1.6361068487167358, + "loss": 1.9477, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.2583087682724, + "rewards/margins": 0.3777979612350464, + "rewards/rejected": -1.6361068487167358, + "semantic_entropy": 0.7779967188835144, + "step": 4305 + }, + { + "epoch": 2.3067402575681553, + "grad_norm": 9.214329986138315, + "learning_rate": 1.5384524098227402e-07, + "logits/chosen": -0.15226741135120392, + "logits/rejected": 0.047097109258174896, + "logps/chosen": -1.249534010887146, + "logps/rejected": -1.632002830505371, + "loss": 1.9639, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.249534010887146, + "rewards/margins": 0.38246893882751465, + "rewards/rejected": -1.632002830505371, + "semantic_entropy": 0.796162486076355, + "step": 4310 + }, + { + "epoch": 2.3094162903495565, + "grad_norm": 9.427152395024008, + "learning_rate": 1.5272309918663974e-07, + "logits/chosen": -0.13251027464866638, + "logits/rejected": 0.018474791198968887, + "logps/chosen": -1.2655764818191528, + "logps/rejected": -1.520042061805725, + "loss": 2.0069, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2655764818191528, + "rewards/margins": 0.2544656991958618, + "rewards/rejected": -1.520042061805725, + "semantic_entropy": 0.7987126111984253, + "step": 4315 + }, + { + "epoch": 2.3120923231309582, + "grad_norm": 9.13175447086419, + "learning_rate": 1.516043266730201e-07, + "logits/chosen": -0.16241349279880524, + "logits/rejected": -0.036643363535404205, + "logps/chosen": -1.30803382396698, + "logps/rejected": -1.5431315898895264, + "loss": 2.026, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.30803382396698, + "rewards/margins": 0.23509757220745087, + "rewards/rejected": -1.5431315898895264, + "semantic_entropy": 0.7765775918960571, + "step": 4320 + }, + { + "epoch": 2.31476835591236, + "grad_norm": 13.967831571275822, + "learning_rate": 1.504889342957512e-07, + "logits/chosen": -0.15671613812446594, + "logits/rejected": -0.0015728086000308394, + "logps/chosen": -1.2362313270568848, + "logps/rejected": -1.6587120294570923, + "loss": 1.957, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.2362313270568848, + "rewards/margins": 0.42248082160949707, + "rewards/rejected": -1.6587120294570923, + "semantic_entropy": 0.7952734231948853, + "step": 4325 + }, + { + "epoch": 2.3174443886937617, + "grad_norm": 7.5115255304772806, + "learning_rate": 1.4937693287637453e-07, + "logits/chosen": -0.136986643075943, + "logits/rejected": -0.029413629323244095, + "logps/chosen": -1.346637487411499, + "logps/rejected": -1.5729660987854004, + "loss": 2.0729, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.346637487411499, + "rewards/margins": 0.22632858157157898, + "rewards/rejected": -1.5729660987854004, + "semantic_entropy": 0.7861993908882141, + "step": 4330 + }, + { + "epoch": 2.320120421475163, + "grad_norm": 11.798958316728067, + "learning_rate": 1.4826833320353305e-07, + "logits/chosen": -0.10806053876876831, + "logits/rejected": -0.043422140181064606, + "logps/chosen": -1.3138458728790283, + "logps/rejected": -1.6097075939178467, + "loss": 2.042, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.3138458728790283, + "rewards/margins": 0.29586172103881836, + "rewards/rejected": -1.6097075939178467, + "semantic_entropy": 0.774515688419342, + "step": 4335 + }, + { + "epoch": 2.3227964542565647, + "grad_norm": 9.40444803947687, + "learning_rate": 1.4716314603286528e-07, + "logits/chosen": -0.15934400260448456, + "logits/rejected": 0.005942508578300476, + "logps/chosen": -1.175724983215332, + "logps/rejected": -1.696840524673462, + "loss": 1.8892, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.175724983215332, + "rewards/margins": 0.5211154818534851, + "rewards/rejected": -1.696840524673462, + "semantic_entropy": 0.8058635592460632, + "step": 4340 + }, + { + "epoch": 2.3254724870379664, + "grad_norm": 11.730118366986853, + "learning_rate": 1.4606138208690233e-07, + "logits/chosen": -0.19282224774360657, + "logits/rejected": -0.11361217498779297, + "logps/chosen": -1.3833080530166626, + "logps/rejected": -1.4851402044296265, + "loss": 2.1013, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.3833080530166626, + "rewards/margins": 0.10183216631412506, + "rewards/rejected": -1.4851402044296265, + "semantic_entropy": 0.7746142745018005, + "step": 4345 + }, + { + "epoch": 2.3281485198193677, + "grad_norm": 8.00361879964917, + "learning_rate": 1.4496305205496251e-07, + "logits/chosen": -0.0834389254450798, + "logits/rejected": -0.018457237631082535, + "logps/chosen": -1.282725214958191, + "logps/rejected": -1.6338307857513428, + "loss": 1.9837, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.282725214958191, + "rewards/margins": 0.3511054515838623, + "rewards/rejected": -1.6338307857513428, + "semantic_entropy": 0.7893123030662537, + "step": 4350 + }, + { + "epoch": 2.3308245526007694, + "grad_norm": 7.258530204816981, + "learning_rate": 1.4386816659304895e-07, + "logits/chosen": -0.20895186066627502, + "logits/rejected": -0.1091470941901207, + "logps/chosen": -1.3068081140518188, + "logps/rejected": -1.569459319114685, + "loss": 1.9947, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.3068081140518188, + "rewards/margins": 0.2626512050628662, + "rewards/rejected": -1.569459319114685, + "semantic_entropy": 0.7754660844802856, + "step": 4355 + }, + { + "epoch": 2.333500585382171, + "grad_norm": 7.252764634339286, + "learning_rate": 1.4277673632374492e-07, + "logits/chosen": -0.216875821352005, + "logits/rejected": -0.004487229976803064, + "logps/chosen": -1.3538143634796143, + "logps/rejected": -1.5858434438705444, + "loss": 2.044, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.3538143634796143, + "rewards/margins": 0.2320290058851242, + "rewards/rejected": -1.5858434438705444, + "semantic_entropy": 0.7725597620010376, + "step": 4360 + }, + { + "epoch": 2.3361766181635724, + "grad_norm": 8.511889251040571, + "learning_rate": 1.416887718361119e-07, + "logits/chosen": -0.0578872449696064, + "logits/rejected": -0.058798693120479584, + "logps/chosen": -1.2196348905563354, + "logps/rejected": -1.5464431047439575, + "loss": 1.9545, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2196348905563354, + "rewards/margins": 0.32680806517601013, + "rewards/rejected": -1.5464431047439575, + "semantic_entropy": 0.8095118403434753, + "step": 4365 + }, + { + "epoch": 2.338852650944974, + "grad_norm": 10.827800542341361, + "learning_rate": 1.406042836855859e-07, + "logits/chosen": -0.12696470320224762, + "logits/rejected": 0.003375391010195017, + "logps/chosen": -1.1783816814422607, + "logps/rejected": -1.6317615509033203, + "loss": 1.8999, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.1783816814422607, + "rewards/margins": 0.45337972044944763, + "rewards/rejected": -1.6317615509033203, + "semantic_entropy": 0.8124502301216125, + "step": 4370 + }, + { + "epoch": 2.341528683726376, + "grad_norm": 13.967101137336337, + "learning_rate": 1.3952328239387595e-07, + "logits/chosen": -0.23310494422912598, + "logits/rejected": -0.04387233406305313, + "logps/chosen": -1.295742392539978, + "logps/rejected": -1.6016197204589844, + "loss": 1.9967, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.295742392539978, + "rewards/margins": 0.305877149105072, + "rewards/rejected": -1.6016197204589844, + "semantic_entropy": 0.7674862146377563, + "step": 4375 + }, + { + "epoch": 2.344204716507777, + "grad_norm": 7.9268173721038035, + "learning_rate": 1.3844577844886109e-07, + "logits/chosen": -0.18139347434043884, + "logits/rejected": -0.004594838712364435, + "logps/chosen": -1.3157854080200195, + "logps/rejected": -1.582556962966919, + "loss": 2.0215, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.3157854080200195, + "rewards/margins": 0.26677125692367554, + "rewards/rejected": -1.582556962966919, + "semantic_entropy": 0.7893223762512207, + "step": 4380 + }, + { + "epoch": 2.346880749289179, + "grad_norm": 10.589401436703588, + "learning_rate": 1.3737178230448955e-07, + "logits/chosen": -0.20681917667388916, + "logits/rejected": -0.06286197155714035, + "logps/chosen": -1.2401567697525024, + "logps/rejected": -1.6419627666473389, + "loss": 1.9314, + "rewards/accuracies": 0.65625, + "rewards/chosen": -1.2401567697525024, + "rewards/margins": 0.4018060266971588, + "rewards/rejected": -1.6419627666473389, + "semantic_entropy": 0.795062243938446, + "step": 4385 + }, + { + "epoch": 2.3495567820705805, + "grad_norm": 6.967609256646875, + "learning_rate": 1.363013043806764e-07, + "logits/chosen": -0.17762355506420135, + "logits/rejected": -0.05149609595537186, + "logps/chosen": -1.2417309284210205, + "logps/rejected": -1.5034621953964233, + "loss": 1.9855, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2417309284210205, + "rewards/margins": 0.26173120737075806, + "rewards/rejected": -1.5034621953964233, + "semantic_entropy": 0.8084238171577454, + "step": 4390 + }, + { + "epoch": 2.3522328148519818, + "grad_norm": 8.132486478881162, + "learning_rate": 1.352343550632034e-07, + "logits/chosen": -0.1426798701286316, + "logits/rejected": 0.0015723813557997346, + "logps/chosen": -1.280442476272583, + "logps/rejected": -1.7234470844268799, + "loss": 1.9787, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.280442476272583, + "rewards/margins": 0.44300445914268494, + "rewards/rejected": -1.7234470844268799, + "semantic_entropy": 0.7840474843978882, + "step": 4395 + }, + { + "epoch": 2.3549088476333835, + "grad_norm": 6.714940579293801, + "learning_rate": 1.3417094470361722e-07, + "logits/chosen": -0.2102196216583252, + "logits/rejected": -0.06310750544071198, + "logps/chosen": -1.2375186681747437, + "logps/rejected": -1.6087543964385986, + "loss": 1.9438, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.2375186681747437, + "rewards/margins": 0.37123560905456543, + "rewards/rejected": -1.6087543964385986, + "semantic_entropy": 0.799569845199585, + "step": 4400 + }, + { + "epoch": 2.3549088476333835, + "eval_logits/chosen": 0.16329367458820343, + "eval_logits/rejected": 0.24929870665073395, + "eval_logps/chosen": -1.3380868434906006, + "eval_logps/rejected": -1.615173578262329, + "eval_loss": 2.044917583465576, + "eval_rewards/accuracies": 0.5942136645317078, + "eval_rewards/chosen": -1.3380868434906006, + "eval_rewards/margins": 0.27708685398101807, + "eval_rewards/rejected": -1.615173578262329, + "eval_runtime": 34.7176, + "eval_samples_per_second": 38.741, + "eval_semantic_entropy": 0.7753984332084656, + "eval_steps_per_second": 9.707, + "step": 4400 + }, + { + "epoch": 2.357584880414785, + "grad_norm": 7.596164051086343, + "learning_rate": 1.3311108361913015e-07, + "logits/chosen": -0.21881377696990967, + "logits/rejected": -0.17788389325141907, + "logps/chosen": -1.2354278564453125, + "logps/rejected": -1.4550416469573975, + "loss": 1.9945, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2354278564453125, + "rewards/margins": 0.21961364150047302, + "rewards/rejected": -1.4550416469573975, + "semantic_entropy": 0.8208361864089966, + "step": 4405 + }, + { + "epoch": 2.3602609131961865, + "grad_norm": 7.299831982798536, + "learning_rate": 1.3205478209251874e-07, + "logits/chosen": -0.17249411344528198, + "logits/rejected": -0.14244119822978973, + "logps/chosen": -1.3397963047027588, + "logps/rejected": -1.7014557123184204, + "loss": 2.0249, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -1.3397963047027588, + "rewards/margins": 0.3616591989994049, + "rewards/rejected": -1.7014557123184204, + "semantic_entropy": 0.7657409906387329, + "step": 4410 + }, + { + "epoch": 2.362936945977588, + "grad_norm": 8.269555867593573, + "learning_rate": 1.310020503720254e-07, + "logits/chosen": -0.13108180463314056, + "logits/rejected": 0.027709677815437317, + "logps/chosen": -1.2893644571304321, + "logps/rejected": -1.5708500146865845, + "loss": 2.0016, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.2893644571304321, + "rewards/margins": 0.2814854681491852, + "rewards/rejected": -1.5708500146865845, + "semantic_entropy": 0.7902665734291077, + "step": 4415 + }, + { + "epoch": 2.36561297875899, + "grad_norm": 10.519560075344469, + "learning_rate": 1.2995289867125752e-07, + "logits/chosen": -0.16770415008068085, + "logits/rejected": -0.08232822269201279, + "logps/chosen": -1.3121337890625, + "logps/rejected": -1.4900834560394287, + "loss": 2.0503, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.3121337890625, + "rewards/margins": 0.17794954776763916, + "rewards/rejected": -1.4900834560394287, + "semantic_entropy": 0.7878513932228088, + "step": 4420 + }, + { + "epoch": 2.368289011540391, + "grad_norm": 9.567672683788386, + "learning_rate": 1.2890733716908986e-07, + "logits/chosen": -0.14897114038467407, + "logits/rejected": -0.04493894428014755, + "logps/chosen": -1.2099311351776123, + "logps/rejected": -1.5338737964630127, + "loss": 1.9236, + "rewards/accuracies": 0.65625, + "rewards/chosen": -1.2099311351776123, + "rewards/margins": 0.3239426612854004, + "rewards/rejected": -1.5338737964630127, + "semantic_entropy": 0.8020665049552917, + "step": 4425 + }, + { + "epoch": 2.370965044321793, + "grad_norm": 7.5503797512692135, + "learning_rate": 1.2786537600956454e-07, + "logits/chosen": -0.1726982593536377, + "logits/rejected": -0.00577501067891717, + "logps/chosen": -1.3117141723632812, + "logps/rejected": -1.6250488758087158, + "loss": 2.0053, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.3117141723632812, + "rewards/margins": 0.3133348226547241, + "rewards/rejected": -1.6250488758087158, + "semantic_entropy": 0.7708010673522949, + "step": 4430 + }, + { + "epoch": 2.3736410771031946, + "grad_norm": 7.026259775073373, + "learning_rate": 1.268270253017933e-07, + "logits/chosen": -0.25417202711105347, + "logits/rejected": -0.04576032608747482, + "logps/chosen": -1.2097904682159424, + "logps/rejected": -1.5549266338348389, + "loss": 1.9324, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -1.2097904682159424, + "rewards/margins": 0.34513604640960693, + "rewards/rejected": -1.5549266338348389, + "semantic_entropy": 0.8113786578178406, + "step": 4435 + }, + { + "epoch": 2.376317109884596, + "grad_norm": 11.649166736443838, + "learning_rate": 1.257922951198591e-07, + "logits/chosen": -0.24857386946678162, + "logits/rejected": -0.04840501770377159, + "logps/chosen": -1.3066952228546143, + "logps/rejected": -1.5375475883483887, + "loss": 2.0276, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.3066952228546143, + "rewards/margins": 0.23085255920886993, + "rewards/rejected": -1.5375475883483887, + "semantic_entropy": 0.7833686470985413, + "step": 4440 + }, + { + "epoch": 2.3789931426659976, + "grad_norm": 9.528891181841956, + "learning_rate": 1.24761195502719e-07, + "logits/chosen": -0.20785681903362274, + "logits/rejected": -0.026444310322403908, + "logps/chosen": -1.2633240222930908, + "logps/rejected": -1.555964708328247, + "loss": 2.0104, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": -1.2633240222930908, + "rewards/margins": 0.2926408350467682, + "rewards/rejected": -1.555964708328247, + "semantic_entropy": 0.7821134924888611, + "step": 4445 + }, + { + "epoch": 2.3816691754473993, + "grad_norm": 10.98983209743162, + "learning_rate": 1.2373373645410573e-07, + "logits/chosen": -0.11539150774478912, + "logits/rejected": 0.014820380136370659, + "logps/chosen": -1.3207120895385742, + "logps/rejected": -1.648510217666626, + "loss": 2.0113, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.3207120895385742, + "rewards/margins": 0.3277982473373413, + "rewards/rejected": -1.648510217666626, + "semantic_entropy": 0.7733095288276672, + "step": 4450 + }, + { + "epoch": 2.384345208228801, + "grad_norm": 8.857866072317233, + "learning_rate": 1.2270992794243175e-07, + "logits/chosen": -0.22098317742347717, + "logits/rejected": -0.12461896240711212, + "logps/chosen": -1.3009580373764038, + "logps/rejected": -1.5917580127716064, + "loss": 2.0043, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.3009580373764038, + "rewards/margins": 0.29079997539520264, + "rewards/rejected": -1.5917580127716064, + "semantic_entropy": 0.7838797569274902, + "step": 4455 + }, + { + "epoch": 2.3870212410102023, + "grad_norm": 11.89718596474689, + "learning_rate": 1.2168977990069147e-07, + "logits/chosen": -0.2289695292711258, + "logits/rejected": -0.0018982291221618652, + "logps/chosen": -1.2188758850097656, + "logps/rejected": -1.6825908422470093, + "loss": 1.9148, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.2188758850097656, + "rewards/margins": 0.463715136051178, + "rewards/rejected": -1.6825908422470093, + "semantic_entropy": 0.802188515663147, + "step": 4460 + }, + { + "epoch": 2.389697273791604, + "grad_norm": 7.267727110713755, + "learning_rate": 1.206733022263659e-07, + "logits/chosen": -0.20777158439159393, + "logits/rejected": -0.022397834807634354, + "logps/chosen": -1.3242433071136475, + "logps/rejected": -1.5615819692611694, + "loss": 2.0477, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.3242433071136475, + "rewards/margins": 0.23733873665332794, + "rewards/rejected": -1.5615819692611694, + "semantic_entropy": 0.7919968366622925, + "step": 4465 + }, + { + "epoch": 2.3923733065730053, + "grad_norm": 9.276616217142173, + "learning_rate": 1.1966050478132572e-07, + "logits/chosen": -0.08144637942314148, + "logits/rejected": -0.006868818309158087, + "logps/chosen": -1.1734609603881836, + "logps/rejected": -1.466123104095459, + "loss": 1.9461, + "rewards/accuracies": 0.65625, + "rewards/chosen": -1.1734609603881836, + "rewards/margins": 0.29266220331192017, + "rewards/rejected": -1.466123104095459, + "semantic_entropy": 0.8322067260742188, + "step": 4470 + }, + { + "epoch": 2.395049339354407, + "grad_norm": 10.285711021808112, + "learning_rate": 1.1865139739173635e-07, + "logits/chosen": -0.17131993174552917, + "logits/rejected": 0.04961296170949936, + "logps/chosen": -1.2460860013961792, + "logps/rejected": -1.560231328010559, + "loss": 1.9621, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.2460860013961792, + "rewards/margins": 0.31414538621902466, + "rewards/rejected": -1.560231328010559, + "semantic_entropy": 0.7961411476135254, + "step": 4475 + }, + { + "epoch": 2.3977253721358087, + "grad_norm": 12.552218630123326, + "learning_rate": 1.1764598984796187e-07, + "logits/chosen": -0.15812304615974426, + "logits/rejected": -0.07235546410083771, + "logps/chosen": -1.180234432220459, + "logps/rejected": -1.4472579956054688, + "loss": 1.9585, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.180234432220459, + "rewards/margins": 0.26702359318733215, + "rewards/rejected": -1.4472579956054688, + "semantic_entropy": 0.8320823907852173, + "step": 4480 + }, + { + "epoch": 2.4004014049172104, + "grad_norm": 14.335288849532814, + "learning_rate": 1.1664429190447095e-07, + "logits/chosen": -0.15513862669467926, + "logits/rejected": -0.05408088117837906, + "logps/chosen": -1.2913730144500732, + "logps/rejected": -1.6561282873153687, + "loss": 2.0005, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.2913730144500732, + "rewards/margins": 0.364755243062973, + "rewards/rejected": -1.6561282873153687, + "semantic_entropy": 0.7903246879577637, + "step": 4485 + }, + { + "epoch": 2.4030774376986117, + "grad_norm": 9.286973520864837, + "learning_rate": 1.1564631327974122e-07, + "logits/chosen": -0.20581397414207458, + "logits/rejected": 0.0023672953248023987, + "logps/chosen": -1.251208782196045, + "logps/rejected": -1.6634395122528076, + "loss": 1.9302, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -1.251208782196045, + "rewards/margins": 0.412230908870697, + "rewards/rejected": -1.6634395122528076, + "semantic_entropy": 0.781873881816864, + "step": 4490 + }, + { + "epoch": 2.4057534704800134, + "grad_norm": 11.572794160860106, + "learning_rate": 1.1465206365616587e-07, + "logits/chosen": -0.27398204803466797, + "logits/rejected": -0.06987808644771576, + "logps/chosen": -1.2382861375808716, + "logps/rejected": -1.5518062114715576, + "loss": 1.9656, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2382861375808716, + "rewards/margins": 0.31352001428604126, + "rewards/rejected": -1.5518062114715576, + "semantic_entropy": 0.8056244850158691, + "step": 4495 + }, + { + "epoch": 2.408429503261415, + "grad_norm": 7.479658537224136, + "learning_rate": 1.1366155267995887e-07, + "logits/chosen": -0.0675680860877037, + "logits/rejected": -0.06864787638187408, + "logps/chosen": -1.2645561695098877, + "logps/rejected": -1.5393455028533936, + "loss": 1.9899, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2645561695098877, + "rewards/margins": 0.2747894823551178, + "rewards/rejected": -1.5393455028533936, + "semantic_entropy": 0.7883538007736206, + "step": 4500 + }, + { + "epoch": 2.4111055360428164, + "grad_norm": 12.011976990634052, + "learning_rate": 1.1267478996106228e-07, + "logits/chosen": -0.21940676867961884, + "logits/rejected": -0.0764898955821991, + "logps/chosen": -1.252802848815918, + "logps/rejected": -1.6330541372299194, + "loss": 1.9569, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.252802848815918, + "rewards/margins": 0.3802511990070343, + "rewards/rejected": -1.6330541372299194, + "semantic_entropy": 0.7870553135871887, + "step": 4505 + }, + { + "epoch": 2.413781568824218, + "grad_norm": 13.002078309301691, + "learning_rate": 1.116917850730521e-07, + "logits/chosen": -0.22240960597991943, + "logits/rejected": -0.09078870713710785, + "logps/chosen": -1.2161216735839844, + "logps/rejected": -1.4928818941116333, + "loss": 1.9765, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.2161216735839844, + "rewards/margins": 0.27676016092300415, + "rewards/rejected": -1.4928818941116333, + "semantic_entropy": 0.8293957710266113, + "step": 4510 + }, + { + "epoch": 2.41645760160562, + "grad_norm": 6.662239576361183, + "learning_rate": 1.1071254755304637e-07, + "logits/chosen": -0.19843384623527527, + "logits/rejected": -0.12356531620025635, + "logps/chosen": -1.2522376775741577, + "logps/rejected": -1.585841417312622, + "loss": 1.96, + "rewards/accuracies": 0.65625, + "rewards/chosen": -1.2522376775741577, + "rewards/margins": 0.3336038589477539, + "rewards/rejected": -1.585841417312622, + "semantic_entropy": 0.7871413230895996, + "step": 4515 + }, + { + "epoch": 2.419133634387021, + "grad_norm": 9.333033231682505, + "learning_rate": 1.0973708690161143e-07, + "logits/chosen": -0.16699953377246857, + "logits/rejected": -0.07821333408355713, + "logps/chosen": -1.233888864517212, + "logps/rejected": -1.620827317237854, + "loss": 1.9267, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.233888864517212, + "rewards/margins": 0.38693851232528687, + "rewards/rejected": -1.620827317237854, + "semantic_entropy": 0.7967000007629395, + "step": 4520 + }, + { + "epoch": 2.421809667168423, + "grad_norm": 11.82695740036326, + "learning_rate": 1.0876541258267119e-07, + "logits/chosen": -0.24223777651786804, + "logits/rejected": -0.06421225517988205, + "logps/chosen": -1.353435754776001, + "logps/rejected": -1.679324746131897, + "loss": 2.0156, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.353435754776001, + "rewards/margins": 0.32588905096054077, + "rewards/rejected": -1.679324746131897, + "semantic_entropy": 0.7549802660942078, + "step": 4525 + }, + { + "epoch": 2.4244856999498245, + "grad_norm": 6.298179954701871, + "learning_rate": 1.0779753402341379e-07, + "logits/chosen": -0.21974477171897888, + "logits/rejected": -0.12402723729610443, + "logps/chosen": -1.2834287881851196, + "logps/rejected": -1.5347378253936768, + "loss": 2.0122, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2834287881851196, + "rewards/margins": 0.25130897760391235, + "rewards/rejected": -1.5347378253936768, + "semantic_entropy": 0.793082594871521, + "step": 4530 + }, + { + "epoch": 2.427161732731226, + "grad_norm": 8.08713473417835, + "learning_rate": 1.0683346061420157e-07, + "logits/chosen": -0.0673285499215126, + "logits/rejected": 0.03805375471711159, + "logps/chosen": -1.2332861423492432, + "logps/rejected": -1.5473854541778564, + "loss": 1.9842, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.2332861423492432, + "rewards/margins": 0.31409937143325806, + "rewards/rejected": -1.5473854541778564, + "semantic_entropy": 0.7957872152328491, + "step": 4535 + }, + { + "epoch": 2.4298377655126275, + "grad_norm": 7.452381023087329, + "learning_rate": 1.0587320170847874e-07, + "logits/chosen": -0.10687969624996185, + "logits/rejected": -0.037527017295360565, + "logps/chosen": -1.216997504234314, + "logps/rejected": -1.5355608463287354, + "loss": 1.9651, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.216997504234314, + "rewards/margins": 0.3185632824897766, + "rewards/rejected": -1.5355608463287354, + "semantic_entropy": 0.8129452466964722, + "step": 4540 + }, + { + "epoch": 2.4325137982940293, + "grad_norm": 8.387316558142318, + "learning_rate": 1.0491676662268156e-07, + "logits/chosen": -0.13831308484077454, + "logits/rejected": -0.007977311499416828, + "logps/chosen": -1.2193646430969238, + "logps/rejected": -1.551476240158081, + "loss": 1.978, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2193646430969238, + "rewards/margins": 0.33211150765419006, + "rewards/rejected": -1.551476240158081, + "semantic_entropy": 0.8200357556343079, + "step": 4545 + }, + { + "epoch": 2.4351898310754305, + "grad_norm": 11.386155774547941, + "learning_rate": 1.0396416463614732e-07, + "logits/chosen": -0.2331191748380661, + "logits/rejected": -0.1288275122642517, + "logps/chosen": -1.1922945976257324, + "logps/rejected": -1.5517548322677612, + "loss": 1.9585, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.1922945976257324, + "rewards/margins": 0.35946017503738403, + "rewards/rejected": -1.5517548322677612, + "semantic_entropy": 0.8209033012390137, + "step": 4550 + }, + { + "epoch": 2.4378658638568322, + "grad_norm": 9.123494885824242, + "learning_rate": 1.0301540499102479e-07, + "logits/chosen": -0.16231347620487213, + "logits/rejected": -0.09428876638412476, + "logps/chosen": -1.315527081489563, + "logps/rejected": -1.5254814624786377, + "loss": 2.0267, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.315527081489563, + "rewards/margins": 0.2099544107913971, + "rewards/rejected": -1.5254814624786377, + "semantic_entropy": 0.7753323316574097, + "step": 4555 + }, + { + "epoch": 2.440541896638234, + "grad_norm": 10.900991920728584, + "learning_rate": 1.0207049689218405e-07, + "logits/chosen": -0.1922842115163803, + "logits/rejected": -0.01617896556854248, + "logps/chosen": -1.2199078798294067, + "logps/rejected": -1.5163733959197998, + "loss": 1.9736, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.2199078798294067, + "rewards/margins": 0.29646551609039307, + "rewards/rejected": -1.5163733959197998, + "semantic_entropy": 0.8171917796134949, + "step": 4560 + }, + { + "epoch": 2.4432179294196352, + "grad_norm": 15.26690120576109, + "learning_rate": 1.0112944950712782e-07, + "logits/chosen": -0.14947383105754852, + "logits/rejected": -0.06626245379447937, + "logps/chosen": -1.3093255758285522, + "logps/rejected": -1.574352741241455, + "loss": 2.0129, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.3093255758285522, + "rewards/margins": 0.26502716541290283, + "rewards/rejected": -1.574352741241455, + "semantic_entropy": 0.7754411101341248, + "step": 4565 + }, + { + "epoch": 2.445893962201037, + "grad_norm": 10.773928342514377, + "learning_rate": 1.0019227196590174e-07, + "logits/chosen": -0.10572180896997452, + "logits/rejected": 0.0469868965446949, + "logps/chosen": -1.2659401893615723, + "logps/rejected": -1.6247440576553345, + "loss": 1.969, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.2659401893615723, + "rewards/margins": 0.35880374908447266, + "rewards/rejected": -1.6247440576553345, + "semantic_entropy": 0.7877820134162903, + "step": 4570 + }, + { + "epoch": 2.4485699949824387, + "grad_norm": 8.452581423842831, + "learning_rate": 9.925897336100664e-08, + "logits/chosen": -0.06749458611011505, + "logits/rejected": -0.013055408373475075, + "logps/chosen": -1.2595086097717285, + "logps/rejected": -1.55685293674469, + "loss": 1.9857, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2595086097717285, + "rewards/margins": 0.2973443865776062, + "rewards/rejected": -1.55685293674469, + "semantic_entropy": 0.7922778129577637, + "step": 4575 + }, + { + "epoch": 2.45124602776384, + "grad_norm": 9.189277140411864, + "learning_rate": 9.832956274730946e-08, + "logits/chosen": -0.16322560608386993, + "logits/rejected": -0.12031744420528412, + "logps/chosen": -1.2119220495224, + "logps/rejected": -1.5800087451934814, + "loss": 1.9331, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -1.2119220495224, + "rewards/margins": 0.36808669567108154, + "rewards/rejected": -1.5800087451934814, + "semantic_entropy": 0.7977925539016724, + "step": 4580 + }, + { + "epoch": 2.4539220605452416, + "grad_norm": 7.486266111441383, + "learning_rate": 9.740404914195633e-08, + "logits/chosen": -0.13839499652385712, + "logits/rejected": -0.005443167872726917, + "logps/chosen": -1.3202592134475708, + "logps/rejected": -1.5658947229385376, + "loss": 2.0481, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.3202592134475708, + "rewards/margins": 0.24563536047935486, + "rewards/rejected": -1.5658947229385376, + "semantic_entropy": 0.7758022546768188, + "step": 4585 + }, + { + "epoch": 2.4565980933266434, + "grad_norm": 7.631634394696681, + "learning_rate": 9.648244152428392e-08, + "logits/chosen": -0.26290029287338257, + "logits/rejected": -0.08336208015680313, + "logps/chosen": -1.2126243114471436, + "logps/rejected": -1.52330482006073, + "loss": 1.9656, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.2126243114471436, + "rewards/margins": 0.3106805682182312, + "rewards/rejected": -1.52330482006073, + "semantic_entropy": 0.8142677545547485, + "step": 4590 + }, + { + "epoch": 2.4592741261080446, + "grad_norm": 8.523281182411818, + "learning_rate": 9.556474883573379e-08, + "logits/chosen": -0.20482425391674042, + "logits/rejected": -0.09075477719306946, + "logps/chosen": -1.264404535293579, + "logps/rejected": -1.6324313879013062, + "loss": 1.9884, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.264404535293579, + "rewards/margins": 0.3680269420146942, + "rewards/rejected": -1.6324313879013062, + "semantic_entropy": 0.7956129312515259, + "step": 4595 + }, + { + "epoch": 2.4619501588894463, + "grad_norm": 10.034992700566619, + "learning_rate": 9.465097997976412e-08, + "logits/chosen": -0.21117672324180603, + "logits/rejected": 0.03872231766581535, + "logps/chosen": -1.3046801090240479, + "logps/rejected": -1.6931383609771729, + "loss": 1.9877, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.3046801090240479, + "rewards/margins": 0.38845834136009216, + "rewards/rejected": -1.6931383609771729, + "semantic_entropy": 0.769001841545105, + "step": 4600 + }, + { + "epoch": 2.464626191670848, + "grad_norm": 10.548298161954385, + "learning_rate": 9.374114382176457e-08, + "logits/chosen": -0.13307562470436096, + "logits/rejected": -0.00199460843577981, + "logps/chosen": -1.2904552221298218, + "logps/rejected": -1.6387020349502563, + "loss": 2.0013, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2904552221298218, + "rewards/margins": 0.3482467532157898, + "rewards/rejected": -1.6387020349502563, + "semantic_entropy": 0.7897846698760986, + "step": 4605 + }, + { + "epoch": 2.46730222445225, + "grad_norm": 9.189176275006018, + "learning_rate": 9.283524918896945e-08, + "logits/chosen": -0.14514726400375366, + "logits/rejected": 0.01468361634761095, + "logps/chosen": -1.2731126546859741, + "logps/rejected": -1.6553637981414795, + "loss": 1.952, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -1.2731126546859741, + "rewards/margins": 0.38225096464157104, + "rewards/rejected": -1.6553637981414795, + "semantic_entropy": 0.7842705845832825, + "step": 4610 + }, + { + "epoch": 2.469978257233651, + "grad_norm": 8.431537133463804, + "learning_rate": 9.193330487037232e-08, + "logits/chosen": -0.10504718869924545, + "logits/rejected": 0.026652539148926735, + "logps/chosen": -1.288559913635254, + "logps/rejected": -1.7076818943023682, + "loss": 1.9672, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.288559913635254, + "rewards/margins": 0.4191219210624695, + "rewards/rejected": -1.7076818943023682, + "semantic_entropy": 0.7821754217147827, + "step": 4615 + }, + { + "epoch": 2.4726542900150528, + "grad_norm": 9.264085580424796, + "learning_rate": 9.103531961664118e-08, + "logits/chosen": -0.11797323077917099, + "logits/rejected": 0.07366932928562164, + "logps/chosen": -1.18318772315979, + "logps/rejected": -1.5540214776992798, + "loss": 1.9128, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -1.18318772315979, + "rewards/margins": 0.37083372473716736, + "rewards/rejected": -1.5540214776992798, + "semantic_entropy": 0.8131403923034668, + "step": 4620 + }, + { + "epoch": 2.475330322796454, + "grad_norm": 6.970011750620556, + "learning_rate": 9.014130214003269e-08, + "logits/chosen": -0.24690942466259003, + "logits/rejected": -0.19920854270458221, + "logps/chosen": -1.297821283340454, + "logps/rejected": -1.5923163890838623, + "loss": 1.982, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.297821283340454, + "rewards/margins": 0.2944951057434082, + "rewards/rejected": -1.5923163890838623, + "semantic_entropy": 0.7800196409225464, + "step": 4625 + }, + { + "epoch": 2.4780063555778558, + "grad_norm": 12.967707284344408, + "learning_rate": 8.925126111430848e-08, + "logits/chosen": -0.10687782615423203, + "logits/rejected": -0.025706391781568527, + "logps/chosen": -1.2271325588226318, + "logps/rejected": -1.5873157978057861, + "loss": 1.9681, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2271325588226318, + "rewards/margins": 0.3601832687854767, + "rewards/rejected": -1.5873157978057861, + "semantic_entropy": 0.8013485670089722, + "step": 4630 + }, + { + "epoch": 2.4806823883592575, + "grad_norm": 12.220403329312555, + "learning_rate": 8.83652051746504e-08, + "logits/chosen": -0.052401043474674225, + "logits/rejected": 0.09878160059452057, + "logps/chosen": -1.2342487573623657, + "logps/rejected": -1.657254934310913, + "loss": 1.9672, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.2342487573623657, + "rewards/margins": 0.42300620675086975, + "rewards/rejected": -1.657254934310913, + "semantic_entropy": 0.7962632179260254, + "step": 4635 + }, + { + "epoch": 2.483358421140659, + "grad_norm": 7.803313803458224, + "learning_rate": 8.748314291757696e-08, + "logits/chosen": -0.10491069406270981, + "logits/rejected": -0.0009174048900604248, + "logps/chosen": -1.259453296661377, + "logps/rejected": -1.5711205005645752, + "loss": 1.9676, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.259453296661377, + "rewards/margins": 0.3116670548915863, + "rewards/rejected": -1.5711205005645752, + "semantic_entropy": 0.7823437452316284, + "step": 4640 + }, + { + "epoch": 2.4860344539220605, + "grad_norm": 16.97457748415567, + "learning_rate": 8.660508290086032e-08, + "logits/chosen": -0.12484784424304962, + "logits/rejected": 0.010466617532074451, + "logps/chosen": -1.2484409809112549, + "logps/rejected": -1.5821198225021362, + "loss": 1.969, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.2484409809112549, + "rewards/margins": 0.3336789011955261, + "rewards/rejected": -1.5821198225021362, + "semantic_entropy": 0.7921417951583862, + "step": 4645 + }, + { + "epoch": 2.488710486703462, + "grad_norm": 13.315787077562575, + "learning_rate": 8.573103364344231e-08, + "logits/chosen": -0.17663438618183136, + "logits/rejected": 0.06814370304346085, + "logps/chosen": -1.268634557723999, + "logps/rejected": -1.6621404886245728, + "loss": 1.9485, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.268634557723999, + "rewards/margins": 0.3935058116912842, + "rewards/rejected": -1.6621404886245728, + "semantic_entropy": 0.7878329753875732, + "step": 4650 + }, + { + "epoch": 2.4913865194848634, + "grad_norm": 9.25210617668145, + "learning_rate": 8.486100362535292e-08, + "logits/chosen": -0.15664373338222504, + "logits/rejected": -0.006619095802307129, + "logps/chosen": -1.241071343421936, + "logps/rejected": -1.499688744544983, + "loss": 1.9827, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.241071343421936, + "rewards/margins": 0.25861743092536926, + "rewards/rejected": -1.499688744544983, + "semantic_entropy": 0.8064199686050415, + "step": 4655 + }, + { + "epoch": 2.494062552266265, + "grad_norm": 7.998057347357029, + "learning_rate": 8.399500128762693e-08, + "logits/chosen": -0.1831817924976349, + "logits/rejected": -0.06365980952978134, + "logps/chosen": -1.3237661123275757, + "logps/rejected": -1.558573603630066, + "loss": 2.0178, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.3237661123275757, + "rewards/margins": 0.23480753600597382, + "rewards/rejected": -1.558573603630066, + "semantic_entropy": 0.771876335144043, + "step": 4660 + }, + { + "epoch": 2.496738585047667, + "grad_norm": 7.958903616032643, + "learning_rate": 8.313303503222313e-08, + "logits/chosen": -0.18261002004146576, + "logits/rejected": -0.12167014926671982, + "logps/chosen": -1.3508309125900269, + "logps/rejected": -1.6414368152618408, + "loss": 2.0504, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.3508309125900269, + "rewards/margins": 0.2906058430671692, + "rewards/rejected": -1.6414368152618408, + "semantic_entropy": 0.7719950675964355, + "step": 4665 + }, + { + "epoch": 2.4994146178290686, + "grad_norm": 11.210113296265586, + "learning_rate": 8.227511322194164e-08, + "logits/chosen": -0.16465625166893005, + "logits/rejected": -0.0402381494641304, + "logps/chosen": -1.3120533227920532, + "logps/rejected": -1.5447263717651367, + "loss": 2.016, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.3120533227920532, + "rewards/margins": 0.23267300426959991, + "rewards/rejected": -1.5447263717651367, + "semantic_entropy": 0.7820819616317749, + "step": 4670 + }, + { + "epoch": 2.50209065061047, + "grad_norm": 7.5048010774702005, + "learning_rate": 8.142124418034385e-08, + "logits/chosen": -0.11845330893993378, + "logits/rejected": 0.023418676108121872, + "logps/chosen": -1.2196156978607178, + "logps/rejected": -1.5629394054412842, + "loss": 1.9521, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2196156978607178, + "rewards/margins": 0.34332379698753357, + "rewards/rejected": -1.5629394054412842, + "semantic_entropy": 0.8040634989738464, + "step": 4675 + }, + { + "epoch": 2.5047666833918716, + "grad_norm": 8.96692595819377, + "learning_rate": 8.057143619167073e-08, + "logits/chosen": -0.12013927847146988, + "logits/rejected": -0.011678531765937805, + "logps/chosen": -1.2727725505828857, + "logps/rejected": -1.6061283349990845, + "loss": 1.9921, + "rewards/accuracies": 0.65625, + "rewards/chosen": -1.2727725505828857, + "rewards/margins": 0.3333559036254883, + "rewards/rejected": -1.6061283349990845, + "semantic_entropy": 0.7913545966148376, + "step": 4680 + }, + { + "epoch": 2.507442716173273, + "grad_norm": 7.332356348791178, + "learning_rate": 7.97256975007633e-08, + "logits/chosen": -0.191193088889122, + "logits/rejected": 0.005521965213119984, + "logps/chosen": -1.2749216556549072, + "logps/rejected": -1.5832358598709106, + "loss": 1.9818, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2749216556549072, + "rewards/margins": 0.30831417441368103, + "rewards/rejected": -1.5832358598709106, + "semantic_entropy": 0.7828585505485535, + "step": 4685 + }, + { + "epoch": 2.5101187489546746, + "grad_norm": 10.237642721143716, + "learning_rate": 7.888403631298186e-08, + "logits/chosen": -0.11178383976221085, + "logits/rejected": -0.070093534886837, + "logps/chosen": -1.287856936454773, + "logps/rejected": -1.567197561264038, + "loss": 2.0032, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.287856936454773, + "rewards/margins": 0.27934056520462036, + "rewards/rejected": -1.567197561264038, + "semantic_entropy": 0.7800172567367554, + "step": 4690 + }, + { + "epoch": 2.5127947817360763, + "grad_norm": 8.237172823392987, + "learning_rate": 7.804646079412719e-08, + "logits/chosen": -0.1218346506357193, + "logits/rejected": 0.06685183197259903, + "logps/chosen": -1.2876611948013306, + "logps/rejected": -1.5944554805755615, + "loss": 1.9876, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2876611948013306, + "rewards/margins": 0.30679452419281006, + "rewards/rejected": -1.5944554805755615, + "semantic_entropy": 0.7881008982658386, + "step": 4695 + }, + { + "epoch": 2.515470814517478, + "grad_norm": 7.592310399773866, + "learning_rate": 7.72129790703604e-08, + "logits/chosen": -0.2171975076198578, + "logits/rejected": -0.0944719910621643, + "logps/chosen": -1.2436479330062866, + "logps/rejected": -1.553193211555481, + "loss": 1.9712, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2436479330062866, + "rewards/margins": 0.30954509973526, + "rewards/rejected": -1.553193211555481, + "semantic_entropy": 0.8062686920166016, + "step": 4700 + }, + { + "epoch": 2.5181468472988793, + "grad_norm": 10.166467740095245, + "learning_rate": 7.638359922812504e-08, + "logits/chosen": -0.08607305586338043, + "logits/rejected": -0.05110453441739082, + "logps/chosen": -1.3230863809585571, + "logps/rejected": -1.6123307943344116, + "loss": 2.0252, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.3230863809585571, + "rewards/margins": 0.28924453258514404, + "rewards/rejected": -1.6123307943344116, + "semantic_entropy": 0.7647134065628052, + "step": 4705 + }, + { + "epoch": 2.520822880080281, + "grad_norm": 11.52393554481213, + "learning_rate": 7.555832931406774e-08, + "logits/chosen": -0.182388037443161, + "logits/rejected": -0.031195128336548805, + "logps/chosen": -1.2906062602996826, + "logps/rejected": -1.5945990085601807, + "loss": 1.9873, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.2906062602996826, + "rewards/margins": 0.3039928078651428, + "rewards/rejected": -1.5945990085601807, + "semantic_entropy": 0.7708030939102173, + "step": 4710 + }, + { + "epoch": 2.5234989128616827, + "grad_norm": 7.291958045293431, + "learning_rate": 7.47371773349611e-08, + "logits/chosen": -0.17179132997989655, + "logits/rejected": -0.13924379646778107, + "logps/chosen": -1.347267508506775, + "logps/rejected": -1.642747163772583, + "loss": 2.0267, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.347267508506775, + "rewards/margins": 0.2954796850681305, + "rewards/rejected": -1.642747163772583, + "semantic_entropy": 0.7599186897277832, + "step": 4715 + }, + { + "epoch": 2.526174945643084, + "grad_norm": 11.405677460777579, + "learning_rate": 7.392015125762496e-08, + "logits/chosen": -0.14520487189292908, + "logits/rejected": -0.04902344197034836, + "logps/chosen": -1.197825312614441, + "logps/rejected": -1.5305838584899902, + "loss": 1.9289, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.197825312614441, + "rewards/margins": 0.33275845646858215, + "rewards/rejected": -1.5305838584899902, + "semantic_entropy": 0.8214031457901001, + "step": 4720 + }, + { + "epoch": 2.5288509784244857, + "grad_norm": 9.447896137190856, + "learning_rate": 7.310725900885018e-08, + "logits/chosen": -0.20286524295806885, + "logits/rejected": -0.1489933729171753, + "logps/chosen": -1.2946898937225342, + "logps/rejected": -1.5375152826309204, + "loss": 2.0094, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2946898937225342, + "rewards/margins": 0.2428254783153534, + "rewards/rejected": -1.5375152826309204, + "semantic_entropy": 0.7823761105537415, + "step": 4725 + }, + { + "epoch": 2.5315270112058874, + "grad_norm": 9.610255064005486, + "learning_rate": 7.229850847532076e-08, + "logits/chosen": -0.13549140095710754, + "logits/rejected": -0.020894240587949753, + "logps/chosen": -1.183825969696045, + "logps/rejected": -1.5215822458267212, + "loss": 1.9285, + "rewards/accuracies": 0.65625, + "rewards/chosen": -1.183825969696045, + "rewards/margins": 0.33775612711906433, + "rewards/rejected": -1.5215822458267212, + "semantic_entropy": 0.8025429844856262, + "step": 4730 + }, + { + "epoch": 2.5342030439872887, + "grad_norm": 8.168702490510011, + "learning_rate": 7.149390750353779e-08, + "logits/chosen": -0.12599562108516693, + "logits/rejected": -0.18436439335346222, + "logps/chosen": -1.3078311681747437, + "logps/rejected": -1.533595085144043, + "loss": 2.0131, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.3078311681747437, + "rewards/margins": 0.22576388716697693, + "rewards/rejected": -1.533595085144043, + "semantic_entropy": 0.7823559641838074, + "step": 4735 + }, + { + "epoch": 2.5368790767686904, + "grad_norm": 8.254126468029884, + "learning_rate": 7.069346389974374e-08, + "logits/chosen": -0.20458774268627167, + "logits/rejected": -0.06394211202859879, + "logps/chosen": -1.3023638725280762, + "logps/rejected": -1.6589126586914062, + "loss": 2.0008, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.3023638725280762, + "rewards/margins": 0.35654881596565247, + "rewards/rejected": -1.6589126586914062, + "semantic_entropy": 0.7820869088172913, + "step": 4740 + }, + { + "epoch": 2.539555109550092, + "grad_norm": 8.474729875307984, + "learning_rate": 6.989718542984563e-08, + "logits/chosen": -0.14035843312740326, + "logits/rejected": -0.09613653272390366, + "logps/chosen": -1.3093879222869873, + "logps/rejected": -1.60835862159729, + "loss": 1.9904, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.3093879222869873, + "rewards/margins": 0.29897063970565796, + "rewards/rejected": -1.60835862159729, + "semantic_entropy": 0.7700130343437195, + "step": 4745 + }, + { + "epoch": 2.5422311423314934, + "grad_norm": 7.371448992180354, + "learning_rate": 6.9105079819341e-08, + "logits/chosen": -0.10261497646570206, + "logits/rejected": 0.09542595595121384, + "logps/chosen": -1.2490769624710083, + "logps/rejected": -1.7567031383514404, + "loss": 1.9241, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -1.2490769624710083, + "rewards/margins": 0.5076262950897217, + "rewards/rejected": -1.7567031383514404, + "semantic_entropy": 0.7950537800788879, + "step": 4750 + }, + { + "epoch": 2.544907175112895, + "grad_norm": 6.68530095678338, + "learning_rate": 6.831715475324163e-08, + "logits/chosen": -0.18308278918266296, + "logits/rejected": -0.01879027858376503, + "logps/chosen": -1.1776317358016968, + "logps/rejected": -1.644796371459961, + "loss": 1.9031, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.1776317358016968, + "rewards/margins": 0.46716445684432983, + "rewards/rejected": -1.644796371459961, + "semantic_entropy": 0.8084004521369934, + "step": 4755 + }, + { + "epoch": 2.547583207894297, + "grad_norm": 7.494571745163121, + "learning_rate": 6.753341787600026e-08, + "logits/chosen": -0.2286044806241989, + "logits/rejected": -0.09256772696971893, + "logps/chosen": -1.1995595693588257, + "logps/rejected": -1.5113732814788818, + "loss": 1.951, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.1995595693588257, + "rewards/margins": 0.31181374192237854, + "rewards/rejected": -1.5113732814788818, + "semantic_entropy": 0.8131352663040161, + "step": 4760 + }, + { + "epoch": 2.5502592406756985, + "grad_norm": 10.150397112366806, + "learning_rate": 6.67538767914353e-08, + "logits/chosen": -0.24224725365638733, + "logits/rejected": -0.06934880465269089, + "logps/chosen": -1.3018126487731934, + "logps/rejected": -1.5710437297821045, + "loss": 2.0319, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.3018126487731934, + "rewards/margins": 0.2692311406135559, + "rewards/rejected": -1.5710437297821045, + "semantic_entropy": 0.7827295064926147, + "step": 4765 + }, + { + "epoch": 2.5529352734571, + "grad_norm": 9.013106585065046, + "learning_rate": 6.597853906265793e-08, + "logits/chosen": -0.18994879722595215, + "logits/rejected": -0.08487293869256973, + "logps/chosen": -1.292405605316162, + "logps/rejected": -1.6529300212860107, + "loss": 1.9845, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.292405605316162, + "rewards/margins": 0.3605244755744934, + "rewards/rejected": -1.6529300212860107, + "semantic_entropy": 0.7887686491012573, + "step": 4770 + }, + { + "epoch": 2.5556113062385015, + "grad_norm": 6.103315524429753, + "learning_rate": 6.5207412211998e-08, + "logits/chosen": -0.05231186002492905, + "logits/rejected": 0.053858887404203415, + "logps/chosen": -1.2260487079620361, + "logps/rejected": -1.6088154315948486, + "loss": 1.9389, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.2260487079620361, + "rewards/margins": 0.3827666640281677, + "rewards/rejected": -1.6088154315948486, + "semantic_entropy": 0.8044970631599426, + "step": 4775 + }, + { + "epoch": 2.558287339019903, + "grad_norm": 6.102321488627527, + "learning_rate": 6.444050372093186e-08, + "logits/chosen": -0.13956710696220398, + "logits/rejected": -0.05834800750017166, + "logps/chosen": -1.2656276226043701, + "logps/rejected": -1.5484168529510498, + "loss": 1.9897, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.2656276226043701, + "rewards/margins": 0.28278931975364685, + "rewards/rejected": -1.5484168529510498, + "semantic_entropy": 0.7972785234451294, + "step": 4780 + }, + { + "epoch": 2.5609633718013045, + "grad_norm": 11.168837796629282, + "learning_rate": 6.367782103000873e-08, + "logits/chosen": -0.1260305792093277, + "logits/rejected": -0.06556358188390732, + "logps/chosen": -1.2994545698165894, + "logps/rejected": -1.5403183698654175, + "loss": 2.0189, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.2994545698165894, + "rewards/margins": 0.2408638894557953, + "rewards/rejected": -1.5403183698654175, + "semantic_entropy": 0.7918559312820435, + "step": 4785 + }, + { + "epoch": 2.5636394045827062, + "grad_norm": 6.517739089204047, + "learning_rate": 6.29193715387798e-08, + "logits/chosen": -0.23115773499011993, + "logits/rejected": -0.09202119708061218, + "logps/chosen": -1.2879234552383423, + "logps/rejected": -1.6790473461151123, + "loss": 1.9696, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2879234552383423, + "rewards/margins": 0.3911239504814148, + "rewards/rejected": -1.6790473461151123, + "semantic_entropy": 0.7856905460357666, + "step": 4790 + }, + { + "epoch": 2.566315437364108, + "grad_norm": 13.27204614413823, + "learning_rate": 6.216516260572502e-08, + "logits/chosen": -0.09322036802768707, + "logits/rejected": -0.020159423351287842, + "logps/chosen": -1.3097180128097534, + "logps/rejected": -1.6087557077407837, + "loss": 2.0216, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.3097180128097534, + "rewards/margins": 0.29903754591941833, + "rewards/rejected": -1.6087557077407837, + "semantic_entropy": 0.7815183401107788, + "step": 4795 + }, + { + "epoch": 2.568991470145509, + "grad_norm": 11.004880769709997, + "learning_rate": 6.141520154818297e-08, + "logits/chosen": -0.14857342839241028, + "logits/rejected": -0.031513821333646774, + "logps/chosen": -1.218299150466919, + "logps/rejected": -1.5542099475860596, + "loss": 1.9434, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.218299150466919, + "rewards/margins": 0.33591070771217346, + "rewards/rejected": -1.5542099475860596, + "semantic_entropy": 0.7982202172279358, + "step": 4800 + }, + { + "epoch": 2.568991470145509, + "eval_logits/chosen": 0.17427968978881836, + "eval_logits/rejected": 0.2611408829689026, + "eval_logps/chosen": -1.3386183977127075, + "eval_logps/rejected": -1.6198391914367676, + "eval_loss": 2.0458827018737793, + "eval_rewards/accuracies": 0.5942136645317078, + "eval_rewards/chosen": -1.3386183977127075, + "eval_rewards/margins": 0.2812207043170929, + "eval_rewards/rejected": -1.6198391914367676, + "eval_runtime": 34.75, + "eval_samples_per_second": 38.705, + "eval_semantic_entropy": 0.7744117975234985, + "eval_steps_per_second": 9.698, + "step": 4800 + }, + { + "epoch": 2.571667502926911, + "grad_norm": 11.503730963858805, + "learning_rate": 6.066949564227897e-08, + "logits/chosen": -0.224358469247818, + "logits/rejected": -0.10400880873203278, + "logps/chosen": -1.3242347240447998, + "logps/rejected": -1.7001020908355713, + "loss": 2.0023, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.3242347240447998, + "rewards/margins": 0.3758672773838043, + "rewards/rejected": -1.7001020908355713, + "semantic_entropy": 0.7753196358680725, + "step": 4805 + }, + { + "epoch": 2.574343535708312, + "grad_norm": 9.11957728544841, + "learning_rate": 5.992805212285523e-08, + "logits/chosen": -0.090094193816185, + "logits/rejected": 0.04532430320978165, + "logps/chosen": -1.3409744501113892, + "logps/rejected": -1.6526081562042236, + "loss": 2.0314, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.3409744501113892, + "rewards/margins": 0.31163376569747925, + "rewards/rejected": -1.6526081562042236, + "semantic_entropy": 0.7715145349502563, + "step": 4810 + }, + { + "epoch": 2.577019568489714, + "grad_norm": 11.01571030763835, + "learning_rate": 5.9190878183399684e-08, + "logits/chosen": -0.09173809736967087, + "logits/rejected": 0.03900834172964096, + "logps/chosen": -1.1757429838180542, + "logps/rejected": -1.646345853805542, + "loss": 1.8842, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.1757429838180542, + "rewards/margins": 0.47060298919677734, + "rewards/rejected": -1.646345853805542, + "semantic_entropy": 0.8084053993225098, + "step": 4815 + }, + { + "epoch": 2.5796956012711156, + "grad_norm": 9.665148218805912, + "learning_rate": 5.845798097597748e-08, + "logits/chosen": -0.0845046415925026, + "logits/rejected": -0.007058621849864721, + "logps/chosen": -1.3429194688796997, + "logps/rejected": -1.614829421043396, + "loss": 2.023, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.3429194688796997, + "rewards/margins": 0.2719099223613739, + "rewards/rejected": -1.614829421043396, + "semantic_entropy": 0.7687947154045105, + "step": 4820 + }, + { + "epoch": 2.5823716340525174, + "grad_norm": 15.917517242871915, + "learning_rate": 5.772936761116026e-08, + "logits/chosen": -0.07637304067611694, + "logits/rejected": 0.04699677973985672, + "logps/chosen": -1.242073655128479, + "logps/rejected": -1.5095151662826538, + "loss": 1.9891, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.242073655128479, + "rewards/margins": 0.26744160056114197, + "rewards/rejected": -1.5095151662826538, + "semantic_entropy": 0.8077836036682129, + "step": 4825 + }, + { + "epoch": 2.5850476668339186, + "grad_norm": 5.713470101292522, + "learning_rate": 5.700504515795829e-08, + "logits/chosen": -0.14389801025390625, + "logits/rejected": 0.005694887135177851, + "logps/chosen": -1.3192150592803955, + "logps/rejected": -1.5601211786270142, + "loss": 2.0238, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.3192150592803955, + "rewards/margins": 0.24090604484081268, + "rewards/rejected": -1.5601211786270142, + "semantic_entropy": 0.7768298387527466, + "step": 4830 + }, + { + "epoch": 2.5877236996153203, + "grad_norm": 11.826039372971426, + "learning_rate": 5.628502064375101e-08, + "logits/chosen": -0.24123668670654297, + "logits/rejected": -0.05609578639268875, + "logps/chosen": -1.264898657798767, + "logps/rejected": -1.575839638710022, + "loss": 1.9891, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.264898657798767, + "rewards/margins": 0.31094080209732056, + "rewards/rejected": -1.575839638710022, + "semantic_entropy": 0.8016520738601685, + "step": 4835 + }, + { + "epoch": 2.5903997323967216, + "grad_norm": 22.226769592188973, + "learning_rate": 5.55693010542197e-08, + "logits/chosen": -0.1912364810705185, + "logits/rejected": 0.011147690005600452, + "logps/chosen": -1.2765791416168213, + "logps/rejected": -1.5787981748580933, + "loss": 1.9914, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.2765791416168213, + "rewards/margins": 0.30221912264823914, + "rewards/rejected": -1.5787981748580933, + "semantic_entropy": 0.7809934020042419, + "step": 4840 + }, + { + "epoch": 2.5930757651781233, + "grad_norm": 7.512142408229105, + "learning_rate": 5.485789333327856e-08, + "logits/chosen": -0.14734336733818054, + "logits/rejected": -0.035708121955394745, + "logps/chosen": -1.252044439315796, + "logps/rejected": -1.5797019004821777, + "loss": 1.9539, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.252044439315796, + "rewards/margins": 0.3276575803756714, + "rewards/rejected": -1.5797019004821777, + "semantic_entropy": 0.8009160161018372, + "step": 4845 + }, + { + "epoch": 2.595751797959525, + "grad_norm": 12.7462575378253, + "learning_rate": 5.4150804383008675e-08, + "logits/chosen": -0.26015961170196533, + "logits/rejected": -0.09845755994319916, + "logps/chosen": -1.2424649000167847, + "logps/rejected": -1.6106786727905273, + "loss": 1.9626, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.2424649000167847, + "rewards/margins": 0.3682136833667755, + "rewards/rejected": -1.6106786727905273, + "semantic_entropy": 0.8059800863265991, + "step": 4850 + }, + { + "epoch": 2.5984278307409268, + "grad_norm": 18.122164837107288, + "learning_rate": 5.344804106359002e-08, + "logits/chosen": -0.09411051124334335, + "logits/rejected": 0.06524305045604706, + "logps/chosen": -1.2202339172363281, + "logps/rejected": -1.5384161472320557, + "loss": 1.9856, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2202339172363281, + "rewards/margins": 0.31818222999572754, + "rewards/rejected": -1.5384161472320557, + "semantic_entropy": 0.805999755859375, + "step": 4855 + }, + { + "epoch": 2.601103863522328, + "grad_norm": 12.956987300274212, + "learning_rate": 5.274961019323559e-08, + "logits/chosen": -0.13933970034122467, + "logits/rejected": -0.07091189920902252, + "logps/chosen": -1.1746845245361328, + "logps/rejected": -1.6169769763946533, + "loss": 1.8937, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": -1.1746845245361328, + "rewards/margins": 0.4422924518585205, + "rewards/rejected": -1.6169769763946533, + "semantic_entropy": 0.8040294647216797, + "step": 4860 + }, + { + "epoch": 2.6037798963037297, + "grad_norm": 9.04594431262119, + "learning_rate": 5.205551854812451e-08, + "logits/chosen": -0.23502866923809052, + "logits/rejected": -0.14637461304664612, + "logps/chosen": -1.2417190074920654, + "logps/rejected": -1.627515435218811, + "loss": 1.9502, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.2417190074920654, + "rewards/margins": 0.38579636812210083, + "rewards/rejected": -1.627515435218811, + "semantic_entropy": 0.7967454791069031, + "step": 4865 + }, + { + "epoch": 2.606455929085131, + "grad_norm": 12.300434612266919, + "learning_rate": 5.1365772862337177e-08, + "logits/chosen": -0.09480637311935425, + "logits/rejected": 0.03487342968583107, + "logps/chosen": -1.2769651412963867, + "logps/rejected": -1.583880066871643, + "loss": 2.0036, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2769651412963867, + "rewards/margins": 0.3069148361682892, + "rewards/rejected": -1.583880066871643, + "semantic_entropy": 0.7944985628128052, + "step": 4870 + }, + { + "epoch": 2.6091319618665327, + "grad_norm": 8.881988481533577, + "learning_rate": 5.068037982778905e-08, + "logits/chosen": 0.027025306597352028, + "logits/rejected": 0.09920650720596313, + "logps/chosen": -1.240953803062439, + "logps/rejected": -1.6911252737045288, + "loss": 1.9596, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.240953803062439, + "rewards/margins": 0.45017147064208984, + "rewards/rejected": -1.6911252737045288, + "semantic_entropy": 0.7965016961097717, + "step": 4875 + }, + { + "epoch": 2.6118079946479344, + "grad_norm": 9.975007837075847, + "learning_rate": 4.999934609416656e-08, + "logits/chosen": -0.05672222375869751, + "logits/rejected": 0.05705530196428299, + "logps/chosen": -1.2177865505218506, + "logps/rejected": -1.640758752822876, + "loss": 1.9217, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2177865505218506, + "rewards/margins": 0.4229722023010254, + "rewards/rejected": -1.640758752822876, + "semantic_entropy": 0.7946897745132446, + "step": 4880 + }, + { + "epoch": 2.614484027429336, + "grad_norm": 10.150182185057119, + "learning_rate": 4.932267826886183e-08, + "logits/chosen": -0.03622272610664368, + "logits/rejected": 0.041126228868961334, + "logps/chosen": -1.3333375453948975, + "logps/rejected": -1.683574914932251, + "loss": 2.0232, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.3333375453948975, + "rewards/margins": 0.35023751854896545, + "rewards/rejected": -1.683574914932251, + "semantic_entropy": 0.7540980577468872, + "step": 4885 + }, + { + "epoch": 2.6171600602107374, + "grad_norm": 10.730980754841438, + "learning_rate": 4.8650382916909206e-08, + "logits/chosen": -0.2052614986896515, + "logits/rejected": -0.034829698503017426, + "logps/chosen": -1.263850212097168, + "logps/rejected": -1.6367954015731812, + "loss": 1.9642, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.263850212097168, + "rewards/margins": 0.37294501066207886, + "rewards/rejected": -1.6367954015731812, + "semantic_entropy": 0.7861441373825073, + "step": 4890 + }, + { + "epoch": 2.619836092992139, + "grad_norm": 7.368577885518247, + "learning_rate": 4.7982466560920976e-08, + "logits/chosen": -0.1294822245836258, + "logits/rejected": -0.03858218714594841, + "logps/chosen": -1.3017622232437134, + "logps/rejected": -1.5246281623840332, + "loss": 2.023, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.3017622232437134, + "rewards/margins": 0.22286593914031982, + "rewards/rejected": -1.5246281623840332, + "semantic_entropy": 0.7833074331283569, + "step": 4895 + }, + { + "epoch": 2.622512125773541, + "grad_norm": 5.928285248215213, + "learning_rate": 4.7318935681024685e-08, + "logits/chosen": -0.08297251164913177, + "logits/rejected": 0.05140919238328934, + "logps/chosen": -1.244057536125183, + "logps/rejected": -1.6229749917984009, + "loss": 1.9445, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.244057536125183, + "rewards/margins": 0.378917396068573, + "rewards/rejected": -1.6229749917984009, + "semantic_entropy": 0.7902035117149353, + "step": 4900 + }, + { + "epoch": 2.625188158554942, + "grad_norm": 6.2002842955489115, + "learning_rate": 4.6659796714799745e-08, + "logits/chosen": -0.11116425693035126, + "logits/rejected": 0.042223114520311356, + "logps/chosen": -1.2663477659225464, + "logps/rejected": -1.6212007999420166, + "loss": 1.9768, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.2663477659225464, + "rewards/margins": 0.3548528552055359, + "rewards/rejected": -1.6212007999420166, + "semantic_entropy": 0.7857402563095093, + "step": 4905 + }, + { + "epoch": 2.627864191336344, + "grad_norm": 8.281462378102345, + "learning_rate": 4.60050560572155e-08, + "logits/chosen": -0.14414629340171814, + "logits/rejected": -0.16859321296215057, + "logps/chosen": -1.2700074911117554, + "logps/rejected": -1.735364317893982, + "loss": 1.9645, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.2700074911117554, + "rewards/margins": 0.4653567373752594, + "rewards/rejected": -1.735364317893982, + "semantic_entropy": 0.789490818977356, + "step": 4910 + }, + { + "epoch": 2.6305402241177456, + "grad_norm": 9.741518580503993, + "learning_rate": 4.535472006056834e-08, + "logits/chosen": -0.06438411772251129, + "logits/rejected": 0.03645631670951843, + "logps/chosen": -1.1731632947921753, + "logps/rejected": -1.563751220703125, + "loss": 1.897, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.1731632947921753, + "rewards/margins": 0.3905879557132721, + "rewards/rejected": -1.563751220703125, + "semantic_entropy": 0.8115876317024231, + "step": 4915 + }, + { + "epoch": 2.6332162568991473, + "grad_norm": 9.399338449222602, + "learning_rate": 4.470879503442132e-08, + "logits/chosen": -0.06840424239635468, + "logits/rejected": 0.012408537790179253, + "logps/chosen": -1.2594525814056396, + "logps/rejected": -1.536690354347229, + "loss": 2.0095, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.2594525814056396, + "rewards/margins": 0.2772377133369446, + "rewards/rejected": -1.536690354347229, + "semantic_entropy": 0.8033695220947266, + "step": 4920 + }, + { + "epoch": 2.6358922896805486, + "grad_norm": 11.481991942896826, + "learning_rate": 4.406728724554154e-08, + "logits/chosen": -0.3004254996776581, + "logits/rejected": -0.050283849239349365, + "logps/chosen": -1.2204532623291016, + "logps/rejected": -1.577864646911621, + "loss": 1.9369, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.2204532623291016, + "rewards/margins": 0.3574114441871643, + "rewards/rejected": -1.577864646911621, + "semantic_entropy": 0.7966974973678589, + "step": 4925 + }, + { + "epoch": 2.6385683224619503, + "grad_norm": 6.512321673294674, + "learning_rate": 4.3430202917840664e-08, + "logits/chosen": -0.09682625532150269, + "logits/rejected": 0.04881001263856888, + "logps/chosen": -1.3104798793792725, + "logps/rejected": -1.7430320978164673, + "loss": 1.9896, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.3104798793792725, + "rewards/margins": 0.4325522482395172, + "rewards/rejected": -1.7430320978164673, + "semantic_entropy": 0.7690348625183105, + "step": 4930 + }, + { + "epoch": 2.6412443552433515, + "grad_norm": 16.111192810330614, + "learning_rate": 4.279754823231346e-08, + "logits/chosen": -0.21507367491722107, + "logits/rejected": -0.06278637796640396, + "logps/chosen": -1.2675609588623047, + "logps/rejected": -1.5637404918670654, + "loss": 1.9877, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.2675609588623047, + "rewards/margins": 0.2961795926094055, + "rewards/rejected": -1.5637404918670654, + "semantic_entropy": 0.7901414632797241, + "step": 4935 + }, + { + "epoch": 2.6439203880247533, + "grad_norm": 6.774101283588008, + "learning_rate": 4.216932932697859e-08, + "logits/chosen": -0.13060271739959717, + "logits/rejected": -0.05203498527407646, + "logps/chosen": -1.2672488689422607, + "logps/rejected": -1.4680770635604858, + "loss": 1.9994, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.2672488689422607, + "rewards/margins": 0.2008281946182251, + "rewards/rejected": -1.4680770635604858, + "semantic_entropy": 0.8053982853889465, + "step": 4940 + }, + { + "epoch": 2.646596420806155, + "grad_norm": 8.441149104353624, + "learning_rate": 4.154555229681844e-08, + "logits/chosen": -0.15102128684520721, + "logits/rejected": 0.04940281808376312, + "logps/chosen": -1.2585450410842896, + "logps/rejected": -1.7040252685546875, + "loss": 1.9433, + "rewards/accuracies": 0.65625, + "rewards/chosen": -1.2585450410842896, + "rewards/margins": 0.4454802870750427, + "rewards/rejected": -1.7040252685546875, + "semantic_entropy": 0.7860039472579956, + "step": 4945 + }, + { + "epoch": 2.6492724535875567, + "grad_norm": 14.170996265703746, + "learning_rate": 4.092622319372069e-08, + "logits/chosen": -0.15491417050361633, + "logits/rejected": -0.032993149012327194, + "logps/chosen": -1.2248013019561768, + "logps/rejected": -1.553957462310791, + "loss": 1.9588, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2248013019561768, + "rewards/margins": 0.3291561007499695, + "rewards/rejected": -1.553957462310791, + "semantic_entropy": 0.7976547479629517, + "step": 4950 + }, + { + "epoch": 2.651948486368958, + "grad_norm": 12.696191026034306, + "learning_rate": 4.031134802641889e-08, + "logits/chosen": -0.1630629301071167, + "logits/rejected": -0.1627640724182129, + "logps/chosen": -1.2603944540023804, + "logps/rejected": -1.6164932250976562, + "loss": 1.9535, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.2603944540023804, + "rewards/margins": 0.3560987114906311, + "rewards/rejected": -1.6164932250976562, + "semantic_entropy": 0.7857304215431213, + "step": 4955 + }, + { + "epoch": 2.6546245191503597, + "grad_norm": 7.404253383249965, + "learning_rate": 3.970093276043468e-08, + "logits/chosen": -0.03796197846531868, + "logits/rejected": 0.059105951339006424, + "logps/chosen": -1.2659322023391724, + "logps/rejected": -1.5594804286956787, + "loss": 1.9762, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.2659322023391724, + "rewards/margins": 0.29354843497276306, + "rewards/rejected": -1.5594804286956787, + "semantic_entropy": 0.7827733755111694, + "step": 4960 + }, + { + "epoch": 2.657300551931761, + "grad_norm": 9.56732882815061, + "learning_rate": 3.9094983318019584e-08, + "logits/chosen": -0.18269088864326477, + "logits/rejected": -0.05542262643575668, + "logps/chosen": -1.1641151905059814, + "logps/rejected": -1.533266544342041, + "loss": 1.8922, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.1641151905059814, + "rewards/margins": 0.3691512942314148, + "rewards/rejected": -1.533266544342041, + "semantic_entropy": 0.8211628794670105, + "step": 4965 + }, + { + "epoch": 2.6599765847131627, + "grad_norm": 13.076802645156093, + "learning_rate": 3.849350557809789e-08, + "logits/chosen": -0.0741979256272316, + "logits/rejected": -0.005497545935213566, + "logps/chosen": -1.2310014963150024, + "logps/rejected": -1.601223349571228, + "loss": 1.9484, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": -1.2310014963150024, + "rewards/margins": 0.37022197246551514, + "rewards/rejected": -1.601223349571228, + "semantic_entropy": 0.8146511316299438, + "step": 4970 + }, + { + "epoch": 2.6626526174945644, + "grad_norm": 9.43279246779188, + "learning_rate": 3.789650537620903e-08, + "logits/chosen": -0.10716526210308075, + "logits/rejected": -0.0495462603867054, + "logps/chosen": -1.304774522781372, + "logps/rejected": -1.5925801992416382, + "loss": 2.0143, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.304774522781372, + "rewards/margins": 0.2878056764602661, + "rewards/rejected": -1.5925801992416382, + "semantic_entropy": 0.7799090147018433, + "step": 4975 + }, + { + "epoch": 2.665328650275966, + "grad_norm": 10.763393837295931, + "learning_rate": 3.730398850445182e-08, + "logits/chosen": -0.025856634601950645, + "logits/rejected": 0.041945237666368484, + "logps/chosen": -1.3856089115142822, + "logps/rejected": -1.7057222127914429, + "loss": 2.0548, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.3856089115142822, + "rewards/margins": 0.3201134204864502, + "rewards/rejected": -1.7057222127914429, + "semantic_entropy": 0.7589296102523804, + "step": 4980 + }, + { + "epoch": 2.6680046830573674, + "grad_norm": 13.416545083305897, + "learning_rate": 3.671596071142735e-08, + "logits/chosen": -0.11143984645605087, + "logits/rejected": 0.05205560848116875, + "logps/chosen": -1.2196766138076782, + "logps/rejected": -1.6010757684707642, + "loss": 1.9485, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2196766138076782, + "rewards/margins": 0.3813990652561188, + "rewards/rejected": -1.6010757684707642, + "semantic_entropy": 0.7851622700691223, + "step": 4985 + }, + { + "epoch": 2.670680715838769, + "grad_norm": 15.420414884524414, + "learning_rate": 3.6132427702183996e-08, + "logits/chosen": -0.2223101556301117, + "logits/rejected": -0.00972900353372097, + "logps/chosen": -1.2277615070343018, + "logps/rejected": -1.6345646381378174, + "loss": 1.9466, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.2277615070343018, + "rewards/margins": 0.4068028926849365, + "rewards/rejected": -1.6345646381378174, + "semantic_entropy": 0.8061507940292358, + "step": 4990 + }, + { + "epoch": 2.6733567486201704, + "grad_norm": 10.77387194862738, + "learning_rate": 3.555339513816147e-08, + "logits/chosen": -0.16990795731544495, + "logits/rejected": -0.15832272171974182, + "logps/chosen": -1.279496431350708, + "logps/rejected": -1.5370452404022217, + "loss": 2.0197, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.279496431350708, + "rewards/margins": 0.25754863023757935, + "rewards/rejected": -1.5370452404022217, + "semantic_entropy": 0.7910841703414917, + "step": 4995 + }, + { + "epoch": 2.676032781401572, + "grad_norm": 8.193883232131919, + "learning_rate": 3.497886863713639e-08, + "logits/chosen": -0.1418825089931488, + "logits/rejected": -0.11908824741840363, + "logps/chosen": -1.2753219604492188, + "logps/rejected": -1.6523866653442383, + "loss": 1.9877, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2753219604492188, + "rewards/margins": 0.3770645260810852, + "rewards/rejected": -1.6523866653442383, + "semantic_entropy": 0.7843011617660522, + "step": 5000 + }, + { + "epoch": 2.678708814182974, + "grad_norm": 11.719300104728754, + "learning_rate": 3.440885377316721e-08, + "logits/chosen": -0.05981435626745224, + "logits/rejected": -0.013632726855576038, + "logps/chosen": -1.2534570693969727, + "logps/rejected": -1.5460814237594604, + "loss": 1.9694, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.2534570693969727, + "rewards/margins": 0.2926243245601654, + "rewards/rejected": -1.5460814237594604, + "semantic_entropy": 0.8032326698303223, + "step": 5005 + }, + { + "epoch": 2.6813848469643755, + "grad_norm": 11.762943253097305, + "learning_rate": 3.384335607654082e-08, + "logits/chosen": -0.025763485580682755, + "logits/rejected": 0.07047827541828156, + "logps/chosen": -1.3849637508392334, + "logps/rejected": -1.6733529567718506, + "loss": 2.0828, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.3849637508392334, + "rewards/margins": 0.28838926553726196, + "rewards/rejected": -1.6733529567718506, + "semantic_entropy": 0.7628467082977295, + "step": 5010 + }, + { + "epoch": 2.684060879745777, + "grad_norm": 10.864227585313829, + "learning_rate": 3.328238103371811e-08, + "logits/chosen": -0.1647218018770218, + "logits/rejected": -0.09952450543642044, + "logps/chosen": -1.2642019987106323, + "logps/rejected": -1.6397594213485718, + "loss": 1.9762, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2642019987106323, + "rewards/margins": 0.3755575716495514, + "rewards/rejected": -1.6397594213485718, + "semantic_entropy": 0.7904716730117798, + "step": 5015 + }, + { + "epoch": 2.6867369125271785, + "grad_norm": 13.4307602774936, + "learning_rate": 3.272593408728169e-08, + "logits/chosen": -0.17871826887130737, + "logits/rejected": 0.019997352734208107, + "logps/chosen": -1.2422888278961182, + "logps/rejected": -1.505932092666626, + "loss": 1.9959, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2422888278961182, + "rewards/margins": 0.26364344358444214, + "rewards/rejected": -1.505932092666626, + "semantic_entropy": 0.8106265068054199, + "step": 5020 + }, + { + "epoch": 2.6894129453085798, + "grad_norm": 7.923356519451293, + "learning_rate": 3.217402063588204e-08, + "logits/chosen": -0.1537868082523346, + "logits/rejected": -0.014590066857635975, + "logps/chosen": -1.2995110750198364, + "logps/rejected": -1.5679407119750977, + "loss": 2.0147, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.2995110750198364, + "rewards/margins": 0.26842954754829407, + "rewards/rejected": -1.5679407119750977, + "semantic_entropy": 0.7905632853507996, + "step": 5025 + }, + { + "epoch": 2.6920889780899815, + "grad_norm": 7.611333717939541, + "learning_rate": 3.162664603418608e-08, + "logits/chosen": -0.10719075053930283, + "logits/rejected": -0.0370730385184288, + "logps/chosen": -1.2645071744918823, + "logps/rejected": -1.6142085790634155, + "loss": 1.9828, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.2645071744918823, + "rewards/margins": 0.3497016727924347, + "rewards/rejected": -1.6142085790634155, + "semantic_entropy": 0.7915008664131165, + "step": 5030 + }, + { + "epoch": 2.694765010871383, + "grad_norm": 17.417318056705316, + "learning_rate": 3.1083815592824416e-08, + "logits/chosen": -0.14526739716529846, + "logits/rejected": -0.02130313403904438, + "logps/chosen": -1.3097810745239258, + "logps/rejected": -1.5980546474456787, + "loss": 2.0191, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.3097810745239258, + "rewards/margins": 0.2882736027240753, + "rewards/rejected": -1.5980546474456787, + "semantic_entropy": 0.7685422897338867, + "step": 5035 + }, + { + "epoch": 2.697441043652785, + "grad_norm": 17.075460639153633, + "learning_rate": 3.054553457834053e-08, + "logits/chosen": 0.052671514451503754, + "logits/rejected": 0.008498812094330788, + "logps/chosen": -1.231005072593689, + "logps/rejected": -1.6238819360733032, + "loss": 1.941, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.231005072593689, + "rewards/margins": 0.3928767442703247, + "rewards/rejected": -1.6238819360733032, + "semantic_entropy": 0.8041082620620728, + "step": 5040 + }, + { + "epoch": 2.700117076434186, + "grad_norm": 11.695480433858842, + "learning_rate": 3.0011808213139036e-08, + "logits/chosen": -0.09167749434709549, + "logits/rejected": -0.0660163015127182, + "logps/chosen": -1.2336540222167969, + "logps/rejected": -1.5083844661712646, + "loss": 1.9589, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2336540222167969, + "rewards/margins": 0.2747305631637573, + "rewards/rejected": -1.5083844661712646, + "semantic_entropy": 0.8070043325424194, + "step": 5045 + }, + { + "epoch": 2.702793109215588, + "grad_norm": 14.366463277793246, + "learning_rate": 2.948264167543568e-08, + "logits/chosen": -0.12496743351221085, + "logits/rejected": -0.06732519716024399, + "logps/chosen": -1.1421395540237427, + "logps/rejected": -1.5506408214569092, + "loss": 1.8628, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.1421395540237427, + "rewards/margins": 0.4085013270378113, + "rewards/rejected": -1.5506408214569092, + "semantic_entropy": 0.824967086315155, + "step": 5050 + }, + { + "epoch": 2.7054691419969896, + "grad_norm": 8.403527027540157, + "learning_rate": 2.8958040099206216e-08, + "logits/chosen": -0.2553723454475403, + "logits/rejected": -0.17169703543186188, + "logps/chosen": -1.1848903894424438, + "logps/rejected": -1.559682846069336, + "loss": 1.9387, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.1848903894424438, + "rewards/margins": 0.37479257583618164, + "rewards/rejected": -1.559682846069336, + "semantic_entropy": 0.8126693964004517, + "step": 5055 + }, + { + "epoch": 2.708145174778391, + "grad_norm": 18.857515190754253, + "learning_rate": 2.843800857413775e-08, + "logits/chosen": -0.10081684589385986, + "logits/rejected": -0.03810068592429161, + "logps/chosen": -1.2591793537139893, + "logps/rejected": -1.5564645528793335, + "loss": 1.9713, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.2591793537139893, + "rewards/margins": 0.2972853481769562, + "rewards/rejected": -1.5564645528793335, + "semantic_entropy": 0.7883800268173218, + "step": 5060 + }, + { + "epoch": 2.7108212075597926, + "grad_norm": 12.161734792358606, + "learning_rate": 2.7922552145578203e-08, + "logits/chosen": -0.13401418924331665, + "logits/rejected": 0.1141396015882492, + "logps/chosen": -1.3173068761825562, + "logps/rejected": -1.6399991512298584, + "loss": 2.0045, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.3173068761825562, + "rewards/margins": 0.32269221544265747, + "rewards/rejected": -1.6399991512298584, + "semantic_entropy": 0.7699416875839233, + "step": 5065 + }, + { + "epoch": 2.7134972403411943, + "grad_norm": 7.90554243386142, + "learning_rate": 2.7411675814488277e-08, + "logits/chosen": -0.029569268226623535, + "logits/rejected": 0.12846367061138153, + "logps/chosen": -1.1842542886734009, + "logps/rejected": -1.4647448062896729, + "loss": 1.952, + "rewards/accuracies": 0.65625, + "rewards/chosen": -1.1842542886734009, + "rewards/margins": 0.2804903984069824, + "rewards/rejected": -1.4647448062896729, + "semantic_entropy": 0.8209689259529114, + "step": 5070 + }, + { + "epoch": 2.7161732731225956, + "grad_norm": 14.226564064186148, + "learning_rate": 2.690538453739216e-08, + "logits/chosen": -0.11434072256088257, + "logits/rejected": -0.035854704678058624, + "logps/chosen": -1.2371490001678467, + "logps/rejected": -1.4231237173080444, + "loss": 2.0155, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.2371490001678467, + "rewards/margins": 0.18597477674484253, + "rewards/rejected": -1.4231237173080444, + "semantic_entropy": 0.8113747835159302, + "step": 5075 + }, + { + "epoch": 2.7188493059039973, + "grad_norm": 8.766818819474777, + "learning_rate": 2.6403683226330298e-08, + "logits/chosen": -0.17532986402511597, + "logits/rejected": -0.0436691977083683, + "logps/chosen": -1.3013237714767456, + "logps/rejected": -1.5891865491867065, + "loss": 2.0086, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.3013237714767456, + "rewards/margins": 0.28786277770996094, + "rewards/rejected": -1.5891865491867065, + "semantic_entropy": 0.7856767773628235, + "step": 5080 + }, + { + "epoch": 2.721525338685399, + "grad_norm": 12.23805430836543, + "learning_rate": 2.5906576748810804e-08, + "logits/chosen": -0.21402184665203094, + "logits/rejected": -0.09077148139476776, + "logps/chosen": -1.1631752252578735, + "logps/rejected": -1.689199686050415, + "loss": 1.8654, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.1631752252578735, + "rewards/margins": 0.5260245203971863, + "rewards/rejected": -1.689199686050415, + "semantic_entropy": 0.8219331502914429, + "step": 5085 + }, + { + "epoch": 2.7242013714668003, + "grad_norm": 13.707308783644233, + "learning_rate": 2.5414069927763016e-08, + "logits/chosen": -0.24503371119499207, + "logits/rejected": -0.09396155923604965, + "logps/chosen": -1.3182017803192139, + "logps/rejected": -1.6661937236785889, + "loss": 2.01, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.3182017803192139, + "rewards/margins": 0.34799203276634216, + "rewards/rejected": -1.6661937236785889, + "semantic_entropy": 0.7787152528762817, + "step": 5090 + }, + { + "epoch": 2.726877404248202, + "grad_norm": 7.364777652105795, + "learning_rate": 2.4926167541490185e-08, + "logits/chosen": -0.27421778440475464, + "logits/rejected": -0.08374804258346558, + "logps/chosen": -1.2514071464538574, + "logps/rejected": -1.6568397283554077, + "loss": 1.9544, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2514071464538574, + "rewards/margins": 0.4054326117038727, + "rewards/rejected": -1.6568397283554077, + "semantic_entropy": 0.7822387218475342, + "step": 5095 + }, + { + "epoch": 2.7295534370296037, + "grad_norm": 11.643911581815413, + "learning_rate": 2.4442874323623574e-08, + "logits/chosen": -0.0473858080804348, + "logits/rejected": 0.09867437928915024, + "logps/chosen": -1.2100188732147217, + "logps/rejected": -1.7059189081192017, + "loss": 1.9202, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.2100188732147217, + "rewards/margins": 0.49590006470680237, + "rewards/rejected": -1.7059189081192017, + "semantic_entropy": 0.8004026412963867, + "step": 5100 + }, + { + "epoch": 2.7322294698110055, + "grad_norm": 11.921633311514181, + "learning_rate": 2.396419496307589e-08, + "logits/chosen": -0.1327304095029831, + "logits/rejected": 0.032488059252500534, + "logps/chosen": -1.279708981513977, + "logps/rejected": -1.5373016595840454, + "loss": 2.0003, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.279708981513977, + "rewards/margins": 0.25759270787239075, + "rewards/rejected": -1.5373016595840454, + "semantic_entropy": 0.7910350561141968, + "step": 5105 + }, + { + "epoch": 2.7349055025924067, + "grad_norm": 10.5803341934339, + "learning_rate": 2.349013410399653e-08, + "logits/chosen": -0.19295606017112732, + "logits/rejected": -0.04806617647409439, + "logps/chosen": -1.2397714853286743, + "logps/rejected": -1.5628407001495361, + "loss": 1.9677, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2397714853286743, + "rewards/margins": 0.3230692148208618, + "rewards/rejected": -1.5628407001495361, + "semantic_entropy": 0.7900466918945312, + "step": 5110 + }, + { + "epoch": 2.7375815353738084, + "grad_norm": 9.2360531664133, + "learning_rate": 2.3020696345725954e-08, + "logits/chosen": -0.22518637776374817, + "logits/rejected": -0.02344553731381893, + "logps/chosen": -1.2428288459777832, + "logps/rejected": -1.6677284240722656, + "loss": 1.934, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.2428288459777832, + "rewards/margins": 0.42489948868751526, + "rewards/rejected": -1.6677284240722656, + "semantic_entropy": 0.7951303124427795, + "step": 5115 + }, + { + "epoch": 2.7402575681552097, + "grad_norm": 18.21476960792327, + "learning_rate": 2.2555886242751398e-08, + "logits/chosen": -0.1483290195465088, + "logits/rejected": -0.092471644282341, + "logps/chosen": -1.3488088846206665, + "logps/rejected": -1.6935102939605713, + "loss": 2.0112, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.3488088846206665, + "rewards/margins": 0.3447011709213257, + "rewards/rejected": -1.6935102939605713, + "semantic_entropy": 0.7674793004989624, + "step": 5120 + }, + { + "epoch": 2.7429336009366114, + "grad_norm": 19.542866097437877, + "learning_rate": 2.2095708304662453e-08, + "logits/chosen": -0.24939461052417755, + "logits/rejected": -0.03163083642721176, + "logps/chosen": -1.2250001430511475, + "logps/rejected": -1.5612132549285889, + "loss": 1.9479, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": -1.2250001430511475, + "rewards/margins": 0.3362131416797638, + "rewards/rejected": -1.5612132549285889, + "semantic_entropy": 0.7868286371231079, + "step": 5125 + }, + { + "epoch": 2.745609633718013, + "grad_norm": 7.4810492997908495, + "learning_rate": 2.16401669961076e-08, + "logits/chosen": -0.29958608746528625, + "logits/rejected": -0.10414047539234161, + "logps/chosen": -1.2389986515045166, + "logps/rejected": -1.6375420093536377, + "loss": 1.9318, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.2389986515045166, + "rewards/margins": 0.39854365587234497, + "rewards/rejected": -1.6375420093536377, + "semantic_entropy": 0.8021961450576782, + "step": 5130 + }, + { + "epoch": 2.748285666499415, + "grad_norm": 10.75047665436145, + "learning_rate": 2.1189266736750532e-08, + "logits/chosen": -0.07831167429685593, + "logits/rejected": -0.0135183185338974, + "logps/chosen": -1.2490636110305786, + "logps/rejected": -1.5595505237579346, + "loss": 1.9829, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2490636110305786, + "rewards/margins": 0.3104868233203888, + "rewards/rejected": -1.5595505237579346, + "semantic_entropy": 0.7954851388931274, + "step": 5135 + }, + { + "epoch": 2.750961699280816, + "grad_norm": 8.63114102715087, + "learning_rate": 2.0743011901227623e-08, + "logits/chosen": -0.08205153793096542, + "logits/rejected": 0.05202709510922432, + "logps/chosen": -1.3270334005355835, + "logps/rejected": -1.5784505605697632, + "loss": 2.042, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.3270334005355835, + "rewards/margins": 0.25141724944114685, + "rewards/rejected": -1.5784505605697632, + "semantic_entropy": 0.7848073244094849, + "step": 5140 + }, + { + "epoch": 2.753637732062218, + "grad_norm": 14.004505980024575, + "learning_rate": 2.030140681910508e-08, + "logits/chosen": -0.15839698910713196, + "logits/rejected": 0.012042008340358734, + "logps/chosen": -1.2330915927886963, + "logps/rejected": -1.5381847620010376, + "loss": 1.9663, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2330915927886963, + "rewards/margins": 0.30509334802627563, + "rewards/rejected": -1.5381847620010376, + "semantic_entropy": 0.8043828010559082, + "step": 5145 + }, + { + "epoch": 2.756313764843619, + "grad_norm": 7.444852079176876, + "learning_rate": 1.986445577483753e-08, + "logits/chosen": -0.18917617201805115, + "logits/rejected": -0.06302474439144135, + "logps/chosen": -1.2250678539276123, + "logps/rejected": -1.555406093597412, + "loss": 1.9536, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2250678539276123, + "rewards/margins": 0.3303382992744446, + "rewards/rejected": -1.555406093597412, + "semantic_entropy": 0.8019511103630066, + "step": 5150 + }, + { + "epoch": 2.758989797625021, + "grad_norm": 8.680130105829013, + "learning_rate": 1.9432163007725765e-08, + "logits/chosen": -0.18140801787376404, + "logits/rejected": -0.07545305788516998, + "logps/chosen": -1.2941920757293701, + "logps/rejected": -1.6018486022949219, + "loss": 1.985, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2941920757293701, + "rewards/margins": 0.30765634775161743, + "rewards/rejected": -1.6018486022949219, + "semantic_entropy": 0.7732634544372559, + "step": 5155 + }, + { + "epoch": 2.7616658304064226, + "grad_norm": 11.22410178800859, + "learning_rate": 1.9004532711876297e-08, + "logits/chosen": -0.1650318056344986, + "logits/rejected": -0.10949975252151489, + "logps/chosen": -1.2166471481323242, + "logps/rejected": -1.654528021812439, + "loss": 1.8955, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -1.2166471481323242, + "rewards/margins": 0.43788084387779236, + "rewards/rejected": -1.654528021812439, + "semantic_entropy": 0.7718899250030518, + "step": 5160 + }, + { + "epoch": 2.7643418631878243, + "grad_norm": 9.025154317523384, + "learning_rate": 1.8581569036159928e-08, + "logits/chosen": -0.18271777033805847, + "logits/rejected": 0.0020973458886146545, + "logps/chosen": -1.2153449058532715, + "logps/rejected": -1.56462824344635, + "loss": 1.931, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.2153449058532715, + "rewards/margins": 0.3492833971977234, + "rewards/rejected": -1.56462824344635, + "semantic_entropy": 0.8019517660140991, + "step": 5165 + }, + { + "epoch": 2.7670178959692255, + "grad_norm": 10.287380361776217, + "learning_rate": 1.8163276084172285e-08, + "logits/chosen": -0.14131216704845428, + "logits/rejected": -0.015068145468831062, + "logps/chosen": -1.3193836212158203, + "logps/rejected": -1.6025813817977905, + "loss": 2.0232, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.3193836212158203, + "rewards/margins": 0.2831977307796478, + "rewards/rejected": -1.6025813817977905, + "semantic_entropy": 0.7736366987228394, + "step": 5170 + }, + { + "epoch": 2.7696939287506273, + "grad_norm": 10.355275847231685, + "learning_rate": 1.7749657914193194e-08, + "logits/chosen": -0.20772120356559753, + "logits/rejected": -0.10938359797000885, + "logps/chosen": -1.313234567642212, + "logps/rejected": -1.560950517654419, + "loss": 2.0326, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.313234567642212, + "rewards/margins": 0.24771595001220703, + "rewards/rejected": -1.560950517654419, + "semantic_entropy": 0.7819124460220337, + "step": 5175 + }, + { + "epoch": 2.7723699615320285, + "grad_norm": 7.703346828402844, + "learning_rate": 1.7340718539148203e-08, + "logits/chosen": -0.08773273974657059, + "logits/rejected": -0.04304978996515274, + "logps/chosen": -1.3063275814056396, + "logps/rejected": -1.6553189754486084, + "loss": 1.9855, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.3063275814056396, + "rewards/margins": 0.3489914834499359, + "rewards/rejected": -1.6553189754486084, + "semantic_entropy": 0.7809012532234192, + "step": 5180 + }, + { + "epoch": 2.7750459943134302, + "grad_norm": 16.638627028289388, + "learning_rate": 1.6936461926568724e-08, + "logits/chosen": -0.10791786015033722, + "logits/rejected": 0.013361009769141674, + "logps/chosen": -1.1710784435272217, + "logps/rejected": -1.5865617990493774, + "loss": 1.9282, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.1710784435272217, + "rewards/margins": 0.41548317670822144, + "rewards/rejected": -1.5865617990493774, + "semantic_entropy": 0.8109432458877563, + "step": 5185 + }, + { + "epoch": 2.777722027094832, + "grad_norm": 9.333197939438142, + "learning_rate": 1.6536891998554346e-08, + "logits/chosen": -0.22189322113990784, + "logits/rejected": -0.05813136696815491, + "logps/chosen": -1.2453774213790894, + "logps/rejected": -1.5646049976348877, + "loss": 1.9451, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.2453774213790894, + "rewards/margins": 0.3192276954650879, + "rewards/rejected": -1.5646049976348877, + "semantic_entropy": 0.7913827896118164, + "step": 5190 + }, + { + "epoch": 2.7803980598762337, + "grad_norm": 18.888524794439114, + "learning_rate": 1.6142012631734093e-08, + "logits/chosen": -0.13538241386413574, + "logits/rejected": -0.004059618804603815, + "logps/chosen": -1.2707325220108032, + "logps/rejected": -1.5524711608886719, + "loss": 1.9975, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.2707325220108032, + "rewards/margins": 0.28173863887786865, + "rewards/rejected": -1.5524711608886719, + "semantic_entropy": 0.8077930212020874, + "step": 5195 + }, + { + "epoch": 2.783074092657635, + "grad_norm": 10.319272436486468, + "learning_rate": 1.575182765722949e-08, + "logits/chosen": -0.22574253380298615, + "logits/rejected": -0.07820216566324234, + "logps/chosen": -1.2026981115341187, + "logps/rejected": -1.5477980375289917, + "loss": 1.9666, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.2026981115341187, + "rewards/margins": 0.34509986639022827, + "rewards/rejected": -1.5477980375289917, + "semantic_entropy": 0.8090740442276001, + "step": 5200 + }, + { + "epoch": 2.783074092657635, + "eval_logits/chosen": 0.202718585729599, + "eval_logits/rejected": 0.29181304574012756, + "eval_logps/chosen": -1.3354370594024658, + "eval_logps/rejected": -1.6131740808486938, + "eval_loss": 2.0453600883483887, + "eval_rewards/accuracies": 0.5942136645317078, + "eval_rewards/chosen": -1.3354370594024658, + "eval_rewards/margins": 0.27773699164390564, + "eval_rewards/rejected": -1.6131740808486938, + "eval_runtime": 34.6109, + "eval_samples_per_second": 38.861, + "eval_semantic_entropy": 0.7754858136177063, + "eval_steps_per_second": 9.737, + "step": 5200 + }, + { + "epoch": 2.7857501254390367, + "grad_norm": 8.406302503630188, + "learning_rate": 1.536634086061672e-08, + "logits/chosen": -0.07829969376325607, + "logits/rejected": -0.03902588412165642, + "logps/chosen": -1.2435556650161743, + "logps/rejected": -1.53193199634552, + "loss": 1.9881, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.2435556650161743, + "rewards/margins": 0.28837618231773376, + "rewards/rejected": -1.53193199634552, + "semantic_entropy": 0.8077479600906372, + "step": 5205 + }, + { + "epoch": 2.788426158220438, + "grad_norm": 10.640420609743641, + "learning_rate": 1.4985555981890495e-08, + "logits/chosen": -0.15689179301261902, + "logits/rejected": -0.07102429121732712, + "logps/chosen": -1.2550675868988037, + "logps/rejected": -1.6058406829833984, + "loss": 1.964, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.2550675868988037, + "rewards/margins": 0.3507730960845947, + "rewards/rejected": -1.6058406829833984, + "semantic_entropy": 0.7976831197738647, + "step": 5210 + }, + { + "epoch": 2.7911021910018396, + "grad_norm": 12.183156183909475, + "learning_rate": 1.4609476715427226e-08, + "logits/chosen": -0.15650925040245056, + "logits/rejected": -0.06442428380250931, + "logps/chosen": -1.2369836568832397, + "logps/rejected": -1.6222703456878662, + "loss": 1.9431, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2369836568832397, + "rewards/margins": 0.38528645038604736, + "rewards/rejected": -1.6222703456878662, + "semantic_entropy": 0.7931517362594604, + "step": 5215 + }, + { + "epoch": 2.7937782237832414, + "grad_norm": 12.507724678721072, + "learning_rate": 1.4238106709949792e-08, + "logits/chosen": -0.17691577970981598, + "logits/rejected": -0.11024241149425507, + "logps/chosen": -1.2268879413604736, + "logps/rejected": -1.6865144968032837, + "loss": 1.9251, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2268879413604736, + "rewards/margins": 0.45962634682655334, + "rewards/rejected": -1.6865144968032837, + "semantic_entropy": 0.7949502468109131, + "step": 5220 + }, + { + "epoch": 2.796454256564643, + "grad_norm": 10.289025431525555, + "learning_rate": 1.3871449568491511e-08, + "logits/chosen": -0.10499472916126251, + "logits/rejected": 0.023125629872083664, + "logps/chosen": -1.3095515966415405, + "logps/rejected": -1.64128839969635, + "loss": 1.9828, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.3095515966415405, + "rewards/margins": 0.3317367434501648, + "rewards/rejected": -1.64128839969635, + "semantic_entropy": 0.7708584666252136, + "step": 5225 + }, + { + "epoch": 2.7991302893460444, + "grad_norm": 9.972604594873996, + "learning_rate": 1.3509508848361606e-08, + "logits/chosen": -0.2620421350002289, + "logits/rejected": -0.12123207747936249, + "logps/chosen": -1.2767547369003296, + "logps/rejected": -1.5348087549209595, + "loss": 1.9974, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2767547369003296, + "rewards/margins": 0.25805383920669556, + "rewards/rejected": -1.5348087549209595, + "semantic_entropy": 0.794947624206543, + "step": 5230 + }, + { + "epoch": 2.801806322127446, + "grad_norm": 6.219131068931544, + "learning_rate": 1.3152288061110517e-08, + "logits/chosen": -0.20478801429271698, + "logits/rejected": -0.09910500794649124, + "logps/chosen": -1.281906247138977, + "logps/rejected": -1.5762264728546143, + "loss": 1.9851, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.281906247138977, + "rewards/margins": 0.294320285320282, + "rewards/rejected": -1.5762264728546143, + "semantic_entropy": 0.7873275279998779, + "step": 5235 + }, + { + "epoch": 2.804482354908848, + "grad_norm": 10.6291757278702, + "learning_rate": 1.2799790672495814e-08, + "logits/chosen": -0.1837000548839569, + "logits/rejected": 0.024717776104807854, + "logps/chosen": -1.2773462533950806, + "logps/rejected": -1.5936458110809326, + "loss": 2.0204, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2773462533950806, + "rewards/margins": 0.3162994682788849, + "rewards/rejected": -1.5936458110809326, + "semantic_entropy": 0.790886402130127, + "step": 5240 + }, + { + "epoch": 2.807158387690249, + "grad_norm": 6.864364966283842, + "learning_rate": 1.2452020102448835e-08, + "logits/chosen": -0.10664012283086777, + "logits/rejected": -0.060091882944107056, + "logps/chosen": -1.2358882427215576, + "logps/rejected": -1.5273553133010864, + "loss": 1.994, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2358882427215576, + "rewards/margins": 0.2914671301841736, + "rewards/rejected": -1.5273553133010864, + "semantic_entropy": 0.8177428245544434, + "step": 5245 + }, + { + "epoch": 2.8098344204716508, + "grad_norm": 9.134418757055355, + "learning_rate": 1.2108979725041103e-08, + "logits/chosen": -0.20931419730186462, + "logits/rejected": -0.08170606940984726, + "logps/chosen": -1.306196689605713, + "logps/rejected": -1.6155498027801514, + "loss": 2.0098, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.306196689605713, + "rewards/margins": 0.3093533515930176, + "rewards/rejected": -1.6155498027801514, + "semantic_entropy": 0.7775629758834839, + "step": 5250 + }, + { + "epoch": 2.8125104532530525, + "grad_norm": 13.00788199652425, + "learning_rate": 1.1770672868451958e-08, + "logits/chosen": -0.17584146559238434, + "logits/rejected": 0.033514510840177536, + "logps/chosen": -1.2859491109848022, + "logps/rejected": -1.563084363937378, + "loss": 1.9978, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.2859491109848022, + "rewards/margins": 0.27713528275489807, + "rewards/rejected": -1.563084363937378, + "semantic_entropy": 0.7916868329048157, + "step": 5255 + }, + { + "epoch": 2.8151864860344538, + "grad_norm": 13.501783180882384, + "learning_rate": 1.1437102814935872e-08, + "logits/chosen": -0.15966646373271942, + "logits/rejected": -0.0897841602563858, + "logps/chosen": -1.2488842010498047, + "logps/rejected": -1.6675317287445068, + "loss": 1.9729, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2488842010498047, + "rewards/margins": 0.41864776611328125, + "rewards/rejected": -1.6675317287445068, + "semantic_entropy": 0.785289466381073, + "step": 5260 + }, + { + "epoch": 2.8178625188158555, + "grad_norm": 8.89645599420778, + "learning_rate": 1.1108272800791018e-08, + "logits/chosen": -0.27480974793434143, + "logits/rejected": -0.06155538558959961, + "logps/chosen": -1.437657356262207, + "logps/rejected": -1.6343066692352295, + "loss": 2.1194, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -1.437657356262207, + "rewards/margins": 0.19664961099624634, + "rewards/rejected": -1.6343066692352295, + "semantic_entropy": 0.7450114488601685, + "step": 5265 + }, + { + "epoch": 2.820538551597257, + "grad_norm": 7.272694732690356, + "learning_rate": 1.078418601632769e-08, + "logits/chosen": -0.14676833152770996, + "logits/rejected": -0.007456362247467041, + "logps/chosen": -1.2117810249328613, + "logps/rejected": -1.5890445709228516, + "loss": 1.9195, + "rewards/accuracies": 0.65625, + "rewards/chosen": -1.2117810249328613, + "rewards/margins": 0.37726354598999023, + "rewards/rejected": -1.5890445709228516, + "semantic_entropy": 0.8097552061080933, + "step": 5270 + }, + { + "epoch": 2.8232145843786585, + "grad_norm": 9.385479739829243, + "learning_rate": 1.0464845605837159e-08, + "logits/chosen": -0.12716984748840332, + "logits/rejected": 0.021417586132884026, + "logps/chosen": -1.2831096649169922, + "logps/rejected": -1.6105226278305054, + "loss": 1.9832, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2831096649169922, + "rewards/margins": 0.32741299271583557, + "rewards/rejected": -1.6105226278305054, + "semantic_entropy": 0.7869722247123718, + "step": 5275 + }, + { + "epoch": 2.82589061716006, + "grad_norm": 9.165378787106281, + "learning_rate": 1.0150254667561642e-08, + "logits/chosen": -0.133694127202034, + "logits/rejected": 0.03595340996980667, + "logps/chosen": -1.3388880491256714, + "logps/rejected": -1.6667951345443726, + "loss": 2.0435, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.3388880491256714, + "rewards/margins": 0.3279072344303131, + "rewards/rejected": -1.6667951345443726, + "semantic_entropy": 0.7781317830085754, + "step": 5280 + }, + { + "epoch": 2.828566649941462, + "grad_norm": 8.646734065478284, + "learning_rate": 9.840416253663719e-09, + "logits/chosen": -0.19486698508262634, + "logits/rejected": -0.09732584655284882, + "logps/chosen": -1.2249208688735962, + "logps/rejected": -1.6000207662582397, + "loss": 1.9347, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2249208688735962, + "rewards/margins": 0.37510010600090027, + "rewards/rejected": -1.6000207662582397, + "semantic_entropy": 0.8043219447135925, + "step": 5285 + }, + { + "epoch": 2.8312426827228636, + "grad_norm": 8.266243745454222, + "learning_rate": 9.535333370197074e-09, + "logits/chosen": -0.1298667937517166, + "logits/rejected": 0.01398382056504488, + "logps/chosen": -1.2848511934280396, + "logps/rejected": -1.5456931591033936, + "loss": 2.0081, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.2848511934280396, + "rewards/margins": 0.26084208488464355, + "rewards/rejected": -1.5456931591033936, + "semantic_entropy": 0.7974156141281128, + "step": 5290 + }, + { + "epoch": 2.833918715504265, + "grad_norm": 10.837295569106445, + "learning_rate": 9.23500897707713e-09, + "logits/chosen": -0.20668502151966095, + "logits/rejected": -0.016653254628181458, + "logps/chosen": -1.3635807037353516, + "logps/rejected": -1.6869462728500366, + "loss": 2.0255, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.3635807037353516, + "rewards/margins": 0.32336559891700745, + "rewards/rejected": -1.6869462728500366, + "semantic_entropy": 0.7563022971153259, + "step": 5295 + }, + { + "epoch": 2.8365947482856666, + "grad_norm": 8.619693420401413, + "learning_rate": 8.939445988052574e-09, + "logits/chosen": -0.170781672000885, + "logits/rejected": -0.12549729645252228, + "logps/chosen": -1.2618080377578735, + "logps/rejected": -1.6614129543304443, + "loss": 1.962, + "rewards/accuracies": 0.581250011920929, + "rewards/chosen": -1.2618080377578735, + "rewards/margins": 0.3996048867702484, + "rewards/rejected": -1.6614129543304443, + "semantic_entropy": 0.7871630191802979, + "step": 5300 + }, + { + "epoch": 2.839270781067068, + "grad_norm": 10.523548641438046, + "learning_rate": 8.648647270676656e-09, + "logits/chosen": -0.14557881653308868, + "logits/rejected": -0.011387373320758343, + "logps/chosen": -1.3090760707855225, + "logps/rejected": -1.6498647928237915, + "loss": 2.0204, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.3090760707855225, + "rewards/margins": 0.3407888114452362, + "rewards/rejected": -1.6498647928237915, + "semantic_entropy": 0.7654205560684204, + "step": 5305 + }, + { + "epoch": 2.8419468138484696, + "grad_norm": 6.399675035852637, + "learning_rate": 8.362615646279991e-09, + "logits/chosen": -0.31894922256469727, + "logits/rejected": -0.07944828271865845, + "logps/chosen": -1.213585615158081, + "logps/rejected": -1.628617525100708, + "loss": 1.942, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.213585615158081, + "rewards/margins": 0.41503196954727173, + "rewards/rejected": -1.628617525100708, + "semantic_entropy": 0.7991796135902405, + "step": 5310 + }, + { + "epoch": 2.8446228466298713, + "grad_norm": 11.93961968956636, + "learning_rate": 8.081353889942466e-09, + "logits/chosen": -0.06523782014846802, + "logits/rejected": 0.050230931490659714, + "logps/chosen": -1.2478545904159546, + "logps/rejected": -1.4929559230804443, + "loss": 1.9888, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.2478545904159546, + "rewards/margins": 0.2451011836528778, + "rewards/rejected": -1.4929559230804443, + "semantic_entropy": 0.8178671598434448, + "step": 5315 + }, + { + "epoch": 2.847298879411273, + "grad_norm": 9.99117359013494, + "learning_rate": 7.804864730467042e-09, + "logits/chosen": -0.08324486017227173, + "logits/rejected": -0.010287337005138397, + "logps/chosen": -1.2432942390441895, + "logps/rejected": -1.4651365280151367, + "loss": 2.0026, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.2432942390441895, + "rewards/margins": 0.2218422144651413, + "rewards/rejected": -1.4651365280151367, + "semantic_entropy": 0.8178890347480774, + "step": 5320 + }, + { + "epoch": 2.8499749121926743, + "grad_norm": 7.974653615981402, + "learning_rate": 7.533150850352665e-09, + "logits/chosen": -0.13628502190113068, + "logits/rejected": -0.0066189453937113285, + "logps/chosen": -1.2962652444839478, + "logps/rejected": -1.6911674737930298, + "loss": 1.9772, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2962652444839478, + "rewards/margins": 0.39490213990211487, + "rewards/rejected": -1.6911674737930298, + "semantic_entropy": 0.7716677188873291, + "step": 5325 + }, + { + "epoch": 2.852650944974076, + "grad_norm": 12.160552400520343, + "learning_rate": 7.2662148857686175e-09, + "logits/chosen": -0.08609539270401001, + "logits/rejected": -0.019694218412041664, + "logps/chosen": -1.2226903438568115, + "logps/rejected": -1.6135330200195312, + "loss": 1.9282, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -1.2226903438568115, + "rewards/margins": 0.3908424973487854, + "rewards/rejected": -1.6135330200195312, + "semantic_entropy": 0.8041356801986694, + "step": 5330 + }, + { + "epoch": 2.8553269777554773, + "grad_norm": 13.713048310150327, + "learning_rate": 7.0040594265287635e-09, + "logits/chosen": -0.054106198251247406, + "logits/rejected": -0.07883013784885406, + "logps/chosen": -1.274173617362976, + "logps/rejected": -1.532825231552124, + "loss": 2.008, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.274173617362976, + "rewards/margins": 0.25865158438682556, + "rewards/rejected": -1.532825231552124, + "semantic_entropy": 0.8100245594978333, + "step": 5335 + }, + { + "epoch": 2.858003010536879, + "grad_norm": 8.870225797744203, + "learning_rate": 6.746687016066566e-09, + "logits/chosen": -0.1050344929099083, + "logits/rejected": -0.07736798375844955, + "logps/chosen": -1.2842061519622803, + "logps/rejected": -1.4983108043670654, + "loss": 2.0291, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2842061519622803, + "rewards/margins": 0.21410492062568665, + "rewards/rejected": -1.4983108043670654, + "semantic_entropy": 0.7951924204826355, + "step": 5340 + }, + { + "epoch": 2.8606790433182807, + "grad_norm": 7.400982060956275, + "learning_rate": 6.494100151410276e-09, + "logits/chosen": -0.25141647458076477, + "logits/rejected": -0.08219093829393387, + "logps/chosen": -1.214036226272583, + "logps/rejected": -1.517608880996704, + "loss": 1.9179, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.214036226272583, + "rewards/margins": 0.3035728335380554, + "rewards/rejected": -1.517608880996704, + "semantic_entropy": 0.8065169453620911, + "step": 5345 + }, + { + "epoch": 2.8633550760996824, + "grad_norm": 12.611434321679779, + "learning_rate": 6.246301283158728e-09, + "logits/chosen": -0.08424808084964752, + "logits/rejected": -0.10715341567993164, + "logps/chosen": -1.3435392379760742, + "logps/rejected": -1.6009973287582397, + "loss": 2.0603, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.3435392379760742, + "rewards/margins": 0.257457971572876, + "rewards/rejected": -1.6009973287582397, + "semantic_entropy": 0.7735006213188171, + "step": 5350 + }, + { + "epoch": 2.8660311088810837, + "grad_norm": 9.774989125777068, + "learning_rate": 6.0032928154576944e-09, + "logits/chosen": -0.17313161492347717, + "logits/rejected": -0.08946957439184189, + "logps/chosen": -1.2829643487930298, + "logps/rejected": -1.5711157321929932, + "loss": 2.0195, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.2829643487930298, + "rewards/margins": 0.28815120458602905, + "rewards/rejected": -1.5711157321929932, + "semantic_entropy": 0.8007059097290039, + "step": 5355 + }, + { + "epoch": 2.8687071416624854, + "grad_norm": 12.97437785965202, + "learning_rate": 5.76507710597629e-09, + "logits/chosen": -0.16783717274665833, + "logits/rejected": 0.017929133027791977, + "logps/chosen": -1.2705787420272827, + "logps/rejected": -1.6087089776992798, + "loss": 1.994, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.2705787420272827, + "rewards/margins": 0.3381301462650299, + "rewards/rejected": -1.6087089776992798, + "semantic_entropy": 0.7808153629302979, + "step": 5360 + }, + { + "epoch": 2.8713831744438867, + "grad_norm": 7.617332004084715, + "learning_rate": 5.531656465884438e-09, + "logits/chosen": -0.21880276501178741, + "logits/rejected": -0.06172800809144974, + "logps/chosen": -1.2684425115585327, + "logps/rejected": -1.650254487991333, + "loss": 1.9603, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -1.2684425115585327, + "rewards/margins": 0.38181185722351074, + "rewards/rejected": -1.650254487991333, + "semantic_entropy": 0.7841113805770874, + "step": 5365 + }, + { + "epoch": 2.8740592072252884, + "grad_norm": 11.370064221928539, + "learning_rate": 5.303033159830217e-09, + "logits/chosen": -0.0645141750574112, + "logits/rejected": -0.03054744563996792, + "logps/chosen": -1.2745743989944458, + "logps/rejected": -1.4681590795516968, + "loss": 2.027, + "rewards/accuracies": 0.53125, + "rewards/chosen": -1.2745743989944458, + "rewards/margins": 0.19358472526073456, + "rewards/rejected": -1.4681590795516968, + "semantic_entropy": 0.8125517964363098, + "step": 5370 + }, + { + "epoch": 2.87673524000669, + "grad_norm": 8.848175566333316, + "learning_rate": 5.079209405917939e-09, + "logits/chosen": -0.15581555664539337, + "logits/rejected": -0.06532729417085648, + "logps/chosen": -1.217376947402954, + "logps/rejected": -1.6858150959014893, + "loss": 1.9122, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.217376947402954, + "rewards/margins": 0.4684379994869232, + "rewards/rejected": -1.6858150959014893, + "semantic_entropy": 0.7822630405426025, + "step": 5375 + }, + { + "epoch": 2.879411272788092, + "grad_norm": 8.075282901662117, + "learning_rate": 4.860187375686664e-09, + "logits/chosen": -0.180607870221138, + "logits/rejected": 0.031008031219244003, + "logps/chosen": -1.3817294836044312, + "logps/rejected": -1.6795692443847656, + "loss": 2.0293, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.3817294836044312, + "rewards/margins": 0.29783961176872253, + "rewards/rejected": -1.6795692443847656, + "semantic_entropy": 0.7511149644851685, + "step": 5380 + }, + { + "epoch": 2.882087305569493, + "grad_norm": 8.302562710027024, + "learning_rate": 4.64596919408905e-09, + "logits/chosen": -0.10999952256679535, + "logits/rejected": -0.03438536450266838, + "logps/chosen": -1.2459831237792969, + "logps/rejected": -1.5046635866165161, + "loss": 1.9805, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2459831237792969, + "rewards/margins": 0.25868046283721924, + "rewards/rejected": -1.5046635866165161, + "semantic_entropy": 0.7971447706222534, + "step": 5385 + }, + { + "epoch": 2.884763338350895, + "grad_norm": 8.099658967264567, + "learning_rate": 4.436556939470814e-09, + "logits/chosen": -0.12270566076040268, + "logits/rejected": -0.01083715446293354, + "logps/chosen": -1.2990280389785767, + "logps/rejected": -1.554796814918518, + "loss": 1.9949, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2990280389785767, + "rewards/margins": 0.255768746137619, + "rewards/rejected": -1.554796814918518, + "semantic_entropy": 0.7722955942153931, + "step": 5390 + }, + { + "epoch": 2.887439371132296, + "grad_norm": 7.111781634638527, + "learning_rate": 4.23195264355064e-09, + "logits/chosen": -0.29376253485679626, + "logits/rejected": -0.1047648936510086, + "logps/chosen": -1.2102776765823364, + "logps/rejected": -1.550250768661499, + "loss": 1.9123, + "rewards/accuracies": 0.71875, + "rewards/chosen": -1.2102776765823364, + "rewards/margins": 0.3399733006954193, + "rewards/rejected": -1.550250768661499, + "semantic_entropy": 0.800680935382843, + "step": 5395 + }, + { + "epoch": 2.890115403913698, + "grad_norm": 9.097021482847795, + "learning_rate": 4.032158291400245e-09, + "logits/chosen": -0.2035544365644455, + "logits/rejected": 0.0319555401802063, + "logps/chosen": -1.2738873958587646, + "logps/rejected": -1.7766224145889282, + "loss": 1.9624, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -1.2738873958587646, + "rewards/margins": 0.5027349591255188, + "rewards/rejected": -1.7766224145889282, + "semantic_entropy": 0.7796919345855713, + "step": 5400 + }, + { + "epoch": 2.8927914366950995, + "grad_norm": 10.821844566960877, + "learning_rate": 3.837175821425398e-09, + "logits/chosen": -0.10578795522451401, + "logits/rejected": -0.06705756485462189, + "logps/chosen": -1.3472933769226074, + "logps/rejected": -1.668858528137207, + "loss": 2.0212, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.3472933769226074, + "rewards/margins": 0.3215652108192444, + "rewards/rejected": -1.668858528137207, + "semantic_entropy": 0.7665466070175171, + "step": 5405 + }, + { + "epoch": 2.8954674694765012, + "grad_norm": 9.511240934829583, + "learning_rate": 3.6470071253467683e-09, + "logits/chosen": -0.09955920279026031, + "logits/rejected": 0.02503589354455471, + "logps/chosen": -1.250042200088501, + "logps/rejected": -1.6926124095916748, + "loss": 1.9493, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.250042200088501, + "rewards/margins": 0.44257020950317383, + "rewards/rejected": -1.6926124095916748, + "semantic_entropy": 0.7760982513427734, + "step": 5410 + }, + { + "epoch": 2.8981435022579025, + "grad_norm": 21.910192532646565, + "learning_rate": 3.461654048181939e-09, + "logits/chosen": -0.15834368765354156, + "logits/rejected": 0.009910332970321178, + "logps/chosen": -1.2846863269805908, + "logps/rejected": -1.5575840473175049, + "loss": 2.0001, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.2846863269805908, + "rewards/margins": 0.2728978991508484, + "rewards/rejected": -1.5575840473175049, + "semantic_entropy": 0.7935778498649597, + "step": 5415 + }, + { + "epoch": 2.9008195350393042, + "grad_norm": 12.313994859625039, + "learning_rate": 3.281118388227255e-09, + "logits/chosen": -0.12874098122119904, + "logits/rejected": -0.06065679341554642, + "logps/chosen": -1.1999180316925049, + "logps/rejected": -1.5103440284729004, + "loss": 1.9509, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.1999180316925049, + "rewards/margins": 0.3104260265827179, + "rewards/rejected": -1.5103440284729004, + "semantic_entropy": 0.808806300163269, + "step": 5420 + }, + { + "epoch": 2.903495567820706, + "grad_norm": 26.48957031686247, + "learning_rate": 3.1054018970405048e-09, + "logits/chosen": -0.13419437408447266, + "logits/rejected": -0.0012315213680267334, + "logps/chosen": -1.3119560480117798, + "logps/rejected": -1.630934476852417, + "loss": 2.0138, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.3119560480117798, + "rewards/margins": 0.31897860765457153, + "rewards/rejected": -1.630934476852417, + "semantic_entropy": 0.7729989886283875, + "step": 5425 + }, + { + "epoch": 2.906171600602107, + "grad_norm": 10.56228413619259, + "learning_rate": 2.9345062794238207e-09, + "logits/chosen": -0.16747361421585083, + "logits/rejected": -0.005986332893371582, + "logps/chosen": -1.2478454113006592, + "logps/rejected": -1.6454213857650757, + "loss": 1.9365, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.2478454113006592, + "rewards/margins": 0.3975757956504822, + "rewards/rejected": -1.6454213857650757, + "semantic_entropy": 0.7814873456954956, + "step": 5430 + }, + { + "epoch": 2.908847633383509, + "grad_norm": 16.477061576992455, + "learning_rate": 2.7684331934072492e-09, + "logits/chosen": -0.2565791606903076, + "logits/rejected": -0.15934288501739502, + "logps/chosen": -1.2421395778656006, + "logps/rejected": -1.634916067123413, + "loss": 1.9308, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": -1.2421395778656006, + "rewards/margins": 0.3927767276763916, + "rewards/rejected": -1.634916067123413, + "semantic_entropy": 0.7934702634811401, + "step": 5435 + }, + { + "epoch": 2.9115236661649107, + "grad_norm": 8.614201099283617, + "learning_rate": 2.6071842502326526e-09, + "logits/chosen": -0.20132644474506378, + "logits/rejected": -0.08974252641201019, + "logps/chosen": -1.2590628862380981, + "logps/rejected": -1.6281840801239014, + "loss": 1.9777, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -1.2590628862380981, + "rewards/margins": 0.36912113428115845, + "rewards/rejected": -1.6281840801239014, + "semantic_entropy": 0.7918053865432739, + "step": 5440 + }, + { + "epoch": 2.9141996989463124, + "grad_norm": 8.001790955825099, + "learning_rate": 2.450761014337888e-09, + "logits/chosen": 0.013191893696784973, + "logits/rejected": 0.02028048411011696, + "logps/chosen": -1.2388643026351929, + "logps/rejected": -1.761318564414978, + "loss": 1.9365, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.2388643026351929, + "rewards/margins": 0.5224540829658508, + "rewards/rejected": -1.761318564414978, + "semantic_entropy": 0.788965106010437, + "step": 5445 + }, + { + "epoch": 2.9168757317277136, + "grad_norm": 9.697218288679418, + "learning_rate": 2.299165003341985e-09, + "logits/chosen": -0.056405842304229736, + "logits/rejected": 0.04429206997156143, + "logps/chosen": -1.2803170680999756, + "logps/rejected": -1.6816307306289673, + "loss": 1.9666, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.2803170680999756, + "rewards/margins": 0.401313453912735, + "rewards/rejected": -1.6816307306289673, + "semantic_entropy": 0.7831650376319885, + "step": 5450 + }, + { + "epoch": 2.9195517645091154, + "grad_norm": 8.665580772903214, + "learning_rate": 2.1523976880299945e-09, + "logits/chosen": -0.16580912470817566, + "logits/rejected": -0.004101097583770752, + "logps/chosen": -1.2580012083053589, + "logps/rejected": -1.508253574371338, + "loss": 1.9929, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2580012083053589, + "rewards/margins": 0.2502524256706238, + "rewards/rejected": -1.508253574371338, + "semantic_entropy": 0.8098133206367493, + "step": 5455 + }, + { + "epoch": 2.9222277972905166, + "grad_norm": 12.695614547089772, + "learning_rate": 2.010460492339161e-09, + "logits/chosen": -0.14466659724712372, + "logits/rejected": -0.04582420736551285, + "logps/chosen": -1.26729154586792, + "logps/rejected": -1.6190316677093506, + "loss": 1.9557, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.26729154586792, + "rewards/margins": 0.35174015164375305, + "rewards/rejected": -1.6190316677093506, + "semantic_entropy": 0.7803028225898743, + "step": 5460 + }, + { + "epoch": 2.9249038300719183, + "grad_norm": 11.8295707458819, + "learning_rate": 1.8733547933446614e-09, + "logits/chosen": -0.21898195147514343, + "logits/rejected": -0.03481177240610123, + "logps/chosen": -1.3300405740737915, + "logps/rejected": -1.571028470993042, + "loss": 2.0446, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.3300405740737915, + "rewards/margins": 0.240988090634346, + "rewards/rejected": -1.571028470993042, + "semantic_entropy": 0.7773094177246094, + "step": 5465 + }, + { + "epoch": 2.92757986285332, + "grad_norm": 11.067365059351255, + "learning_rate": 1.7410819212467231e-09, + "logits/chosen": -0.12249497324228287, + "logits/rejected": -0.040003784000873566, + "logps/chosen": -1.2132530212402344, + "logps/rejected": -1.5384536981582642, + "loss": 1.9432, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.2132530212402344, + "rewards/margins": 0.3252008259296417, + "rewards/rejected": -1.5384536981582642, + "semantic_entropy": 0.82013338804245, + "step": 5470 + }, + { + "epoch": 2.9302558956347218, + "grad_norm": 9.52612200576922, + "learning_rate": 1.613643159357192e-09, + "logits/chosen": -0.08768186718225479, + "logits/rejected": -0.1272381991147995, + "logps/chosen": -1.1816210746765137, + "logps/rejected": -1.5005667209625244, + "loss": 1.9438, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": -1.1816210746765137, + "rewards/margins": 0.3189457654953003, + "rewards/rejected": -1.5005667209625244, + "semantic_entropy": 0.8184798955917358, + "step": 5475 + }, + { + "epoch": 2.932931928416123, + "grad_norm": 8.608822633708195, + "learning_rate": 1.4910397440875967e-09, + "logits/chosen": -0.13726715743541718, + "logits/rejected": -0.033788178116083145, + "logps/chosen": -1.2667338848114014, + "logps/rejected": -1.5312033891677856, + "loss": 1.9977, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2667338848114014, + "rewards/margins": 0.26446956396102905, + "rewards/rejected": -1.5312033891677856, + "semantic_entropy": 0.7881470918655396, + "step": 5480 + }, + { + "epoch": 2.9356079611975248, + "grad_norm": 10.212245406357974, + "learning_rate": 1.3732728649368253e-09, + "logits/chosen": -0.08094757795333862, + "logits/rejected": 0.08302084356546402, + "logps/chosen": -1.253039836883545, + "logps/rejected": -1.552504062652588, + "loss": 1.9854, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.253039836883545, + "rewards/margins": 0.2994643449783325, + "rewards/rejected": -1.552504062652588, + "semantic_entropy": 0.811646580696106, + "step": 5485 + }, + { + "epoch": 2.938283993978926, + "grad_norm": 15.191474078296592, + "learning_rate": 1.260343664479524e-09, + "logits/chosen": -0.15114764869213104, + "logits/rejected": -0.1167159229516983, + "logps/chosen": -1.2422395944595337, + "logps/rejected": -1.5190513134002686, + "loss": 1.9799, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.2422395944595337, + "rewards/margins": 0.27681177854537964, + "rewards/rejected": -1.5190513134002686, + "semantic_entropy": 0.8092382550239563, + "step": 5490 + }, + { + "epoch": 2.9409600267603278, + "grad_norm": 9.324040014918797, + "learning_rate": 1.1522532383554384e-09, + "logits/chosen": -0.21880176663398743, + "logits/rejected": -0.016771014779806137, + "logps/chosen": -1.2089433670043945, + "logps/rejected": -1.6005357503890991, + "loss": 1.9122, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -1.2089433670043945, + "rewards/margins": 0.3915923535823822, + "rewards/rejected": -1.6005357503890991, + "semantic_entropy": 0.8016504049301147, + "step": 5495 + }, + { + "epoch": 2.9436360595417295, + "grad_norm": 7.290687595120567, + "learning_rate": 1.049002635258256e-09, + "logits/chosen": -0.12340692430734634, + "logits/rejected": -0.015453631989657879, + "logps/chosen": -1.3105392456054688, + "logps/rejected": -1.5042502880096436, + "loss": 2.0316, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.3105392456054688, + "rewards/margins": 0.19371113181114197, + "rewards/rejected": -1.5042502880096436, + "semantic_entropy": 0.7904552817344666, + "step": 5500 + }, + { + "epoch": 2.946312092323131, + "grad_norm": 6.2992627661020295, + "learning_rate": 9.505928569258358e-10, + "logits/chosen": -0.09954291582107544, + "logits/rejected": -0.0960114449262619, + "logps/chosen": -1.2732259035110474, + "logps/rejected": -1.577926754951477, + "loss": 1.965, + "rewards/accuracies": 0.59375, + "rewards/chosen": -1.2732259035110474, + "rewards/margins": 0.30470094084739685, + "rewards/rejected": -1.577926754951477, + "semantic_entropy": 0.7664059400558472, + "step": 5505 + }, + { + "epoch": 2.9489881251045325, + "grad_norm": 9.891123526520488, + "learning_rate": 8.57024858130273e-10, + "logits/chosen": -0.18454578518867493, + "logits/rejected": -0.08242712914943695, + "logps/chosen": -1.2829363346099854, + "logps/rejected": -1.7573705911636353, + "loss": 1.9526, + "rewards/accuracies": 0.6875, + "rewards/chosen": -1.2829363346099854, + "rewards/margins": 0.4744341969490051, + "rewards/rejected": -1.7573705911636353, + "semantic_entropy": 0.7814774513244629, + "step": 5510 + }, + { + "epoch": 2.951664157885934, + "grad_norm": 13.079909249227054, + "learning_rate": 7.682995466686826e-10, + "logits/chosen": -0.24678631126880646, + "logits/rejected": -0.11660070717334747, + "logps/chosen": -1.235660433769226, + "logps/rejected": -1.7078622579574585, + "loss": 1.9316, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.235660433769226, + "rewards/margins": 0.47220176458358765, + "rewards/rejected": -1.7078622579574585, + "semantic_entropy": 0.7808048129081726, + "step": 5515 + }, + { + "epoch": 2.9543401906673354, + "grad_norm": 13.000873712454275, + "learning_rate": 6.844177833543741e-10, + "logits/chosen": -0.16099712252616882, + "logits/rejected": -0.1062990203499794, + "logps/chosen": -1.2108200788497925, + "logps/rejected": -1.515880823135376, + "loss": 1.9706, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2108200788497925, + "rewards/margins": 0.3050605058670044, + "rewards/rejected": -1.515880823135376, + "semantic_entropy": 0.8106793165206909, + "step": 5520 + }, + { + "epoch": 2.957016223448737, + "grad_norm": 7.343368786137372, + "learning_rate": 6.053803820087467e-10, + "logits/chosen": -0.15496540069580078, + "logits/rejected": -0.03871823102235794, + "logps/chosen": -1.269761323928833, + "logps/rejected": -1.7179292440414429, + "loss": 1.9548, + "rewards/accuracies": 0.65625, + "rewards/chosen": -1.269761323928833, + "rewards/margins": 0.44816771149635315, + "rewards/rejected": -1.7179292440414429, + "semantic_entropy": 0.7783750891685486, + "step": 5525 + }, + { + "epoch": 2.959692256230139, + "grad_norm": 13.65669677320013, + "learning_rate": 5.311881094528514e-10, + "logits/chosen": -0.18310664594173431, + "logits/rejected": 0.036461032927036285, + "logps/chosen": -1.361135721206665, + "logps/rejected": -1.5666468143463135, + "loss": 2.0688, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.361135721206665, + "rewards/margins": 0.20551109313964844, + "rewards/rejected": -1.5666468143463135, + "semantic_entropy": 0.7689114809036255, + "step": 5530 + }, + { + "epoch": 2.9623682890115406, + "grad_norm": 15.509024609341324, + "learning_rate": 4.6184168550050806e-10, + "logits/chosen": -0.13887521624565125, + "logits/rejected": -0.09963810443878174, + "logps/chosen": -1.234496831893921, + "logps/rejected": -1.4671688079833984, + "loss": 1.9948, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.234496831893921, + "rewards/margins": 0.23267188668251038, + "rewards/rejected": -1.4671688079833984, + "semantic_entropy": 0.8018797636032104, + "step": 5535 + }, + { + "epoch": 2.965044321792942, + "grad_norm": 10.718545297303793, + "learning_rate": 3.973417829510328e-10, + "logits/chosen": -0.25150126218795776, + "logits/rejected": -0.09888546913862228, + "logps/chosen": -1.3156968355178833, + "logps/rejected": -1.5430917739868164, + "loss": 2.0171, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.3156968355178833, + "rewards/margins": 0.2273949384689331, + "rewards/rejected": -1.5430917739868164, + "semantic_entropy": 0.7827169895172119, + "step": 5540 + }, + { + "epoch": 2.9677203545743436, + "grad_norm": 8.659532986487385, + "learning_rate": 3.3768902758274377e-10, + "logits/chosen": -0.15638864040374756, + "logits/rejected": -0.04953031614422798, + "logps/chosen": -1.2060474157333374, + "logps/rejected": -1.4767600297927856, + "loss": 1.9638, + "rewards/accuracies": 0.5874999761581421, + "rewards/chosen": -1.2060474157333374, + "rewards/margins": 0.2707127630710602, + "rewards/rejected": -1.4767600297927856, + "semantic_entropy": 0.8145160675048828, + "step": 5545 + }, + { + "epoch": 2.970396387355745, + "grad_norm": 10.443161949595284, + "learning_rate": 2.8288399814691e-10, + "logits/chosen": -0.08594232052564621, + "logits/rejected": 0.02057645097374916, + "logps/chosen": -1.3247028589248657, + "logps/rejected": -1.6422897577285767, + "loss": 2.0071, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.3247028589248657, + "rewards/margins": 0.3175868093967438, + "rewards/rejected": -1.6422897577285767, + "semantic_entropy": 0.7709945440292358, + "step": 5550 + }, + { + "epoch": 2.9730724201371466, + "grad_norm": 11.926017722679141, + "learning_rate": 2.3292722636220066e-10, + "logits/chosen": -0.17937864363193512, + "logits/rejected": 0.0162881501019001, + "logps/chosen": -1.352497935295105, + "logps/rejected": -1.6892569065093994, + "loss": 2.0197, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": -1.352497935295105, + "rewards/margins": 0.3367590308189392, + "rewards/rejected": -1.6892569065093994, + "semantic_entropy": 0.7497475743293762, + "step": 5555 + }, + { + "epoch": 2.9757484529185483, + "grad_norm": 9.346052854797605, + "learning_rate": 1.8781919690946668e-10, + "logits/chosen": -0.11221225559711456, + "logits/rejected": -0.10421918332576752, + "logps/chosen": -1.2898824214935303, + "logps/rejected": -1.4666532278060913, + "loss": 2.0231, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.2898824214935303, + "rewards/margins": 0.17677077651023865, + "rewards/rejected": -1.4666532278060913, + "semantic_entropy": 0.794800877571106, + "step": 5560 + }, + { + "epoch": 2.97842448569995, + "grad_norm": 14.60377778075287, + "learning_rate": 1.4756034742696711e-10, + "logits/chosen": -0.2185247242450714, + "logits/rejected": -0.17179787158966064, + "logps/chosen": -1.2479779720306396, + "logps/rejected": -1.4974956512451172, + "loss": 1.9906, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.2479779720306396, + "rewards/margins": 0.2495175302028656, + "rewards/rejected": -1.4974956512451172, + "semantic_entropy": 0.814715564250946, + "step": 5565 + }, + { + "epoch": 2.9811005184813513, + "grad_norm": 9.99143896346487, + "learning_rate": 1.12151068506261e-10, + "logits/chosen": -0.13437816500663757, + "logits/rejected": -0.010555913671851158, + "logps/chosen": -1.2261104583740234, + "logps/rejected": -1.6780275106430054, + "loss": 1.9285, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -1.2261104583740234, + "rewards/margins": 0.4519170820713043, + "rewards/rejected": -1.6780275106430054, + "semantic_entropy": 0.783943772315979, + "step": 5570 + }, + { + "epoch": 2.983776551262753, + "grad_norm": 11.481288312845669, + "learning_rate": 8.159170368826629e-11, + "logits/chosen": -0.18372122943401337, + "logits/rejected": -0.04386281967163086, + "logps/chosen": -1.2467625141143799, + "logps/rejected": -1.6158266067504883, + "loss": 1.9529, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.2467625141143799, + "rewards/margins": 0.3690639138221741, + "rewards/rejected": -1.6158266067504883, + "semantic_entropy": 0.7892996072769165, + "step": 5575 + }, + { + "epoch": 2.9864525840441547, + "grad_norm": 8.394770887084695, + "learning_rate": 5.588254946015114e-11, + "logits/chosen": -0.2578745484352112, + "logits/rejected": -0.018613124266266823, + "logps/chosen": -1.1846576929092407, + "logps/rejected": -1.6049606800079346, + "loss": 1.9194, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -1.1846576929092407, + "rewards/margins": 0.4203028678894043, + "rewards/rejected": -1.6049606800079346, + "semantic_entropy": 0.8201394081115723, + "step": 5580 + }, + { + "epoch": 2.989128616825556, + "grad_norm": 11.272528683106293, + "learning_rate": 3.502385525216978e-11, + "logits/chosen": -0.2203395813703537, + "logits/rejected": -0.06656957417726517, + "logps/chosen": -1.2808977365493774, + "logps/rejected": -1.6387157440185547, + "loss": 1.9687, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -1.2808977365493774, + "rewards/margins": 0.357818067073822, + "rewards/rejected": -1.6387157440185547, + "semantic_entropy": 0.7756115794181824, + "step": 5585 + }, + { + "epoch": 2.9918046496069577, + "grad_norm": 7.44528259986258, + "learning_rate": 1.901582343555308e-11, + "logits/chosen": -0.1454438716173172, + "logits/rejected": -0.1031564474105835, + "logps/chosen": -1.3283231258392334, + "logps/rejected": -1.5797977447509766, + "loss": 2.0296, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.3283231258392334, + "rewards/margins": 0.2514745593070984, + "rewards/rejected": -1.5797977447509766, + "semantic_entropy": 0.7781016230583191, + "step": 5590 + }, + { + "epoch": 2.9944806823883594, + "grad_norm": 12.137871603428197, + "learning_rate": 7.858609320232634e-12, + "logits/chosen": -0.17230184376239777, + "logits/rejected": -0.04320326820015907, + "logps/chosen": -1.2008966207504272, + "logps/rejected": -1.540555715560913, + "loss": 1.9434, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": -1.2008966207504272, + "rewards/margins": 0.3396591544151306, + "rewards/rejected": -1.540555715560913, + "semantic_entropy": 0.8178640604019165, + "step": 5595 + }, + { + "epoch": 2.9971567151697607, + "grad_norm": 12.303664266654412, + "learning_rate": 1.5523211535639624e-12, + "logits/chosen": -0.16199703514575958, + "logits/rejected": -0.0670817419886589, + "logps/chosen": -1.2330716848373413, + "logps/rejected": -1.6515203714370728, + "loss": 1.9498, + "rewards/accuracies": 0.606249988079071, + "rewards/chosen": -1.2330716848373413, + "rewards/margins": 0.4184487462043762, + "rewards/rejected": -1.6515203714370728, + "semantic_entropy": 0.7843411564826965, + "step": 5600 + }, + { + "epoch": 2.9971567151697607, + "eval_logits/chosen": 0.1504385620355606, + "eval_logits/rejected": 0.23508602380752563, + "eval_logps/chosen": -1.3358336687088013, + "eval_logps/rejected": -1.611994743347168, + "eval_loss": 2.0457005500793457, + "eval_rewards/accuracies": 0.5927299857139587, + "eval_rewards/chosen": -1.3358336687088013, + "eval_rewards/margins": 0.27616116404533386, + "eval_rewards/rejected": -1.611994743347168, + "eval_runtime": 34.6344, + "eval_samples_per_second": 38.834, + "eval_semantic_entropy": 0.7755205631256104, + "eval_steps_per_second": 9.73, + "step": 5600 + }, + { + "epoch": 2.999297541394882, + "step": 5604, + "total_flos": 0.0, + "train_loss": 2.028473087245443, + "train_runtime": 28359.3461, + "train_samples_per_second": 6.325, + "train_steps_per_second": 0.198 + } + ], + "logging_steps": 5, + "max_steps": 5604, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 1000000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +}