| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.1607717041800643, | |
| "eval_steps": 100, | |
| "global_step": 200, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0, | |
| "grad_norm": 2.5493941690669297, | |
| "learning_rate": 4e-09, | |
| "logits/policy_chosen": -2.279445171356201, | |
| "logits/policy_rejected": -2.070641279220581, | |
| "logits/reference_chosen": -2.262451410293579, | |
| "logits/reference_rejected": -1.700181484222412, | |
| "logps/policy_chosen": -259.83294677734375, | |
| "logps/policy_rejected": -282.3813171386719, | |
| "logps/reference_chosen": -280.28363037109375, | |
| "logps/reference_rejected": -290.83868408203125, | |
| "loss": 0.6746, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.20450688898563385, | |
| "rewards/margins": 0.11993316560983658, | |
| "rewards/rejected": 0.08457372337579727, | |
| "step": 1, | |
| "tokens/chosen_response_len_mean": 528.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 444.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "grad_norm": 2.2490732615654547, | |
| "learning_rate": 8e-09, | |
| "logits/policy_chosen": -2.745072364807129, | |
| "logits/policy_rejected": -2.607792854309082, | |
| "logits/reference_chosen": -2.701054096221924, | |
| "logits/reference_rejected": -2.529007911682129, | |
| "logps/policy_chosen": -147.3030242919922, | |
| "logps/policy_rejected": -197.99253845214844, | |
| "logps/reference_chosen": -156.72518920898438, | |
| "logps/reference_rejected": -206.69471740722656, | |
| "loss": 0.6863, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.09422144293785095, | |
| "rewards/margins": 0.0071996524930000305, | |
| "rewards/rejected": 0.08702179044485092, | |
| "step": 2, | |
| "tokens/chosen_response_len_mean": 298.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 277.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "grad_norm": 2.2889093862107543, | |
| "learning_rate": 1.2e-08, | |
| "logits/policy_chosen": -2.882375478744507, | |
| "logits/policy_rejected": -2.992877960205078, | |
| "logits/reference_chosen": -2.748257637023926, | |
| "logits/reference_rejected": -2.9701337814331055, | |
| "logps/policy_chosen": -105.87107849121094, | |
| "logps/policy_rejected": -148.20022583007812, | |
| "logps/reference_chosen": -112.85089111328125, | |
| "logps/reference_rejected": -155.7706298828125, | |
| "loss": 0.6832, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": 0.06979808956384659, | |
| "rewards/margins": -0.005905959755182266, | |
| "rewards/rejected": 0.07570404559373856, | |
| "step": 3, | |
| "tokens/chosen_response_len_mean": 239.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 254.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "grad_norm": 2.2808153966501084, | |
| "learning_rate": 1.6e-08, | |
| "logits/policy_chosen": -2.9396884441375732, | |
| "logits/policy_rejected": -2.9384472370147705, | |
| "logits/reference_chosen": -2.9226865768432617, | |
| "logits/reference_rejected": -2.9309639930725098, | |
| "logps/policy_chosen": -123.63784790039062, | |
| "logps/policy_rejected": -180.25296020507812, | |
| "logps/reference_chosen": -129.93817138671875, | |
| "logps/reference_rejected": -184.21334838867188, | |
| "loss": 0.674, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.06300327926874161, | |
| "rewards/margins": 0.023399492725729942, | |
| "rewards/rejected": 0.039603784680366516, | |
| "step": 4, | |
| "tokens/chosen_response_len_mean": 298.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 297.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "grad_norm": 2.2660235876486974, | |
| "learning_rate": 2e-08, | |
| "logits/policy_chosen": -3.221891164779663, | |
| "logits/policy_rejected": -2.8190152645111084, | |
| "logits/reference_chosen": -3.207385778427124, | |
| "logits/reference_rejected": -2.6983580589294434, | |
| "logps/policy_chosen": -179.60354614257812, | |
| "logps/policy_rejected": -206.77960205078125, | |
| "logps/reference_chosen": -189.15216064453125, | |
| "logps/reference_rejected": -209.81790161132812, | |
| "loss": 0.6778, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.09548593312501907, | |
| "rewards/margins": 0.06510297954082489, | |
| "rewards/rejected": 0.03038296103477478, | |
| "step": 5, | |
| "tokens/chosen_response_len_mean": 440.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 232.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "grad_norm": 2.0558578581082307, | |
| "learning_rate": 2.4e-08, | |
| "logits/policy_chosen": -2.4811384677886963, | |
| "logits/policy_rejected": -2.4067883491516113, | |
| "logits/reference_chosen": -2.330796003341675, | |
| "logits/reference_rejected": -2.4005446434020996, | |
| "logps/policy_chosen": -148.8319091796875, | |
| "logps/policy_rejected": -179.03482055664062, | |
| "logps/reference_chosen": -160.83175659179688, | |
| "logps/reference_rejected": -189.98275756835938, | |
| "loss": 0.6989, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.11999829858541489, | |
| "rewards/margins": 0.010519090108573437, | |
| "rewards/rejected": 0.10947921127080917, | |
| "step": 6, | |
| "tokens/chosen_response_len_mean": 289.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 309.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 2.5447256930451734, | |
| "learning_rate": 2.8e-08, | |
| "logits/policy_chosen": -2.2599525451660156, | |
| "logits/policy_rejected": -2.190964698791504, | |
| "logits/reference_chosen": -2.1052117347717285, | |
| "logits/reference_rejected": -2.134941577911377, | |
| "logps/policy_chosen": -19.338191986083984, | |
| "logps/policy_rejected": -38.73615646362305, | |
| "logps/reference_chosen": -20.665983200073242, | |
| "logps/reference_rejected": -40.615394592285156, | |
| "loss": 0.684, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.01327789481729269, | |
| "rewards/margins": -0.005514495074748993, | |
| "rewards/rejected": 0.018792390823364258, | |
| "step": 7, | |
| "tokens/chosen_response_len_mean": 34.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 47.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 2.967658233666017, | |
| "learning_rate": 3.2e-08, | |
| "logits/policy_chosen": -2.4715514183044434, | |
| "logits/policy_rejected": -2.4913477897644043, | |
| "logits/reference_chosen": -2.421693801879883, | |
| "logits/reference_rejected": -2.4608089923858643, | |
| "logps/policy_chosen": -29.782058715820312, | |
| "logps/policy_rejected": -64.6925048828125, | |
| "logps/reference_chosen": -32.50696563720703, | |
| "logps/reference_rejected": -66.7137222290039, | |
| "loss": 0.6712, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.027249068021774292, | |
| "rewards/margins": 0.00703694112598896, | |
| "rewards/rejected": 0.020212125033140182, | |
| "step": 8, | |
| "tokens/chosen_response_len_mean": 40.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 48.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 3.4470069507233316, | |
| "learning_rate": 3.6e-08, | |
| "logits/policy_chosen": -2.7727508544921875, | |
| "logits/policy_rejected": -2.718796730041504, | |
| "logits/reference_chosen": -2.676337957382202, | |
| "logits/reference_rejected": -2.616457223892212, | |
| "logps/policy_chosen": -108.26776123046875, | |
| "logps/policy_rejected": -218.6373748779297, | |
| "logps/reference_chosen": -118.67279815673828, | |
| "logps/reference_rejected": -223.27325439453125, | |
| "loss": 0.6796, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.1040503978729248, | |
| "rewards/margins": 0.05769149586558342, | |
| "rewards/rejected": 0.046358898282051086, | |
| "step": 9, | |
| "tokens/chosen_response_len_mean": 252.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 222.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 2.304016636338041, | |
| "learning_rate": 4e-08, | |
| "logits/policy_chosen": -2.5566985607147217, | |
| "logits/policy_rejected": -2.5395240783691406, | |
| "logits/reference_chosen": -2.4825587272644043, | |
| "logits/reference_rejected": -2.498563051223755, | |
| "logps/policy_chosen": -168.69143676757812, | |
| "logps/policy_rejected": -287.2906494140625, | |
| "logps/reference_chosen": -181.2764129638672, | |
| "logps/reference_rejected": -299.7208557128906, | |
| "loss": 0.6885, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": 0.1258496791124344, | |
| "rewards/margins": 0.001547420397400856, | |
| "rewards/rejected": 0.12430226802825928, | |
| "step": 10, | |
| "tokens/chosen_response_len_mean": 518.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 503.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 2.084799466815333, | |
| "learning_rate": 4.4e-08, | |
| "logits/policy_chosen": -2.7605199813842773, | |
| "logits/policy_rejected": -2.7186694145202637, | |
| "logits/reference_chosen": -2.7691569328308105, | |
| "logits/reference_rejected": -2.7842984199523926, | |
| "logps/policy_chosen": -145.88803100585938, | |
| "logps/policy_rejected": -219.43760681152344, | |
| "logps/reference_chosen": -155.90982055664062, | |
| "logps/reference_rejected": -227.8662109375, | |
| "loss": 0.6893, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.1002180427312851, | |
| "rewards/margins": 0.015932004898786545, | |
| "rewards/rejected": 0.08428603410720825, | |
| "step": 11, | |
| "tokens/chosen_response_len_mean": 604.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 448.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 2.7628214791467736, | |
| "learning_rate": 4.8e-08, | |
| "logits/policy_chosen": -2.9070513248443604, | |
| "logits/policy_rejected": -2.7778186798095703, | |
| "logits/reference_chosen": -2.8774163722991943, | |
| "logits/reference_rejected": -2.6446313858032227, | |
| "logps/policy_chosen": -101.27095794677734, | |
| "logps/policy_rejected": -136.12225341796875, | |
| "logps/reference_chosen": -108.19136047363281, | |
| "logps/reference_rejected": -140.3483123779297, | |
| "loss": 0.6673, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.06920397281646729, | |
| "rewards/margins": 0.026943188160657883, | |
| "rewards/rejected": 0.0422607846558094, | |
| "step": 12, | |
| "tokens/chosen_response_len_mean": 339.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 269.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 2.1795487992104348, | |
| "learning_rate": 5.1999999999999996e-08, | |
| "logits/policy_chosen": -2.4187445640563965, | |
| "logits/policy_rejected": -2.845353364944458, | |
| "logits/reference_chosen": -2.3343820571899414, | |
| "logits/reference_rejected": -2.5970327854156494, | |
| "logps/policy_chosen": -130.312255859375, | |
| "logps/policy_rejected": -73.26902770996094, | |
| "logps/reference_chosen": -134.85186767578125, | |
| "logps/reference_rejected": -73.45825958251953, | |
| "loss": 0.6772, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.045396123081445694, | |
| "rewards/margins": 0.04350382089614868, | |
| "rewards/rejected": 0.0018922993913292885, | |
| "step": 13, | |
| "tokens/chosen_response_len_mean": 177.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 101.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 2.7157502561420026, | |
| "learning_rate": 5.6e-08, | |
| "logits/policy_chosen": -3.211674928665161, | |
| "logits/policy_rejected": -3.0040595531463623, | |
| "logits/reference_chosen": -3.2025675773620605, | |
| "logits/reference_rejected": -3.0620803833007812, | |
| "logps/policy_chosen": -195.99533081054688, | |
| "logps/policy_rejected": -242.1226806640625, | |
| "logps/reference_chosen": -208.40536499023438, | |
| "logps/reference_rejected": -252.9475860595703, | |
| "loss": 0.6781, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.12410032004117966, | |
| "rewards/margins": 0.015851235017180443, | |
| "rewards/rejected": 0.10824908316135406, | |
| "step": 14, | |
| "tokens/chosen_response_len_mean": 543.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 408.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 3.50208846860952, | |
| "learning_rate": 6e-08, | |
| "logits/policy_chosen": -2.6727657318115234, | |
| "logits/policy_rejected": -2.5919432640075684, | |
| "logits/reference_chosen": -2.6382195949554443, | |
| "logits/reference_rejected": -2.417813777923584, | |
| "logps/policy_chosen": -170.3718719482422, | |
| "logps/policy_rejected": -223.43052673339844, | |
| "logps/reference_chosen": -191.32583618164062, | |
| "logps/reference_rejected": -242.33273315429688, | |
| "loss": 0.6686, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.209539532661438, | |
| "rewards/margins": 0.020517464727163315, | |
| "rewards/rejected": 0.18902206420898438, | |
| "step": 15, | |
| "tokens/chosen_response_len_mean": 431.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 403.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 1.6207728056732267, | |
| "learning_rate": 6.4e-08, | |
| "logits/policy_chosen": -2.954475164413452, | |
| "logits/policy_rejected": -2.8040027618408203, | |
| "logits/reference_chosen": -2.9069364070892334, | |
| "logits/reference_rejected": -2.8010973930358887, | |
| "logps/policy_chosen": -54.05670166015625, | |
| "logps/policy_rejected": -109.65800476074219, | |
| "logps/reference_chosen": -55.35580825805664, | |
| "logps/reference_rejected": -108.90874481201172, | |
| "loss": 0.6824, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.012991041876375675, | |
| "rewards/margins": 0.02048366516828537, | |
| "rewards/rejected": -0.007492623291909695, | |
| "step": 16, | |
| "tokens/chosen_response_len_mean": 89.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 96.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 2.409299627131989, | |
| "learning_rate": 6.8e-08, | |
| "logits/policy_chosen": -2.909968137741089, | |
| "logits/policy_rejected": -2.84816837310791, | |
| "logits/reference_chosen": -2.8810513019561768, | |
| "logits/reference_rejected": -2.6653146743774414, | |
| "logps/policy_chosen": -119.80767822265625, | |
| "logps/policy_rejected": -267.8531799316406, | |
| "logps/reference_chosen": -126.68125915527344, | |
| "logps/reference_rejected": -275.761474609375, | |
| "loss": 0.6995, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.06873579323291779, | |
| "rewards/margins": -0.010347230359911919, | |
| "rewards/rejected": 0.07908301800489426, | |
| "step": 17, | |
| "tokens/chosen_response_len_mean": 314.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 251.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 2.703021449933624, | |
| "learning_rate": 7.2e-08, | |
| "logits/policy_chosen": -2.8886208534240723, | |
| "logits/policy_rejected": -2.603667974472046, | |
| "logits/reference_chosen": -2.8919739723205566, | |
| "logits/reference_rejected": -2.336148262023926, | |
| "logps/policy_chosen": -144.8170166015625, | |
| "logps/policy_rejected": -144.6788330078125, | |
| "logps/reference_chosen": -164.3677215576172, | |
| "logps/reference_rejected": -156.78842163085938, | |
| "loss": 0.679, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.1955069750547409, | |
| "rewards/margins": 0.07441116869449615, | |
| "rewards/rejected": 0.12109580636024475, | |
| "step": 18, | |
| "tokens/chosen_response_len_mean": 421.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 314.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 1.9691382131351773, | |
| "learning_rate": 7.599999999999999e-08, | |
| "logits/policy_chosen": -2.766261339187622, | |
| "logits/policy_rejected": -2.697242259979248, | |
| "logits/reference_chosen": -2.710810899734497, | |
| "logits/reference_rejected": -2.557218551635742, | |
| "logps/policy_chosen": -150.92379760742188, | |
| "logps/policy_rejected": -174.591064453125, | |
| "logps/reference_chosen": -160.51756286621094, | |
| "logps/reference_rejected": -185.50238037109375, | |
| "loss": 0.6851, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.0959378108382225, | |
| "rewards/margins": -0.013175351545214653, | |
| "rewards/rejected": 0.1091131642460823, | |
| "step": 19, | |
| "tokens/chosen_response_len_mean": 263.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 253.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 2.1003754961271675, | |
| "learning_rate": 8e-08, | |
| "logits/policy_chosen": -2.8774542808532715, | |
| "logits/policy_rejected": -2.6342382431030273, | |
| "logits/reference_chosen": -2.742877244949341, | |
| "logits/reference_rejected": -2.4385361671447754, | |
| "logps/policy_chosen": -104.96978759765625, | |
| "logps/policy_rejected": -157.80657958984375, | |
| "logps/reference_chosen": -107.06401062011719, | |
| "logps/reference_rejected": -159.94366455078125, | |
| "loss": 0.6774, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.02094227820634842, | |
| "rewards/margins": -0.00042874296195805073, | |
| "rewards/rejected": 0.02137102000415325, | |
| "step": 20, | |
| "tokens/chosen_response_len_mean": 111.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 124.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 2.113555044323313, | |
| "learning_rate": 8.4e-08, | |
| "logits/policy_chosen": -3.1314477920532227, | |
| "logits/policy_rejected": -2.8576135635375977, | |
| "logits/reference_chosen": -3.1075565814971924, | |
| "logits/reference_rejected": -2.6684985160827637, | |
| "logps/policy_chosen": -121.34175109863281, | |
| "logps/policy_rejected": -152.86807250976562, | |
| "logps/reference_chosen": -127.80372619628906, | |
| "logps/reference_rejected": -158.0865478515625, | |
| "loss": 0.6994, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.06461974233388901, | |
| "rewards/margins": 0.012434981763362885, | |
| "rewards/rejected": 0.05218476057052612, | |
| "step": 21, | |
| "tokens/chosen_response_len_mean": 317.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 203.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 2.8572997186380196, | |
| "learning_rate": 8.8e-08, | |
| "logits/policy_chosen": -3.023026466369629, | |
| "logits/policy_rejected": -2.970913887023926, | |
| "logits/reference_chosen": -2.9523911476135254, | |
| "logits/reference_rejected": -2.8476057052612305, | |
| "logps/policy_chosen": -123.84272003173828, | |
| "logps/policy_rejected": -210.03341674804688, | |
| "logps/reference_chosen": -129.73898315429688, | |
| "logps/reference_rejected": -216.26409912109375, | |
| "loss": 0.672, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": 0.05896264314651489, | |
| "rewards/margins": -0.0033443691208958626, | |
| "rewards/rejected": 0.06230701133608818, | |
| "step": 22, | |
| "tokens/chosen_response_len_mean": 241.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 224.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 2.4344792354730034, | |
| "learning_rate": 9.199999999999999e-08, | |
| "logits/policy_chosen": -2.45721435546875, | |
| "logits/policy_rejected": -2.4645514488220215, | |
| "logits/reference_chosen": -2.429760694503784, | |
| "logits/reference_rejected": -2.448065757751465, | |
| "logps/policy_chosen": -107.37543487548828, | |
| "logps/policy_rejected": -120.32754516601562, | |
| "logps/reference_chosen": -113.06903839111328, | |
| "logps/reference_rejected": -126.56575012207031, | |
| "loss": 0.6748, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": 0.056935906410217285, | |
| "rewards/margins": -0.005446167662739754, | |
| "rewards/rejected": 0.06238207593560219, | |
| "step": 23, | |
| "tokens/chosen_response_len_mean": 314.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 315.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 3.5341818462448504, | |
| "learning_rate": 9.6e-08, | |
| "logits/policy_chosen": -2.5186548233032227, | |
| "logits/policy_rejected": -2.3946735858917236, | |
| "logits/reference_chosen": -2.5037264823913574, | |
| "logits/reference_rejected": -2.4204611778259277, | |
| "logps/policy_chosen": -71.1528549194336, | |
| "logps/policy_rejected": -96.15353393554688, | |
| "logps/reference_chosen": -74.45181274414062, | |
| "logps/reference_rejected": -93.63554382324219, | |
| "loss": 0.6593, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.03298955038189888, | |
| "rewards/margins": 0.05816952511668205, | |
| "rewards/rejected": -0.025179967284202576, | |
| "step": 24, | |
| "tokens/chosen_response_len_mean": 247.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 155.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 2.6672688539385514, | |
| "learning_rate": 1e-07, | |
| "logits/policy_chosen": -3.052779197692871, | |
| "logits/policy_rejected": -2.9980740547180176, | |
| "logits/reference_chosen": -3.0356476306915283, | |
| "logits/reference_rejected": -2.9704558849334717, | |
| "logps/policy_chosen": -158.24560546875, | |
| "logps/policy_rejected": -246.05154418945312, | |
| "logps/reference_chosen": -166.8022003173828, | |
| "logps/reference_rejected": -252.77267456054688, | |
| "loss": 0.6801, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.08556582033634186, | |
| "rewards/margins": 0.01835446059703827, | |
| "rewards/rejected": 0.06721135973930359, | |
| "step": 25, | |
| "tokens/chosen_response_len_mean": 302.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 309.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 2.157863179356832, | |
| "learning_rate": 1.0399999999999999e-07, | |
| "logits/policy_chosen": -2.8244199752807617, | |
| "logits/policy_rejected": -2.8272688388824463, | |
| "logits/reference_chosen": -2.8120579719543457, | |
| "logits/reference_rejected": -2.81868839263916, | |
| "logps/policy_chosen": -188.75546264648438, | |
| "logps/policy_rejected": -245.15794372558594, | |
| "logps/reference_chosen": -201.83450317382812, | |
| "logps/reference_rejected": -256.3631591796875, | |
| "loss": 0.6824, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.13079024851322174, | |
| "rewards/margins": 0.018737995997071266, | |
| "rewards/rejected": 0.11205224692821503, | |
| "step": 26, | |
| "tokens/chosen_response_len_mean": 499.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 500.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 2.6785946054818295, | |
| "learning_rate": 1.0799999999999999e-07, | |
| "logits/policy_chosen": -3.041701078414917, | |
| "logits/policy_rejected": -3.0683014392852783, | |
| "logits/reference_chosen": -3.021995782852173, | |
| "logits/reference_rejected": -3.0257372856140137, | |
| "logps/policy_chosen": -102.09321594238281, | |
| "logps/policy_rejected": -163.72860717773438, | |
| "logps/reference_chosen": -109.2179946899414, | |
| "logps/reference_rejected": -167.15447998046875, | |
| "loss": 0.6755, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.07124768942594528, | |
| "rewards/margins": 0.036988936364650726, | |
| "rewards/rejected": 0.03425874561071396, | |
| "step": 27, | |
| "tokens/chosen_response_len_mean": 243.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 237.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 3.0106278359730614, | |
| "learning_rate": 1.12e-07, | |
| "logits/policy_chosen": -3.058267116546631, | |
| "logits/policy_rejected": -3.0097880363464355, | |
| "logits/reference_chosen": -3.000871181488037, | |
| "logits/reference_rejected": -2.860621690750122, | |
| "logps/policy_chosen": -177.05154418945312, | |
| "logps/policy_rejected": -241.18173217773438, | |
| "logps/reference_chosen": -187.62625122070312, | |
| "logps/reference_rejected": -242.83314514160156, | |
| "loss": 0.6642, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.10574702173471451, | |
| "rewards/margins": 0.08923279494047165, | |
| "rewards/rejected": 0.01651422120630741, | |
| "step": 28, | |
| "tokens/chosen_response_len_mean": 391.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 363.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 1.9714081999119137, | |
| "learning_rate": 1.16e-07, | |
| "logits/policy_chosen": -2.7036781311035156, | |
| "logits/policy_rejected": -2.706456422805786, | |
| "logits/reference_chosen": -2.6821694374084473, | |
| "logits/reference_rejected": -2.70880126953125, | |
| "logps/policy_chosen": -50.51161193847656, | |
| "logps/policy_rejected": -97.51376342773438, | |
| "logps/reference_chosen": -53.98440170288086, | |
| "logps/reference_rejected": -100.45523834228516, | |
| "loss": 0.6839, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.03472791612148285, | |
| "rewards/margins": 0.005313164554536343, | |
| "rewards/rejected": 0.029414750635623932, | |
| "step": 29, | |
| "tokens/chosen_response_len_mean": 246.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 215.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 3.3945073696477333, | |
| "learning_rate": 1.2e-07, | |
| "logits/policy_chosen": -2.679271936416626, | |
| "logits/policy_rejected": -2.747288227081299, | |
| "logits/reference_chosen": -2.648017406463623, | |
| "logits/reference_rejected": -2.542226552963257, | |
| "logps/policy_chosen": -181.09521484375, | |
| "logps/policy_rejected": -349.05694580078125, | |
| "logps/reference_chosen": -202.54962158203125, | |
| "logps/reference_rejected": -355.8513488769531, | |
| "loss": 0.6558, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.21454395353794098, | |
| "rewards/margins": 0.14660006761550903, | |
| "rewards/rejected": 0.06794389337301254, | |
| "step": 30, | |
| "tokens/chosen_response_len_mean": 309.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 268.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 2.729481181496147, | |
| "learning_rate": 1.24e-07, | |
| "logits/policy_chosen": -3.1615757942199707, | |
| "logits/policy_rejected": -2.618534564971924, | |
| "logits/reference_chosen": -3.148972511291504, | |
| "logits/reference_rejected": -2.28782320022583, | |
| "logps/policy_chosen": -244.06692504882812, | |
| "logps/policy_rejected": -395.2162780761719, | |
| "logps/reference_chosen": -254.54090881347656, | |
| "logps/reference_rejected": -397.2469177246094, | |
| "loss": 0.6873, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.10473988205194473, | |
| "rewards/margins": 0.08443356305360794, | |
| "rewards/rejected": 0.020306318998336792, | |
| "step": 31, | |
| "tokens/chosen_response_len_mean": 410.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 269.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 2.1927971055522133, | |
| "learning_rate": 1.28e-07, | |
| "logits/policy_chosen": -2.935199499130249, | |
| "logits/policy_rejected": -2.862617254257202, | |
| "logits/reference_chosen": -2.9165947437286377, | |
| "logits/reference_rejected": -2.809278964996338, | |
| "logps/policy_chosen": -156.64752197265625, | |
| "logps/policy_rejected": -224.47006225585938, | |
| "logps/reference_chosen": -176.1259765625, | |
| "logps/reference_rejected": -239.154296875, | |
| "loss": 0.6749, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.19478438794612885, | |
| "rewards/margins": 0.04794217646121979, | |
| "rewards/rejected": 0.14684221148490906, | |
| "step": 32, | |
| "tokens/chosen_response_len_mean": 545.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 527.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 2.6358653856327257, | |
| "learning_rate": 1.32e-07, | |
| "logits/policy_chosen": -2.5262393951416016, | |
| "logits/policy_rejected": -2.364823341369629, | |
| "logits/reference_chosen": -2.506678342819214, | |
| "logits/reference_rejected": -1.6685891151428223, | |
| "logps/policy_chosen": -104.21517181396484, | |
| "logps/policy_rejected": -126.63694763183594, | |
| "logps/reference_chosen": -114.83543395996094, | |
| "logps/reference_rejected": -136.51809692382812, | |
| "loss": 0.6837, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.10620267689228058, | |
| "rewards/margins": 0.007391175255179405, | |
| "rewards/rejected": 0.09881150722503662, | |
| "step": 33, | |
| "tokens/chosen_response_len_mean": 448.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 339.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 2.626247472893481, | |
| "learning_rate": 1.36e-07, | |
| "logits/policy_chosen": -3.0164244174957275, | |
| "logits/policy_rejected": -2.670468330383301, | |
| "logits/reference_chosen": -2.9480087757110596, | |
| "logits/reference_rejected": -2.077363967895508, | |
| "logps/policy_chosen": -144.32318115234375, | |
| "logps/policy_rejected": -178.09408569335938, | |
| "logps/reference_chosen": -154.3887939453125, | |
| "logps/reference_rejected": -182.9420623779297, | |
| "loss": 0.6748, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.1006559506058693, | |
| "rewards/margins": 0.052176106721162796, | |
| "rewards/rejected": 0.0484798438847065, | |
| "step": 34, | |
| "tokens/chosen_response_len_mean": 407.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 217.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 2.296064376513374, | |
| "learning_rate": 1.4e-07, | |
| "logits/policy_chosen": -2.598402738571167, | |
| "logits/policy_rejected": -2.6598832607269287, | |
| "logits/reference_chosen": -2.553514003753662, | |
| "logits/reference_rejected": -2.6580069065093994, | |
| "logps/policy_chosen": -281.2328186035156, | |
| "logps/policy_rejected": -179.6129608154297, | |
| "logps/reference_chosen": -298.13836669921875, | |
| "logps/reference_rejected": -186.5122528076172, | |
| "loss": 0.6684, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.16905567049980164, | |
| "rewards/margins": 0.10006258636713028, | |
| "rewards/rejected": 0.06899307668209076, | |
| "step": 35, | |
| "tokens/chosen_response_len_mean": 377.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 248.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 2.898376793238501, | |
| "learning_rate": 1.44e-07, | |
| "logits/policy_chosen": -2.7883753776550293, | |
| "logits/policy_rejected": -2.6082558631896973, | |
| "logits/reference_chosen": -2.780714273452759, | |
| "logits/reference_rejected": -2.583864212036133, | |
| "logps/policy_chosen": -105.63471984863281, | |
| "logps/policy_rejected": -245.55908203125, | |
| "logps/reference_chosen": -110.00314331054688, | |
| "logps/reference_rejected": -244.04986572265625, | |
| "loss": 0.6715, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.04368434101343155, | |
| "rewards/margins": 0.05877665430307388, | |
| "rewards/rejected": -0.015092315152287483, | |
| "step": 36, | |
| "tokens/chosen_response_len_mean": 183.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 284.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 2.6935110955386, | |
| "learning_rate": 1.4799999999999998e-07, | |
| "logits/policy_chosen": -3.0653228759765625, | |
| "logits/policy_rejected": -2.882253646850586, | |
| "logits/reference_chosen": -3.0485243797302246, | |
| "logits/reference_rejected": -2.6053519248962402, | |
| "logps/policy_chosen": -186.7164764404297, | |
| "logps/policy_rejected": -267.58123779296875, | |
| "logps/reference_chosen": -204.80267333984375, | |
| "logps/reference_rejected": -272.190185546875, | |
| "loss": 0.6697, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.1808619499206543, | |
| "rewards/margins": 0.13477247953414917, | |
| "rewards/rejected": 0.046089474111795425, | |
| "step": 37, | |
| "tokens/chosen_response_len_mean": 433.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 321.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 2.597922955352179, | |
| "learning_rate": 1.5199999999999998e-07, | |
| "logits/policy_chosen": -2.699632167816162, | |
| "logits/policy_rejected": -2.7144150733947754, | |
| "logits/reference_chosen": -2.658318519592285, | |
| "logits/reference_rejected": -2.691765546798706, | |
| "logps/policy_chosen": -135.453369140625, | |
| "logps/policy_rejected": -242.96441650390625, | |
| "logps/reference_chosen": -146.15704345703125, | |
| "logps/reference_rejected": -248.74380493164062, | |
| "loss": 0.6585, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.10703684389591217, | |
| "rewards/margins": 0.049242909997701645, | |
| "rewards/rejected": 0.057793937623500824, | |
| "step": 38, | |
| "tokens/chosen_response_len_mean": 375.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 387.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 2.079049499920605, | |
| "learning_rate": 1.56e-07, | |
| "logits/policy_chosen": -3.077770233154297, | |
| "logits/policy_rejected": -3.098449945449829, | |
| "logits/reference_chosen": -3.0287766456604004, | |
| "logits/reference_rejected": -3.0804648399353027, | |
| "logps/policy_chosen": -12.474931716918945, | |
| "logps/policy_rejected": -25.898088455200195, | |
| "logps/reference_chosen": -12.915287971496582, | |
| "logps/reference_rejected": -27.526098251342773, | |
| "loss": 0.6793, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": 0.00440356507897377, | |
| "rewards/margins": -0.011876537464559078, | |
| "rewards/rejected": 0.016280101612210274, | |
| "step": 39, | |
| "tokens/chosen_response_len_mean": 23.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 25.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 2.9270192927802197, | |
| "learning_rate": 1.6e-07, | |
| "logits/policy_chosen": -2.821016550064087, | |
| "logits/policy_rejected": -2.7184388637542725, | |
| "logits/reference_chosen": -2.5926756858825684, | |
| "logits/reference_rejected": -2.7159690856933594, | |
| "logps/policy_chosen": -115.9990005493164, | |
| "logps/policy_rejected": -150.73153686523438, | |
| "logps/reference_chosen": -120.78485107421875, | |
| "logps/reference_rejected": -157.8815460205078, | |
| "loss": 0.668, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.04785853251814842, | |
| "rewards/margins": -0.02364155277609825, | |
| "rewards/rejected": 0.07150008529424667, | |
| "step": 40, | |
| "tokens/chosen_response_len_mean": 179.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 168.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 2.807071363383845, | |
| "learning_rate": 1.64e-07, | |
| "logits/policy_chosen": -2.6167349815368652, | |
| "logits/policy_rejected": -2.5763139724731445, | |
| "logits/reference_chosen": -2.513126850128174, | |
| "logits/reference_rejected": -2.5481503009796143, | |
| "logps/policy_chosen": -213.66064453125, | |
| "logps/policy_rejected": -274.35003662109375, | |
| "logps/reference_chosen": -240.58090209960938, | |
| "logps/reference_rejected": -297.9310607910156, | |
| "loss": 0.6847, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.26920264959335327, | |
| "rewards/margins": 0.03339245170354843, | |
| "rewards/rejected": 0.23581019043922424, | |
| "step": 41, | |
| "tokens/chosen_response_len_mean": 468.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 442.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 3.1342954098759788, | |
| "learning_rate": 1.68e-07, | |
| "logits/policy_chosen": -2.98238205909729, | |
| "logits/policy_rejected": -2.6742758750915527, | |
| "logits/reference_chosen": -2.9786105155944824, | |
| "logits/reference_rejected": -2.5011706352233887, | |
| "logps/policy_chosen": -400.793701171875, | |
| "logps/policy_rejected": -471.21826171875, | |
| "logps/reference_chosen": -424.1178283691406, | |
| "logps/reference_rejected": -484.6050720214844, | |
| "loss": 0.6671, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.23324130475521088, | |
| "rewards/margins": 0.09937307238578796, | |
| "rewards/rejected": 0.1338682323694229, | |
| "step": 42, | |
| "tokens/chosen_response_len_mean": 593.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 479.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 1.7422493551589444, | |
| "learning_rate": 1.7199999999999998e-07, | |
| "logits/policy_chosen": -2.763662338256836, | |
| "logits/policy_rejected": -2.434535503387451, | |
| "logits/reference_chosen": -2.710179567337036, | |
| "logits/reference_rejected": -1.868179440498352, | |
| "logps/policy_chosen": -77.93124389648438, | |
| "logps/policy_rejected": -110.21360778808594, | |
| "logps/reference_chosen": -86.2520751953125, | |
| "logps/reference_rejected": -110.11550903320312, | |
| "loss": 0.679, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.08320826292037964, | |
| "rewards/margins": 0.08418922126293182, | |
| "rewards/rejected": -0.0009809592738747597, | |
| "step": 43, | |
| "tokens/chosen_response_len_mean": 152.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 85.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 3.138281711904061, | |
| "learning_rate": 1.76e-07, | |
| "logits/policy_chosen": -2.8771750926971436, | |
| "logits/policy_rejected": -2.9175660610198975, | |
| "logits/reference_chosen": -2.8088254928588867, | |
| "logits/reference_rejected": -2.877462863922119, | |
| "logps/policy_chosen": -37.14703369140625, | |
| "logps/policy_rejected": -55.703121185302734, | |
| "logps/reference_chosen": -38.25192642211914, | |
| "logps/reference_rejected": -56.50023651123047, | |
| "loss": 0.686, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.011048920452594757, | |
| "rewards/margins": 0.0030777817592024803, | |
| "rewards/rejected": 0.007971138693392277, | |
| "step": 44, | |
| "tokens/chosen_response_len_mean": 66.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 70.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 3.284990775967774, | |
| "learning_rate": 1.8e-07, | |
| "logits/policy_chosen": -2.8581864833831787, | |
| "logits/policy_rejected": -2.766713857650757, | |
| "logits/reference_chosen": -2.8522939682006836, | |
| "logits/reference_rejected": -2.7666430473327637, | |
| "logps/policy_chosen": -165.7818603515625, | |
| "logps/policy_rejected": -241.84002685546875, | |
| "logps/reference_chosen": -181.072998046875, | |
| "logps/reference_rejected": -250.3277587890625, | |
| "loss": 0.6537, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.15291133522987366, | |
| "rewards/margins": 0.06803382933139801, | |
| "rewards/rejected": 0.08487750589847565, | |
| "step": 45, | |
| "tokens/chosen_response_len_mean": 560.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 473.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 2.463717582717747, | |
| "learning_rate": 1.8399999999999998e-07, | |
| "logits/policy_chosen": -2.900191068649292, | |
| "logits/policy_rejected": -2.9184107780456543, | |
| "logits/reference_chosen": -2.8673532009124756, | |
| "logits/reference_rejected": -2.892207145690918, | |
| "logps/policy_chosen": -51.21653747558594, | |
| "logps/policy_rejected": -107.85240936279297, | |
| "logps/reference_chosen": -54.76094436645508, | |
| "logps/reference_rejected": -117.80521392822266, | |
| "loss": 0.695, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.035444095730781555, | |
| "rewards/margins": -0.06408397108316422, | |
| "rewards/rejected": 0.09952806681394577, | |
| "step": 46, | |
| "tokens/chosen_response_len_mean": 109.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 108.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 1.9738986795582658, | |
| "learning_rate": 1.88e-07, | |
| "logits/policy_chosen": -2.2502527236938477, | |
| "logits/policy_rejected": -2.245668649673462, | |
| "logits/reference_chosen": -2.177823781967163, | |
| "logits/reference_rejected": -2.223409652709961, | |
| "logps/policy_chosen": -37.820743560791016, | |
| "logps/policy_rejected": -99.87039184570312, | |
| "logps/reference_chosen": -44.7762451171875, | |
| "logps/reference_rejected": -104.45420837402344, | |
| "loss": 0.6825, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.06955502927303314, | |
| "rewards/margins": 0.023716777563095093, | |
| "rewards/rejected": 0.04583824798464775, | |
| "step": 47, | |
| "tokens/chosen_response_len_mean": 93.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 110.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 2.8813658188320335, | |
| "learning_rate": 1.92e-07, | |
| "logits/policy_chosen": -2.6205151081085205, | |
| "logits/policy_rejected": -2.5673413276672363, | |
| "logits/reference_chosen": -2.5005288124084473, | |
| "logits/reference_rejected": -2.561748504638672, | |
| "logps/policy_chosen": -65.36790466308594, | |
| "logps/policy_rejected": -167.23562622070312, | |
| "logps/reference_chosen": -72.64520263671875, | |
| "logps/reference_rejected": -172.6849365234375, | |
| "loss": 0.6706, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": 0.07277299463748932, | |
| "rewards/margins": 0.018280120566487312, | |
| "rewards/rejected": 0.054492875933647156, | |
| "step": 48, | |
| "tokens/chosen_response_len_mean": 199.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 209.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 2.862318181422901, | |
| "learning_rate": 1.96e-07, | |
| "logits/policy_chosen": -2.701411247253418, | |
| "logits/policy_rejected": -2.8611505031585693, | |
| "logits/reference_chosen": -2.2373385429382324, | |
| "logits/reference_rejected": -2.845200777053833, | |
| "logps/policy_chosen": -126.9096908569336, | |
| "logps/policy_rejected": -269.68243408203125, | |
| "logps/reference_chosen": -136.80958557128906, | |
| "logps/reference_rejected": -277.586181640625, | |
| "loss": 0.6796, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.09899893403053284, | |
| "rewards/margins": 0.01996154710650444, | |
| "rewards/rejected": 0.0790373906493187, | |
| "step": 49, | |
| "tokens/chosen_response_len_mean": 262.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 337.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 2.324007938345189, | |
| "learning_rate": 2e-07, | |
| "logits/policy_chosen": -2.2418417930603027, | |
| "logits/policy_rejected": -2.1697628498077393, | |
| "logits/reference_chosen": -2.142324924468994, | |
| "logits/reference_rejected": -2.1179537773132324, | |
| "logps/policy_chosen": -136.041259765625, | |
| "logps/policy_rejected": -266.0906677246094, | |
| "logps/reference_chosen": -151.6305694580078, | |
| "logps/reference_rejected": -273.1885986328125, | |
| "loss": 0.6677, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.15589313209056854, | |
| "rewards/margins": 0.08491363376379013, | |
| "rewards/rejected": 0.07097949832677841, | |
| "step": 50, | |
| "tokens/chosen_response_len_mean": 526.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 469.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 2.3504496124001744, | |
| "learning_rate": 2.0399999999999997e-07, | |
| "logits/policy_chosen": -2.899348497390747, | |
| "logits/policy_rejected": -2.8138720989227295, | |
| "logits/reference_chosen": -2.8722875118255615, | |
| "logits/reference_rejected": -2.8029227256774902, | |
| "logps/policy_chosen": -63.48197555541992, | |
| "logps/policy_rejected": -101.47474670410156, | |
| "logps/reference_chosen": -68.96565246582031, | |
| "logps/reference_rejected": -108.34456634521484, | |
| "loss": 0.6831, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": 0.05483681336045265, | |
| "rewards/margins": -0.013861382380127907, | |
| "rewards/rejected": 0.06869819760322571, | |
| "step": 51, | |
| "tokens/chosen_response_len_mean": 212.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 181.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 2.4664067603984665, | |
| "learning_rate": 2.0799999999999998e-07, | |
| "logits/policy_chosen": -2.710914134979248, | |
| "logits/policy_rejected": -2.6489503383636475, | |
| "logits/reference_chosen": -2.6887543201446533, | |
| "logits/reference_rejected": -2.565074920654297, | |
| "logps/policy_chosen": -120.6403579711914, | |
| "logps/policy_rejected": -180.31842041015625, | |
| "logps/reference_chosen": -129.82321166992188, | |
| "logps/reference_rejected": -187.31112670898438, | |
| "loss": 0.6824, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.0918285921216011, | |
| "rewards/margins": 0.021901624277234077, | |
| "rewards/rejected": 0.06992696225643158, | |
| "step": 52, | |
| "tokens/chosen_response_len_mean": 340.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 269.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 3.7734473565962108, | |
| "learning_rate": 2.12e-07, | |
| "logits/policy_chosen": -2.9076874256134033, | |
| "logits/policy_rejected": -2.8471598625183105, | |
| "logits/reference_chosen": -2.837496280670166, | |
| "logits/reference_rejected": -2.783947467803955, | |
| "logps/policy_chosen": -88.39242553710938, | |
| "logps/policy_rejected": -232.25318908691406, | |
| "logps/reference_chosen": -96.45068359375, | |
| "logps/reference_rejected": -237.35009765625, | |
| "loss": 0.6784, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.08058254420757294, | |
| "rewards/margins": 0.02961355820298195, | |
| "rewards/rejected": 0.05096898972988129, | |
| "step": 53, | |
| "tokens/chosen_response_len_mean": 195.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 194.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 2.8319761915445367, | |
| "learning_rate": 2.1599999999999998e-07, | |
| "logits/policy_chosen": -3.144252300262451, | |
| "logits/policy_rejected": -3.0141947269439697, | |
| "logits/reference_chosen": -3.116751194000244, | |
| "logits/reference_rejected": -2.8330681324005127, | |
| "logps/policy_chosen": -132.089111328125, | |
| "logps/policy_rejected": -225.1402587890625, | |
| "logps/reference_chosen": -136.70138549804688, | |
| "logps/reference_rejected": -224.64724731445312, | |
| "loss": 0.6737, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.046122774481773376, | |
| "rewards/margins": 0.05105280876159668, | |
| "rewards/rejected": -0.004930033814162016, | |
| "step": 54, | |
| "tokens/chosen_response_len_mean": 299.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 257.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 2.947847201514034, | |
| "learning_rate": 2.1999999999999998e-07, | |
| "logits/policy_chosen": -2.8887228965759277, | |
| "logits/policy_rejected": -2.68259596824646, | |
| "logits/reference_chosen": -2.864752769470215, | |
| "logits/reference_rejected": -2.628319263458252, | |
| "logps/policy_chosen": -135.53025817871094, | |
| "logps/policy_rejected": -133.9132843017578, | |
| "logps/reference_chosen": -145.9614715576172, | |
| "logps/reference_rejected": -137.02508544921875, | |
| "loss": 0.666, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.10431211441755295, | |
| "rewards/margins": 0.07319390773773193, | |
| "rewards/rejected": 0.031118206679821014, | |
| "step": 55, | |
| "tokens/chosen_response_len_mean": 349.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 196.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 2.771906040366205, | |
| "learning_rate": 2.24e-07, | |
| "logits/policy_chosen": -2.231527328491211, | |
| "logits/policy_rejected": -2.0439558029174805, | |
| "logits/reference_chosen": -2.230081796646118, | |
| "logits/reference_rejected": -2.080432176589966, | |
| "logps/policy_chosen": -399.80303955078125, | |
| "logps/policy_rejected": -446.4791259765625, | |
| "logps/reference_chosen": -431.472412109375, | |
| "logps/reference_rejected": -470.2456970214844, | |
| "loss": 0.6709, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.31669357419013977, | |
| "rewards/margins": 0.07902783155441284, | |
| "rewards/rejected": 0.23766574263572693, | |
| "step": 56, | |
| "tokens/chosen_response_len_mean": 805.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 711.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 1.7428598517814506, | |
| "learning_rate": 2.28e-07, | |
| "logits/policy_chosen": -2.7684319019317627, | |
| "logits/policy_rejected": -2.71665096282959, | |
| "logits/reference_chosen": -2.747512102127075, | |
| "logits/reference_rejected": -2.690124034881592, | |
| "logps/policy_chosen": -85.4046630859375, | |
| "logps/policy_rejected": -111.96269226074219, | |
| "logps/reference_chosen": -96.96040344238281, | |
| "logps/reference_rejected": -121.33839416503906, | |
| "loss": 0.6888, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.11555740237236023, | |
| "rewards/margins": 0.021800296381115913, | |
| "rewards/rejected": 0.09375710785388947, | |
| "step": 57, | |
| "tokens/chosen_response_len_mean": 261.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 248.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 2.6040542171359817, | |
| "learning_rate": 2.32e-07, | |
| "logits/policy_chosen": -2.989000082015991, | |
| "logits/policy_rejected": -2.8631539344787598, | |
| "logits/reference_chosen": -2.9670627117156982, | |
| "logits/reference_rejected": -2.5129096508026123, | |
| "logps/policy_chosen": -87.29337310791016, | |
| "logps/policy_rejected": -119.82539367675781, | |
| "logps/reference_chosen": -95.52102661132812, | |
| "logps/reference_rejected": -122.27404022216797, | |
| "loss": 0.6657, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.08227652311325073, | |
| "rewards/margins": 0.05779005587100983, | |
| "rewards/rejected": 0.024486469104886055, | |
| "step": 58, | |
| "tokens/chosen_response_len_mean": 162.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 138.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 3.249420407772776, | |
| "learning_rate": 2.3599999999999997e-07, | |
| "logits/policy_chosen": -2.9996423721313477, | |
| "logits/policy_rejected": -2.974095582962036, | |
| "logits/reference_chosen": -2.964367151260376, | |
| "logits/reference_rejected": -2.953282356262207, | |
| "logps/policy_chosen": -56.97121810913086, | |
| "logps/policy_rejected": -141.20883178710938, | |
| "logps/reference_chosen": -58.637413024902344, | |
| "logps/reference_rejected": -143.08058166503906, | |
| "loss": 0.679, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.0166619960218668, | |
| "rewards/margins": -0.0020555637311190367, | |
| "rewards/rejected": 0.018717560917139053, | |
| "step": 59, | |
| "tokens/chosen_response_len_mean": 121.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 127.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 3.503891824715986, | |
| "learning_rate": 2.4e-07, | |
| "logits/policy_chosen": -3.207751512527466, | |
| "logits/policy_rejected": -2.8613648414611816, | |
| "logits/reference_chosen": -3.1973516941070557, | |
| "logits/reference_rejected": -2.244464874267578, | |
| "logps/policy_chosen": -115.4853286743164, | |
| "logps/policy_rejected": -205.64161682128906, | |
| "logps/reference_chosen": -123.15602111816406, | |
| "logps/reference_rejected": -206.07244873046875, | |
| "loss": 0.6713, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.0767069011926651, | |
| "rewards/margins": 0.07239863276481628, | |
| "rewards/rejected": 0.004308272153139114, | |
| "step": 60, | |
| "tokens/chosen_response_len_mean": 262.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 185.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 2.987682806191956, | |
| "learning_rate": 2.4399999999999996e-07, | |
| "logits/policy_chosen": -2.65006947517395, | |
| "logits/policy_rejected": -2.60097599029541, | |
| "logits/reference_chosen": -2.6217522621154785, | |
| "logits/reference_rejected": -2.507542610168457, | |
| "logps/policy_chosen": -184.2285614013672, | |
| "logps/policy_rejected": -184.65708923339844, | |
| "logps/reference_chosen": -215.3209228515625, | |
| "logps/reference_rejected": -187.43243408203125, | |
| "loss": 0.6592, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.31092363595962524, | |
| "rewards/margins": 0.2831703722476959, | |
| "rewards/rejected": 0.027753248810768127, | |
| "step": 61, | |
| "tokens/chosen_response_len_mean": 344.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 292.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 3.02187794129979, | |
| "learning_rate": 2.48e-07, | |
| "logits/policy_chosen": -3.102184534072876, | |
| "logits/policy_rejected": -3.0169076919555664, | |
| "logits/reference_chosen": -3.0980794429779053, | |
| "logits/reference_rejected": -3.06693434715271, | |
| "logps/policy_chosen": -180.20120239257812, | |
| "logps/policy_rejected": -290.76556396484375, | |
| "logps/reference_chosen": -186.42239379882812, | |
| "logps/reference_rejected": -299.5263977050781, | |
| "loss": 0.6663, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.06221204251050949, | |
| "rewards/margins": -0.025396330282092094, | |
| "rewards/rejected": 0.08760837465524673, | |
| "step": 62, | |
| "tokens/chosen_response_len_mean": 628.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 573.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 2.2026851566699364, | |
| "learning_rate": 2.52e-07, | |
| "logits/policy_chosen": -2.9184327125549316, | |
| "logits/policy_rejected": -2.7862141132354736, | |
| "logits/reference_chosen": -2.9029111862182617, | |
| "logits/reference_rejected": -2.562068462371826, | |
| "logps/policy_chosen": -176.180908203125, | |
| "logps/policy_rejected": -224.06822204589844, | |
| "logps/reference_chosen": -191.051025390625, | |
| "logps/reference_rejected": -231.65249633789062, | |
| "loss": 0.682, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.14870098233222961, | |
| "rewards/margins": 0.07285812497138977, | |
| "rewards/rejected": 0.07584285736083984, | |
| "step": 63, | |
| "tokens/chosen_response_len_mean": 383.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 315.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 2.6923985748881183, | |
| "learning_rate": 2.56e-07, | |
| "logits/policy_chosen": -3.1094112396240234, | |
| "logits/policy_rejected": -3.059873104095459, | |
| "logits/reference_chosen": -3.0696709156036377, | |
| "logits/reference_rejected": -3.0277552604675293, | |
| "logps/policy_chosen": -109.50044250488281, | |
| "logps/policy_rejected": -227.6053009033203, | |
| "logps/reference_chosen": -114.45265197753906, | |
| "logps/reference_rejected": -232.44473266601562, | |
| "loss": 0.6689, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.04952218383550644, | |
| "rewards/margins": 0.0011278600431978703, | |
| "rewards/rejected": 0.04839432239532471, | |
| "step": 64, | |
| "tokens/chosen_response_len_mean": 172.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 173.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 2.3372174586928685, | |
| "learning_rate": 2.6e-07, | |
| "logits/policy_chosen": -2.9087581634521484, | |
| "logits/policy_rejected": -2.973167657852173, | |
| "logits/reference_chosen": -2.754256248474121, | |
| "logits/reference_rejected": -2.943002223968506, | |
| "logps/policy_chosen": -16.885406494140625, | |
| "logps/policy_rejected": -58.0799560546875, | |
| "logps/reference_chosen": -16.55327033996582, | |
| "logps/reference_rejected": -58.852867126464844, | |
| "loss": 0.6709, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.0033213470596820116, | |
| "rewards/margins": -0.011050481349229813, | |
| "rewards/rejected": 0.007729134988039732, | |
| "step": 65, | |
| "tokens/chosen_response_len_mean": 25.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 37.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 1.8439881768995727, | |
| "learning_rate": 2.64e-07, | |
| "logits/policy_chosen": -3.2132272720336914, | |
| "logits/policy_rejected": -3.156252861022949, | |
| "logits/reference_chosen": -3.1969757080078125, | |
| "logits/reference_rejected": -3.008265733718872, | |
| "logps/policy_chosen": -89.50849914550781, | |
| "logps/policy_rejected": -105.58460998535156, | |
| "logps/reference_chosen": -96.482421875, | |
| "logps/reference_rejected": -111.33866119384766, | |
| "loss": 0.6855, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.06973925232887268, | |
| "rewards/margins": 0.0121987946331501, | |
| "rewards/rejected": 0.05754045769572258, | |
| "step": 66, | |
| "tokens/chosen_response_len_mean": 163.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 142.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 1.9679864280666897, | |
| "learning_rate": 2.68e-07, | |
| "logits/policy_chosen": -2.443567991256714, | |
| "logits/policy_rejected": -2.2462878227233887, | |
| "logits/reference_chosen": -2.3940353393554688, | |
| "logits/reference_rejected": -1.4334111213684082, | |
| "logps/policy_chosen": -133.6657257080078, | |
| "logps/policy_rejected": -126.65130615234375, | |
| "logps/reference_chosen": -143.77218627929688, | |
| "logps/reference_rejected": -130.96572875976562, | |
| "loss": 0.6801, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": 0.10106448829174042, | |
| "rewards/margins": 0.057920388877391815, | |
| "rewards/rejected": 0.0431441031396389, | |
| "step": 67, | |
| "tokens/chosen_response_len_mean": 360.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 238.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 2.3444172227352817, | |
| "learning_rate": 2.72e-07, | |
| "logits/policy_chosen": -2.447854518890381, | |
| "logits/policy_rejected": -2.4554026126861572, | |
| "logits/reference_chosen": -2.4224636554718018, | |
| "logits/reference_rejected": -2.44649600982666, | |
| "logps/policy_chosen": -116.48789978027344, | |
| "logps/policy_rejected": -196.64881896972656, | |
| "logps/reference_chosen": -129.35415649414062, | |
| "logps/reference_rejected": -209.97300720214844, | |
| "loss": 0.6714, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.12866270542144775, | |
| "rewards/margins": -0.004579143598675728, | |
| "rewards/rejected": 0.13324186205863953, | |
| "step": 68, | |
| "tokens/chosen_response_len_mean": 627.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 597.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 1.9355500776493433, | |
| "learning_rate": 2.7600000000000004e-07, | |
| "logits/policy_chosen": -3.001335620880127, | |
| "logits/policy_rejected": -2.8170900344848633, | |
| "logits/reference_chosen": -2.9873673915863037, | |
| "logits/reference_rejected": -2.8246562480926514, | |
| "logps/policy_chosen": -122.1274642944336, | |
| "logps/policy_rejected": -174.14083862304688, | |
| "logps/reference_chosen": -129.572509765625, | |
| "logps/reference_rejected": -175.02163696289062, | |
| "loss": 0.6814, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.07445055991411209, | |
| "rewards/margins": 0.06564265489578247, | |
| "rewards/rejected": 0.008807908743619919, | |
| "step": 69, | |
| "tokens/chosen_response_len_mean": 316.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 212.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 3.1880887553688613, | |
| "learning_rate": 2.8e-07, | |
| "logits/policy_chosen": -2.666834831237793, | |
| "logits/policy_rejected": -2.602752208709717, | |
| "logits/reference_chosen": -2.5575764179229736, | |
| "logits/reference_rejected": -2.594496965408325, | |
| "logps/policy_chosen": -144.6117401123047, | |
| "logps/policy_rejected": -323.0421447753906, | |
| "logps/reference_chosen": -156.17013549804688, | |
| "logps/reference_rejected": -333.77972412109375, | |
| "loss": 0.6725, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.11558418720960617, | |
| "rewards/margins": 0.00820855051279068, | |
| "rewards/rejected": 0.10737563669681549, | |
| "step": 70, | |
| "tokens/chosen_response_len_mean": 319.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 349.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 2.476324701775898, | |
| "learning_rate": 2.8399999999999995e-07, | |
| "logits/policy_chosen": -2.8258767127990723, | |
| "logits/policy_rejected": -2.808089017868042, | |
| "logits/reference_chosen": -2.8227195739746094, | |
| "logits/reference_rejected": -2.8310012817382812, | |
| "logps/policy_chosen": -141.63214111328125, | |
| "logps/policy_rejected": -178.0096893310547, | |
| "logps/reference_chosen": -151.22640991210938, | |
| "logps/reference_rejected": -182.14822387695312, | |
| "loss": 0.6621, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.09594269841909409, | |
| "rewards/margins": 0.054557494819164276, | |
| "rewards/rejected": 0.04138519987463951, | |
| "step": 71, | |
| "tokens/chosen_response_len_mean": 486.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 302.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 2.699352211419919, | |
| "learning_rate": 2.88e-07, | |
| "logits/policy_chosen": -2.597538709640503, | |
| "logits/policy_rejected": -2.41672945022583, | |
| "logits/reference_chosen": -2.5760765075683594, | |
| "logits/reference_rejected": -1.9195555448532104, | |
| "logps/policy_chosen": -247.6851043701172, | |
| "logps/policy_rejected": -382.4797058105469, | |
| "logps/reference_chosen": -263.0463562011719, | |
| "logps/reference_rejected": -387.8023376464844, | |
| "loss": 0.6657, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.15361259877681732, | |
| "rewards/margins": 0.10038647800683975, | |
| "rewards/rejected": 0.05322612449526787, | |
| "step": 72, | |
| "tokens/chosen_response_len_mean": 411.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 344.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 2.5969468603738255, | |
| "learning_rate": 2.9199999999999997e-07, | |
| "logits/policy_chosen": -2.7702274322509766, | |
| "logits/policy_rejected": -2.5614566802978516, | |
| "logits/reference_chosen": -2.7665748596191406, | |
| "logits/reference_rejected": -2.3978798389434814, | |
| "logps/policy_chosen": -72.09918212890625, | |
| "logps/policy_rejected": -68.87893676757812, | |
| "logps/reference_chosen": -80.68479919433594, | |
| "logps/reference_rejected": -67.986328125, | |
| "loss": 0.6635, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.08585613965988159, | |
| "rewards/margins": 0.0947822630405426, | |
| "rewards/rejected": -0.008926120586693287, | |
| "step": 73, | |
| "tokens/chosen_response_len_mean": 205.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 90.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 2.8994112666814575, | |
| "learning_rate": 2.9599999999999995e-07, | |
| "logits/policy_chosen": -2.9491050243377686, | |
| "logits/policy_rejected": -2.9374337196350098, | |
| "logits/reference_chosen": -2.924247980117798, | |
| "logits/reference_rejected": -2.8732385635375977, | |
| "logps/policy_chosen": -163.5004425048828, | |
| "logps/policy_rejected": -186.22500610351562, | |
| "logps/reference_chosen": -180.18331909179688, | |
| "logps/reference_rejected": -202.66912841796875, | |
| "loss": 0.6673, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.16682888567447662, | |
| "rewards/margins": 0.00238775834441185, | |
| "rewards/rejected": 0.16444112360477448, | |
| "step": 74, | |
| "tokens/chosen_response_len_mean": 365.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 331.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 2.1648523975476306, | |
| "learning_rate": 3e-07, | |
| "logits/policy_chosen": -2.1604349613189697, | |
| "logits/policy_rejected": -2.168158769607544, | |
| "logits/reference_chosen": -2.1622910499572754, | |
| "logits/reference_rejected": -2.1353628635406494, | |
| "logps/policy_chosen": -88.18841552734375, | |
| "logps/policy_rejected": -119.30603790283203, | |
| "logps/reference_chosen": -93.11495971679688, | |
| "logps/reference_rejected": -123.05696868896484, | |
| "loss": 0.6881, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.04926542192697525, | |
| "rewards/margins": 0.011756086722016335, | |
| "rewards/rejected": 0.037509337067604065, | |
| "step": 75, | |
| "tokens/chosen_response_len_mean": 258.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 199.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 2.403809139260171, | |
| "learning_rate": 3.0399999999999997e-07, | |
| "logits/policy_chosen": -2.830843925476074, | |
| "logits/policy_rejected": -2.8032288551330566, | |
| "logits/reference_chosen": -2.8209171295166016, | |
| "logits/reference_rejected": -2.797828435897827, | |
| "logps/policy_chosen": -69.43246459960938, | |
| "logps/policy_rejected": -162.20603942871094, | |
| "logps/reference_chosen": -74.48234558105469, | |
| "logps/reference_rejected": -164.12376403808594, | |
| "loss": 0.6658, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.05049885809421539, | |
| "rewards/margins": 0.03132178634405136, | |
| "rewards/rejected": 0.019177071750164032, | |
| "step": 76, | |
| "tokens/chosen_response_len_mean": 142.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 126.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 2.854272678012172, | |
| "learning_rate": 3.08e-07, | |
| "logits/policy_chosen": -3.0018627643585205, | |
| "logits/policy_rejected": -2.6590089797973633, | |
| "logits/reference_chosen": -2.995105028152466, | |
| "logits/reference_rejected": -2.385610580444336, | |
| "logps/policy_chosen": -85.27000427246094, | |
| "logps/policy_rejected": -132.92564392089844, | |
| "logps/reference_chosen": -90.3302001953125, | |
| "logps/reference_rejected": -134.35781860351562, | |
| "loss": 0.6608, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.05060192570090294, | |
| "rewards/margins": 0.03628016635775566, | |
| "rewards/rejected": 0.014321764931082726, | |
| "step": 77, | |
| "tokens/chosen_response_len_mean": 363.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 320.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 2.7505136356983133, | |
| "learning_rate": 3.12e-07, | |
| "logits/policy_chosen": -2.0338382720947266, | |
| "logits/policy_rejected": -2.276106357574463, | |
| "logits/reference_chosen": -2.0206451416015625, | |
| "logits/reference_rejected": -2.060913562774658, | |
| "logps/policy_chosen": -232.1044464111328, | |
| "logps/policy_rejected": -146.2607879638672, | |
| "logps/reference_chosen": -250.27952575683594, | |
| "logps/reference_rejected": -149.79270935058594, | |
| "loss": 0.6679, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.18175065517425537, | |
| "rewards/margins": 0.14643150568008423, | |
| "rewards/rejected": 0.03531915321946144, | |
| "step": 78, | |
| "tokens/chosen_response_len_mean": 521.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 307.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 2.4952987336105665, | |
| "learning_rate": 3.1599999999999997e-07, | |
| "logits/policy_chosen": -2.8599331378936768, | |
| "logits/policy_rejected": -2.8679845333099365, | |
| "logits/reference_chosen": -2.7079713344573975, | |
| "logits/reference_rejected": -2.4389915466308594, | |
| "logps/policy_chosen": -148.4790496826172, | |
| "logps/policy_rejected": -277.09405517578125, | |
| "logps/reference_chosen": -156.32412719726562, | |
| "logps/reference_rejected": -281.2509765625, | |
| "loss": 0.6709, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.0784510001540184, | |
| "rewards/margins": 0.03688175603747368, | |
| "rewards/rejected": 0.041569244116544724, | |
| "step": 79, | |
| "tokens/chosen_response_len_mean": 276.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 336.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 2.3479129695327297, | |
| "learning_rate": 3.2e-07, | |
| "logits/policy_chosen": -1.8025144338607788, | |
| "logits/policy_rejected": -1.9184333086013794, | |
| "logits/reference_chosen": -1.7402411699295044, | |
| "logits/reference_rejected": -1.906538724899292, | |
| "logps/policy_chosen": -79.56036376953125, | |
| "logps/policy_rejected": -185.89906311035156, | |
| "logps/reference_chosen": -88.7896499633789, | |
| "logps/reference_rejected": -198.23004150390625, | |
| "loss": 0.6921, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.09229286015033722, | |
| "rewards/margins": -0.03101687878370285, | |
| "rewards/rejected": 0.12330973893404007, | |
| "step": 80, | |
| "tokens/chosen_response_len_mean": 432.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 410.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 1.4788277018262954, | |
| "learning_rate": 3.24e-07, | |
| "logits/policy_chosen": -2.7751684188842773, | |
| "logits/policy_rejected": -2.7188775539398193, | |
| "logits/reference_chosen": -2.774470567703247, | |
| "logits/reference_rejected": -2.3951942920684814, | |
| "logps/policy_chosen": -64.14806365966797, | |
| "logps/policy_rejected": -91.78968048095703, | |
| "logps/reference_chosen": -70.32818603515625, | |
| "logps/reference_rejected": -92.31802368164062, | |
| "loss": 0.6813, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.06180126219987869, | |
| "rewards/margins": 0.05651785433292389, | |
| "rewards/rejected": 0.005283408332616091, | |
| "step": 81, | |
| "tokens/chosen_response_len_mean": 345.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 316.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 2.8935594664890503, | |
| "learning_rate": 3.28e-07, | |
| "logits/policy_chosen": -2.936018943786621, | |
| "logits/policy_rejected": -2.9313597679138184, | |
| "logits/reference_chosen": -2.9295578002929688, | |
| "logits/reference_rejected": -2.9169669151306152, | |
| "logps/policy_chosen": -141.27691650390625, | |
| "logps/policy_rejected": -264.863037109375, | |
| "logps/reference_chosen": -152.80865478515625, | |
| "logps/reference_rejected": -274.9104919433594, | |
| "loss": 0.6644, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.11531716585159302, | |
| "rewards/margins": 0.014842674136161804, | |
| "rewards/rejected": 0.10047448426485062, | |
| "step": 82, | |
| "tokens/chosen_response_len_mean": 394.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 390.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 2.4649587565099678, | |
| "learning_rate": 3.32e-07, | |
| "logits/policy_chosen": -2.9249117374420166, | |
| "logits/policy_rejected": -2.9090662002563477, | |
| "logits/reference_chosen": -2.868709087371826, | |
| "logits/reference_rejected": -2.898679256439209, | |
| "logps/policy_chosen": -154.240234375, | |
| "logps/policy_rejected": -297.08160400390625, | |
| "logps/reference_chosen": -161.24609375, | |
| "logps/reference_rejected": -297.5753479003906, | |
| "loss": 0.6687, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.07005855441093445, | |
| "rewards/margins": 0.06512109935283661, | |
| "rewards/rejected": 0.004937455989420414, | |
| "step": 83, | |
| "tokens/chosen_response_len_mean": 334.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 309.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 2.5355636198848175, | |
| "learning_rate": 3.36e-07, | |
| "logits/policy_chosen": -2.9488754272460938, | |
| "logits/policy_rejected": -2.9289844036102295, | |
| "logits/reference_chosen": -2.890922784805298, | |
| "logits/reference_rejected": -2.870727062225342, | |
| "logps/policy_chosen": -18.58037757873535, | |
| "logps/policy_rejected": -47.89413833618164, | |
| "logps/reference_chosen": -19.83108901977539, | |
| "logps/reference_rejected": -48.526878356933594, | |
| "loss": 0.6755, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.012507104314863682, | |
| "rewards/margins": 0.006179675459861755, | |
| "rewards/rejected": 0.006327427923679352, | |
| "step": 84, | |
| "tokens/chosen_response_len_mean": 27.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 32.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 2.5319776077967746, | |
| "learning_rate": 3.4000000000000003e-07, | |
| "logits/policy_chosen": -2.7499167919158936, | |
| "logits/policy_rejected": -2.690596103668213, | |
| "logits/reference_chosen": -2.73750638961792, | |
| "logits/reference_rejected": -2.6311354637145996, | |
| "logps/policy_chosen": -123.83195495605469, | |
| "logps/policy_rejected": -194.00137329101562, | |
| "logps/reference_chosen": -139.81263732910156, | |
| "logps/reference_rejected": -196.44784545898438, | |
| "loss": 0.6523, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.15980683267116547, | |
| "rewards/margins": 0.13534212112426758, | |
| "rewards/rejected": 0.024464718997478485, | |
| "step": 85, | |
| "tokens/chosen_response_len_mean": 606.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 519.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 2.007472503017741, | |
| "learning_rate": 3.4399999999999996e-07, | |
| "logits/policy_chosen": -2.9675540924072266, | |
| "logits/policy_rejected": -2.9613142013549805, | |
| "logits/reference_chosen": -2.9432828426361084, | |
| "logits/reference_rejected": -2.9374613761901855, | |
| "logps/policy_chosen": -12.885274887084961, | |
| "logps/policy_rejected": -34.69200134277344, | |
| "logps/reference_chosen": -13.591547012329102, | |
| "logps/reference_rejected": -35.15093994140625, | |
| "loss": 0.6775, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.007062721066176891, | |
| "rewards/margins": 0.002473349217325449, | |
| "rewards/rejected": 0.004589371383190155, | |
| "step": 86, | |
| "tokens/chosen_response_len_mean": 11.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 12.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 2.3366851120609398, | |
| "learning_rate": 3.4799999999999994e-07, | |
| "logits/policy_chosen": -2.510186195373535, | |
| "logits/policy_rejected": -2.3673720359802246, | |
| "logits/reference_chosen": -2.5208425521850586, | |
| "logits/reference_rejected": -2.307051658630371, | |
| "logps/policy_chosen": -103.87987518310547, | |
| "logps/policy_rejected": -118.88311767578125, | |
| "logps/reference_chosen": -114.24214172363281, | |
| "logps/reference_rejected": -118.43659973144531, | |
| "loss": 0.664, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.10362273454666138, | |
| "rewards/margins": 0.10808798670768738, | |
| "rewards/rejected": -0.004465251229703426, | |
| "step": 87, | |
| "tokens/chosen_response_len_mean": 216.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 106.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 2.4579695955138874, | |
| "learning_rate": 3.52e-07, | |
| "logits/policy_chosen": -2.9871153831481934, | |
| "logits/policy_rejected": -2.8764257431030273, | |
| "logits/reference_chosen": -2.9515204429626465, | |
| "logits/reference_rejected": -2.883835792541504, | |
| "logps/policy_chosen": -151.91807556152344, | |
| "logps/policy_rejected": -141.33587646484375, | |
| "logps/reference_chosen": -162.7379608154297, | |
| "logps/reference_rejected": -147.37442016601562, | |
| "loss": 0.6695, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.10819891095161438, | |
| "rewards/margins": 0.04781348258256912, | |
| "rewards/rejected": 0.06038542836904526, | |
| "step": 88, | |
| "tokens/chosen_response_len_mean": 299.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 223.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 2.2124030442195703, | |
| "learning_rate": 3.5599999999999996e-07, | |
| "logits/policy_chosen": -3.097362995147705, | |
| "logits/policy_rejected": -2.821688652038574, | |
| "logits/reference_chosen": -3.0792651176452637, | |
| "logits/reference_rejected": -2.6497914791107178, | |
| "logps/policy_chosen": -106.85523986816406, | |
| "logps/policy_rejected": -142.70628356933594, | |
| "logps/reference_chosen": -114.98326873779297, | |
| "logps/reference_rejected": -145.11463928222656, | |
| "loss": 0.6769, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.08128029853105545, | |
| "rewards/margins": 0.05719655007123947, | |
| "rewards/rejected": 0.02408374845981598, | |
| "step": 89, | |
| "tokens/chosen_response_len_mean": 288.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 241.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 2.726548509097342, | |
| "learning_rate": 3.6e-07, | |
| "logits/policy_chosen": -2.911972761154175, | |
| "logits/policy_rejected": -2.7545974254608154, | |
| "logits/reference_chosen": -2.8914873600006104, | |
| "logits/reference_rejected": -2.4686713218688965, | |
| "logps/policy_chosen": -90.4947509765625, | |
| "logps/policy_rejected": -144.89512634277344, | |
| "logps/reference_chosen": -100.72172546386719, | |
| "logps/reference_rejected": -144.84951782226562, | |
| "loss": 0.6578, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.1022697389125824, | |
| "rewards/margins": 0.10272593796253204, | |
| "rewards/rejected": -0.0004562041722238064, | |
| "step": 90, | |
| "tokens/chosen_response_len_mean": 254.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 112.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 2.2900988215150644, | |
| "learning_rate": 3.64e-07, | |
| "logits/policy_chosen": -2.8383259773254395, | |
| "logits/policy_rejected": -2.824490547180176, | |
| "logits/reference_chosen": -2.8273422718048096, | |
| "logits/reference_rejected": -2.8102381229400635, | |
| "logps/policy_chosen": -132.3096466064453, | |
| "logps/policy_rejected": -264.598388671875, | |
| "logps/reference_chosen": -139.56736755371094, | |
| "logps/reference_rejected": -264.87725830078125, | |
| "loss": 0.6825, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.07257718592882156, | |
| "rewards/margins": 0.0697886273264885, | |
| "rewards/rejected": 0.002788563258945942, | |
| "step": 91, | |
| "tokens/chosen_response_len_mean": 395.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 397.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 2.8545865680492692, | |
| "learning_rate": 3.6799999999999996e-07, | |
| "logits/policy_chosen": -2.7706384658813477, | |
| "logits/policy_rejected": -2.5063834190368652, | |
| "logits/reference_chosen": -2.7504565715789795, | |
| "logits/reference_rejected": -1.929320216178894, | |
| "logps/policy_chosen": -126.52993774414062, | |
| "logps/policy_rejected": -134.23519897460938, | |
| "logps/reference_chosen": -138.69195556640625, | |
| "logps/reference_rejected": -138.32545471191406, | |
| "loss": 0.6536, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.12162025272846222, | |
| "rewards/margins": 0.08071767538785934, | |
| "rewards/rejected": 0.040902577340602875, | |
| "step": 92, | |
| "tokens/chosen_response_len_mean": 315.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 201.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 2.284769139553738, | |
| "learning_rate": 3.72e-07, | |
| "logits/policy_chosen": -2.872082233428955, | |
| "logits/policy_rejected": -2.6889710426330566, | |
| "logits/reference_chosen": -2.8072874546051025, | |
| "logits/reference_rejected": -2.4844841957092285, | |
| "logps/policy_chosen": -74.13471984863281, | |
| "logps/policy_rejected": -116.40203857421875, | |
| "logps/reference_chosen": -80.3674545288086, | |
| "logps/reference_rejected": -114.1785888671875, | |
| "loss": 0.6602, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.06232735142111778, | |
| "rewards/margins": 0.08456189930438995, | |
| "rewards/rejected": -0.022234544157981873, | |
| "step": 93, | |
| "tokens/chosen_response_len_mean": 162.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 95.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 1.898688154588326, | |
| "learning_rate": 3.76e-07, | |
| "logits/policy_chosen": -2.469783306121826, | |
| "logits/policy_rejected": -2.3272128105163574, | |
| "logits/reference_chosen": -2.425753593444824, | |
| "logits/reference_rejected": -2.3546535968780518, | |
| "logps/policy_chosen": -158.62905883789062, | |
| "logps/policy_rejected": -291.726318359375, | |
| "logps/reference_chosen": -168.30911254882812, | |
| "logps/reference_rejected": -291.2076110839844, | |
| "loss": 0.6805, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.09680052101612091, | |
| "rewards/margins": 0.10198744386434555, | |
| "rewards/rejected": -0.005186920985579491, | |
| "step": 94, | |
| "tokens/chosen_response_len_mean": 359.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 302.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 2.9949420570538363, | |
| "learning_rate": 3.7999999999999996e-07, | |
| "logits/policy_chosen": -2.8909621238708496, | |
| "logits/policy_rejected": -2.906135082244873, | |
| "logits/reference_chosen": -2.792299747467041, | |
| "logits/reference_rejected": -2.848114252090454, | |
| "logps/policy_chosen": -50.312339782714844, | |
| "logps/policy_rejected": -105.15270233154297, | |
| "logps/reference_chosen": -52.856605529785156, | |
| "logps/reference_rejected": -108.09819030761719, | |
| "loss": 0.6483, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.02544267475605011, | |
| "rewards/margins": -0.004012217745184898, | |
| "rewards/rejected": 0.029454896226525307, | |
| "step": 95, | |
| "tokens/chosen_response_len_mean": 116.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 120.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 2.4241955868213405, | |
| "learning_rate": 3.84e-07, | |
| "logits/policy_chosen": -2.789196014404297, | |
| "logits/policy_rejected": -2.543649911880493, | |
| "logits/reference_chosen": -2.7888166904449463, | |
| "logits/reference_rejected": -2.5362277030944824, | |
| "logps/policy_chosen": -127.26165771484375, | |
| "logps/policy_rejected": -104.05661010742188, | |
| "logps/reference_chosen": -139.64187622070312, | |
| "logps/reference_rejected": -108.60116577148438, | |
| "loss": 0.6656, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.12380214780569077, | |
| "rewards/margins": 0.07835659384727478, | |
| "rewards/rejected": 0.04544556140899658, | |
| "step": 96, | |
| "tokens/chosen_response_len_mean": 381.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 272.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 3.4704291601036683, | |
| "learning_rate": 3.88e-07, | |
| "logits/policy_chosen": -2.9072158336639404, | |
| "logits/policy_rejected": -2.845478057861328, | |
| "logits/reference_chosen": -2.882009983062744, | |
| "logits/reference_rejected": -2.943215847015381, | |
| "logps/policy_chosen": -199.15065002441406, | |
| "logps/policy_rejected": -345.61346435546875, | |
| "logps/reference_chosen": -208.939697265625, | |
| "logps/reference_rejected": -351.9114990234375, | |
| "loss": 0.6532, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.09789040684700012, | |
| "rewards/margins": 0.0349101684987545, | |
| "rewards/rejected": 0.06298023462295532, | |
| "step": 97, | |
| "tokens/chosen_response_len_mean": 443.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 334.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 2.413585232959013, | |
| "learning_rate": 3.92e-07, | |
| "logits/policy_chosen": -2.581110954284668, | |
| "logits/policy_rejected": -2.5821545124053955, | |
| "logits/reference_chosen": -2.6214118003845215, | |
| "logits/reference_rejected": -2.6232800483703613, | |
| "logps/policy_chosen": -66.4002914428711, | |
| "logps/policy_rejected": -92.362060546875, | |
| "logps/reference_chosen": -73.42770385742188, | |
| "logps/reference_rejected": -96.90080261230469, | |
| "loss": 0.6682, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.07027411460876465, | |
| "rewards/margins": 0.02488674223423004, | |
| "rewards/rejected": 0.0453873835504055, | |
| "step": 98, | |
| "tokens/chosen_response_len_mean": 369.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 353.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 2.566883896226423, | |
| "learning_rate": 3.96e-07, | |
| "logits/policy_chosen": -3.13887357711792, | |
| "logits/policy_rejected": -3.1172709465026855, | |
| "logits/reference_chosen": -3.127457618713379, | |
| "logits/reference_rejected": -3.1038241386413574, | |
| "logps/policy_chosen": -131.76913452148438, | |
| "logps/policy_rejected": -274.8529052734375, | |
| "logps/reference_chosen": -142.9169158935547, | |
| "logps/reference_rejected": -277.9893798828125, | |
| "loss": 0.68, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.11147774755954742, | |
| "rewards/margins": 0.08011295646429062, | |
| "rewards/rejected": 0.031364794820547104, | |
| "step": 99, | |
| "tokens/chosen_response_len_mean": 291.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 296.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 2.7837569692907294, | |
| "learning_rate": 4e-07, | |
| "logits/policy_chosen": -3.046888589859009, | |
| "logits/policy_rejected": -2.9924631118774414, | |
| "logits/reference_chosen": -3.0203585624694824, | |
| "logits/reference_rejected": -2.9802844524383545, | |
| "logps/policy_chosen": -96.66795349121094, | |
| "logps/policy_rejected": -151.02667236328125, | |
| "logps/reference_chosen": -106.40546417236328, | |
| "logps/reference_rejected": -156.30604553222656, | |
| "loss": 0.6494, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.09737517684698105, | |
| "rewards/margins": 0.04458136856555939, | |
| "rewards/rejected": 0.05279380828142166, | |
| "step": 100, | |
| "tokens/chosen_response_len_mean": 246.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 213.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 2.5533090774871385, | |
| "learning_rate": 4.04e-07, | |
| "logits/policy_chosen": -3.0745935440063477, | |
| "logits/policy_rejected": -2.9924702644348145, | |
| "logits/reference_chosen": -3.0464026927948, | |
| "logits/reference_rejected": -2.9301576614379883, | |
| "logps/policy_chosen": -130.01454162597656, | |
| "logps/policy_rejected": -273.28997802734375, | |
| "logps/reference_chosen": -138.4945831298828, | |
| "logps/reference_rejected": -277.6722106933594, | |
| "loss": 0.6659, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.08480045944452286, | |
| "rewards/margins": 0.04097801819443703, | |
| "rewards/rejected": 0.04382243752479553, | |
| "step": 101, | |
| "tokens/chosen_response_len_mean": 351.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 294.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 2.658053156359227, | |
| "learning_rate": 4.0799999999999995e-07, | |
| "logits/policy_chosen": -2.9965174198150635, | |
| "logits/policy_rejected": -2.756721019744873, | |
| "logits/reference_chosen": -3.0305070877075195, | |
| "logits/reference_rejected": -2.5150346755981445, | |
| "logps/policy_chosen": -150.61041259765625, | |
| "logps/policy_rejected": -145.7916717529297, | |
| "logps/reference_chosen": -169.15542602539062, | |
| "logps/reference_rejected": -147.83485412597656, | |
| "loss": 0.6388, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.18545001745224, | |
| "rewards/margins": 0.16501817107200623, | |
| "rewards/rejected": 0.02043186128139496, | |
| "step": 102, | |
| "tokens/chosen_response_len_mean": 276.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 211.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 1.8527975992911931, | |
| "learning_rate": 4.12e-07, | |
| "logits/policy_chosen": -2.4530792236328125, | |
| "logits/policy_rejected": -2.4156646728515625, | |
| "logits/reference_chosen": -2.4479713439941406, | |
| "logits/reference_rejected": -2.3904917240142822, | |
| "logps/policy_chosen": -173.00030517578125, | |
| "logps/policy_rejected": -199.19415283203125, | |
| "logps/reference_chosen": -189.02093505859375, | |
| "logps/reference_rejected": -209.84201049804688, | |
| "loss": 0.6666, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.16020633280277252, | |
| "rewards/margins": 0.05372770130634308, | |
| "rewards/rejected": 0.10647863149642944, | |
| "step": 103, | |
| "tokens/chosen_response_len_mean": 309.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 310.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 2.5555937206717187, | |
| "learning_rate": 4.1599999999999997e-07, | |
| "logits/policy_chosen": -3.0911600589752197, | |
| "logits/policy_rejected": -3.0214200019836426, | |
| "logits/reference_chosen": -3.089872360229492, | |
| "logits/reference_rejected": -2.882385730743408, | |
| "logps/policy_chosen": -153.61936950683594, | |
| "logps/policy_rejected": -227.18930053710938, | |
| "logps/reference_chosen": -176.565673828125, | |
| "logps/reference_rejected": -239.63970947265625, | |
| "loss": 0.665, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.22946298122406006, | |
| "rewards/margins": 0.10495913028717041, | |
| "rewards/rejected": 0.12450385838747025, | |
| "step": 104, | |
| "tokens/chosen_response_len_mean": 368.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 301.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 2.0081558824498096, | |
| "learning_rate": 4.1999999999999995e-07, | |
| "logits/policy_chosen": -2.958939790725708, | |
| "logits/policy_rejected": -2.80196213722229, | |
| "logits/reference_chosen": -2.944668769836426, | |
| "logits/reference_rejected": -2.6392948627471924, | |
| "logps/policy_chosen": -37.521568298339844, | |
| "logps/policy_rejected": -74.32329559326172, | |
| "logps/reference_chosen": -41.80109786987305, | |
| "logps/reference_rejected": -72.32585906982422, | |
| "loss": 0.664, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.0427953377366066, | |
| "rewards/margins": 0.06276971846818924, | |
| "rewards/rejected": -0.01997438445687294, | |
| "step": 105, | |
| "tokens/chosen_response_len_mean": 105.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 85.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 2.7526269849819696, | |
| "learning_rate": 4.24e-07, | |
| "logits/policy_chosen": -3.100095272064209, | |
| "logits/policy_rejected": -2.8839964866638184, | |
| "logits/reference_chosen": -3.0812511444091797, | |
| "logits/reference_rejected": -2.936655044555664, | |
| "logps/policy_chosen": -207.67698669433594, | |
| "logps/policy_rejected": -240.35739135742188, | |
| "logps/reference_chosen": -223.25807189941406, | |
| "logps/reference_rejected": -239.6046142578125, | |
| "loss": 0.6494, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.15581092238426208, | |
| "rewards/margins": 0.16333873569965363, | |
| "rewards/rejected": -0.007527794688940048, | |
| "step": 106, | |
| "tokens/chosen_response_len_mean": 392.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 203.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 1.9552389682760483, | |
| "learning_rate": 4.2799999999999997e-07, | |
| "logits/policy_chosen": -2.922211170196533, | |
| "logits/policy_rejected": -2.94032621383667, | |
| "logits/reference_chosen": -2.9038805961608887, | |
| "logits/reference_rejected": -2.9326558113098145, | |
| "logps/policy_chosen": -142.2540283203125, | |
| "logps/policy_rejected": -238.67227172851562, | |
| "logps/reference_chosen": -151.64273071289062, | |
| "logps/reference_rejected": -239.12753295898438, | |
| "loss": 0.6753, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.09388697892427444, | |
| "rewards/margins": 0.08933433890342712, | |
| "rewards/rejected": 0.004552632570266724, | |
| "step": 107, | |
| "tokens/chosen_response_len_mean": 347.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 352.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 3.014699368900832, | |
| "learning_rate": 4.3199999999999995e-07, | |
| "logits/policy_chosen": -3.1775548458099365, | |
| "logits/policy_rejected": -3.03210186958313, | |
| "logits/reference_chosen": -3.1847496032714844, | |
| "logits/reference_rejected": -2.531522750854492, | |
| "logps/policy_chosen": -143.90025329589844, | |
| "logps/policy_rejected": -291.92388916015625, | |
| "logps/reference_chosen": -160.6278839111328, | |
| "logps/reference_rejected": -297.986572265625, | |
| "loss": 0.6498, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.16727623343467712, | |
| "rewards/margins": 0.10664979368448257, | |
| "rewards/rejected": 0.060626450926065445, | |
| "step": 108, | |
| "tokens/chosen_response_len_mean": 411.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 381.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 3.3247511691725236, | |
| "learning_rate": 4.36e-07, | |
| "logits/policy_chosen": -3.0070884227752686, | |
| "logits/policy_rejected": -3.006807565689087, | |
| "logits/reference_chosen": -2.9760210514068604, | |
| "logits/reference_rejected": -2.982029438018799, | |
| "logps/policy_chosen": -117.77191925048828, | |
| "logps/policy_rejected": -288.87689208984375, | |
| "logps/reference_chosen": -130.60389709472656, | |
| "logps/reference_rejected": -292.3191833496094, | |
| "loss": 0.6517, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.12831971049308777, | |
| "rewards/margins": 0.09389686584472656, | |
| "rewards/rejected": 0.034422844648361206, | |
| "step": 109, | |
| "tokens/chosen_response_len_mean": 318.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 302.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 3.158975765008515, | |
| "learning_rate": 4.3999999999999997e-07, | |
| "logits/policy_chosen": -2.50785756111145, | |
| "logits/policy_rejected": -2.504612684249878, | |
| "logits/reference_chosen": -2.5301709175109863, | |
| "logits/reference_rejected": -2.5272464752197266, | |
| "logps/policy_chosen": -136.28662109375, | |
| "logps/policy_rejected": -200.89352416992188, | |
| "logps/reference_chosen": -158.42433166503906, | |
| "logps/reference_rejected": -212.5179443359375, | |
| "loss": 0.638, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.22137700021266937, | |
| "rewards/margins": 0.10513283312320709, | |
| "rewards/rejected": 0.11624416708946228, | |
| "step": 110, | |
| "tokens/chosen_response_len_mean": 608.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 525.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 2.8278528577842095, | |
| "learning_rate": 4.44e-07, | |
| "logits/policy_chosen": -2.725069999694824, | |
| "logits/policy_rejected": -2.732088565826416, | |
| "logits/reference_chosen": -2.6485838890075684, | |
| "logits/reference_rejected": -2.76961612701416, | |
| "logps/policy_chosen": -178.09320068359375, | |
| "logps/policy_rejected": -237.05970764160156, | |
| "logps/reference_chosen": -192.22482299804688, | |
| "logps/reference_rejected": -248.2169647216797, | |
| "loss": 0.6253, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.14131632447242737, | |
| "rewards/margins": 0.029743948951363564, | |
| "rewards/rejected": 0.11157237738370895, | |
| "step": 111, | |
| "tokens/chosen_response_len_mean": 589.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 556.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 3.4191485129114714, | |
| "learning_rate": 4.48e-07, | |
| "logits/policy_chosen": -2.7914977073669434, | |
| "logits/policy_rejected": -2.804826259613037, | |
| "logits/reference_chosen": -2.7042627334594727, | |
| "logits/reference_rejected": -2.640076160430908, | |
| "logps/policy_chosen": -146.0531768798828, | |
| "logps/policy_rejected": -232.5332794189453, | |
| "logps/reference_chosen": -168.23060607910156, | |
| "logps/reference_rejected": -237.49801635742188, | |
| "loss": 0.6295, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.22177428007125854, | |
| "rewards/margins": 0.172126904129982, | |
| "rewards/rejected": 0.049647387117147446, | |
| "step": 112, | |
| "tokens/chosen_response_len_mean": 438.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 396.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 2.331597318165628, | |
| "learning_rate": 4.5199999999999997e-07, | |
| "logits/policy_chosen": -3.1060757637023926, | |
| "logits/policy_rejected": -2.961545467376709, | |
| "logits/reference_chosen": -3.1052682399749756, | |
| "logits/reference_rejected": -2.833801746368408, | |
| "logps/policy_chosen": -127.66785430908203, | |
| "logps/policy_rejected": -196.163330078125, | |
| "logps/reference_chosen": -137.27505493164062, | |
| "logps/reference_rejected": -198.14578247070312, | |
| "loss": 0.6605, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.09607198089361191, | |
| "rewards/margins": 0.07624746859073639, | |
| "rewards/rejected": 0.01982451230287552, | |
| "step": 113, | |
| "tokens/chosen_response_len_mean": 314.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 291.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 2.750538166446364, | |
| "learning_rate": 4.56e-07, | |
| "logits/policy_chosen": -2.7500100135803223, | |
| "logits/policy_rejected": -2.681910514831543, | |
| "logits/reference_chosen": -2.789179563522339, | |
| "logits/reference_rejected": -2.3359477519989014, | |
| "logps/policy_chosen": -85.98888397216797, | |
| "logps/policy_rejected": -100.74612426757812, | |
| "logps/reference_chosen": -98.14977264404297, | |
| "logps/reference_rejected": -102.4841537475586, | |
| "loss": 0.6408, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.12160897254943848, | |
| "rewards/margins": 0.10422859340906143, | |
| "rewards/rejected": 0.017380371689796448, | |
| "step": 114, | |
| "tokens/chosen_response_len_mean": 256.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 223.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 2.5440555248438996, | |
| "learning_rate": 4.6e-07, | |
| "logits/policy_chosen": -2.971923351287842, | |
| "logits/policy_rejected": -2.989212989807129, | |
| "logits/reference_chosen": -2.9268057346343994, | |
| "logits/reference_rejected": -2.919334650039673, | |
| "logps/policy_chosen": -98.5408706665039, | |
| "logps/policy_rejected": -166.144775390625, | |
| "logps/reference_chosen": -109.1729965209961, | |
| "logps/reference_rejected": -167.0085906982422, | |
| "loss": 0.6505, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.10632127523422241, | |
| "rewards/margins": 0.09768319875001907, | |
| "rewards/rejected": 0.008638077415525913, | |
| "step": 115, | |
| "tokens/chosen_response_len_mean": 229.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 254.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 2.4244019154261474, | |
| "learning_rate": 4.64e-07, | |
| "logits/policy_chosen": -2.8886287212371826, | |
| "logits/policy_rejected": -2.922623872756958, | |
| "logits/reference_chosen": -2.8570451736450195, | |
| "logits/reference_rejected": -2.888643741607666, | |
| "logps/policy_chosen": -113.93293762207031, | |
| "logps/policy_rejected": -225.3817596435547, | |
| "logps/reference_chosen": -120.10133361816406, | |
| "logps/reference_rejected": -223.7104034423828, | |
| "loss": 0.6579, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.06168397143483162, | |
| "rewards/margins": 0.0783977136015892, | |
| "rewards/rejected": -0.016713742166757584, | |
| "step": 116, | |
| "tokens/chosen_response_len_mean": 330.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 318.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 2.400248188855275, | |
| "learning_rate": 4.68e-07, | |
| "logits/policy_chosen": -2.8977608680725098, | |
| "logits/policy_rejected": -2.9003448486328125, | |
| "logits/reference_chosen": -2.8988900184631348, | |
| "logits/reference_rejected": -2.9036717414855957, | |
| "logps/policy_chosen": -74.31845092773438, | |
| "logps/policy_rejected": -92.14234924316406, | |
| "logps/reference_chosen": -78.81231689453125, | |
| "logps/reference_rejected": -93.07196807861328, | |
| "loss": 0.6588, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.04493863508105278, | |
| "rewards/margins": 0.035642486065626144, | |
| "rewards/rejected": 0.00929614994674921, | |
| "step": 117, | |
| "tokens/chosen_response_len_mean": 275.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 274.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 2.4323431386681595, | |
| "learning_rate": 4.7199999999999994e-07, | |
| "logits/policy_chosen": -2.6670236587524414, | |
| "logits/policy_rejected": -2.518751621246338, | |
| "logits/reference_chosen": -2.666407585144043, | |
| "logits/reference_rejected": -2.5256845951080322, | |
| "logps/policy_chosen": -92.44061279296875, | |
| "logps/policy_rejected": -151.3938446044922, | |
| "logps/reference_chosen": -100.54401397705078, | |
| "logps/reference_rejected": -153.23507690429688, | |
| "loss": 0.6474, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.08103407174348831, | |
| "rewards/margins": 0.06262162327766418, | |
| "rewards/rejected": 0.01841244101524353, | |
| "step": 118, | |
| "tokens/chosen_response_len_mean": 302.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 332.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 2.9673553403256707, | |
| "learning_rate": 4.76e-07, | |
| "logits/policy_chosen": -2.9274580478668213, | |
| "logits/policy_rejected": -2.8594419956207275, | |
| "logits/reference_chosen": -2.9505772590637207, | |
| "logits/reference_rejected": -2.850255012512207, | |
| "logps/policy_chosen": -169.86106872558594, | |
| "logps/policy_rejected": -316.6098327636719, | |
| "logps/reference_chosen": -191.77130126953125, | |
| "logps/reference_rejected": -317.0496520996094, | |
| "loss": 0.6273, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.21910235285758972, | |
| "rewards/margins": 0.21470384299755096, | |
| "rewards/rejected": 0.004398513585329056, | |
| "step": 119, | |
| "tokens/chosen_response_len_mean": 431.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 374.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 2.3505092556042304, | |
| "learning_rate": 4.8e-07, | |
| "logits/policy_chosen": -2.814655303955078, | |
| "logits/policy_rejected": -2.5319275856018066, | |
| "logits/reference_chosen": -2.8454208374023438, | |
| "logits/reference_rejected": -2.315487861633301, | |
| "logps/policy_chosen": -50.07554626464844, | |
| "logps/policy_rejected": -110.84510040283203, | |
| "logps/reference_chosen": -60.05486297607422, | |
| "logps/reference_rejected": -106.57113647460938, | |
| "loss": 0.6368, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.09979310631752014, | |
| "rewards/margins": 0.14253273606300354, | |
| "rewards/rejected": -0.0427396334707737, | |
| "step": 120, | |
| "tokens/chosen_response_len_mean": 267.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 187.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 2.8274965378312586, | |
| "learning_rate": 4.839999999999999e-07, | |
| "logits/policy_chosen": -2.942972421646118, | |
| "logits/policy_rejected": -2.8031089305877686, | |
| "logits/reference_chosen": -2.961139678955078, | |
| "logits/reference_rejected": -2.5472524166107178, | |
| "logps/policy_chosen": -130.38365173339844, | |
| "logps/policy_rejected": -191.11968994140625, | |
| "logps/reference_chosen": -159.17897033691406, | |
| "logps/reference_rejected": -199.572021484375, | |
| "loss": 0.6282, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2879532277584076, | |
| "rewards/margins": 0.2034299373626709, | |
| "rewards/rejected": 0.0845232680439949, | |
| "step": 121, | |
| "tokens/chosen_response_len_mean": 420.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 307.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 2.886588252399639, | |
| "learning_rate": 4.879999999999999e-07, | |
| "logits/policy_chosen": -2.853999376296997, | |
| "logits/policy_rejected": -2.480708122253418, | |
| "logits/reference_chosen": -2.710318088531494, | |
| "logits/reference_rejected": -1.9466197490692139, | |
| "logps/policy_chosen": -167.15240478515625, | |
| "logps/policy_rejected": -201.3469696044922, | |
| "logps/reference_chosen": -181.32119750976562, | |
| "logps/reference_rejected": -195.66384887695312, | |
| "loss": 0.6126, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.14168795943260193, | |
| "rewards/margins": 0.1985190212726593, | |
| "rewards/rejected": -0.056831058114767075, | |
| "step": 122, | |
| "tokens/chosen_response_len_mean": 326.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 201.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 2.826829476556088, | |
| "learning_rate": 4.92e-07, | |
| "logits/policy_chosen": -2.465135335922241, | |
| "logits/policy_rejected": -2.4350569248199463, | |
| "logits/reference_chosen": -2.4818878173828125, | |
| "logits/reference_rejected": -2.3954761028289795, | |
| "logps/policy_chosen": -100.21925354003906, | |
| "logps/policy_rejected": -125.87073516845703, | |
| "logps/reference_chosen": -126.35533905029297, | |
| "logps/reference_rejected": -144.03236389160156, | |
| "loss": 0.6328, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.26136085391044617, | |
| "rewards/margins": 0.07974465936422348, | |
| "rewards/rejected": 0.1816161870956421, | |
| "step": 123, | |
| "tokens/chosen_response_len_mean": 426.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 408.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 2.5289041457286774, | |
| "learning_rate": 4.96e-07, | |
| "logits/policy_chosen": -2.6869404315948486, | |
| "logits/policy_rejected": -2.773239850997925, | |
| "logits/reference_chosen": -2.653362512588501, | |
| "logits/reference_rejected": -2.5821008682250977, | |
| "logps/policy_chosen": -71.6850357055664, | |
| "logps/policy_rejected": -108.95183563232422, | |
| "logps/reference_chosen": -77.52404022216797, | |
| "logps/reference_rejected": -107.73311614990234, | |
| "loss": 0.6376, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.05839002877473831, | |
| "rewards/margins": 0.07057718932628632, | |
| "rewards/rejected": -0.012187155894935131, | |
| "step": 124, | |
| "tokens/chosen_response_len_mean": 229.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 211.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 3.7452560936185804, | |
| "learning_rate": 5e-07, | |
| "logits/policy_chosen": -2.8474879264831543, | |
| "logits/policy_rejected": -2.77992844581604, | |
| "logits/reference_chosen": -2.8415565490722656, | |
| "logits/reference_rejected": -2.658388376235962, | |
| "logps/policy_chosen": -85.31348419189453, | |
| "logps/policy_rejected": -100.18048095703125, | |
| "logps/reference_chosen": -91.71619415283203, | |
| "logps/reference_rejected": -100.39627838134766, | |
| "loss": 0.6215, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.0640270784497261, | |
| "rewards/margins": 0.06186922267079353, | |
| "rewards/rejected": 0.00215785950422287, | |
| "step": 125, | |
| "tokens/chosen_response_len_mean": 91.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 71.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 2.791877501388191, | |
| "learning_rate": 4.999990147435196e-07, | |
| "logits/policy_chosen": -2.772634506225586, | |
| "logits/policy_rejected": -2.7663345336914062, | |
| "logits/reference_chosen": -2.8182811737060547, | |
| "logits/reference_rejected": -2.821171760559082, | |
| "logps/policy_chosen": -114.60906982421875, | |
| "logps/policy_rejected": -122.4944839477539, | |
| "logps/reference_chosen": -141.17056274414062, | |
| "logps/reference_rejected": -135.64431762695312, | |
| "loss": 0.63, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2656148672103882, | |
| "rewards/margins": 0.13411659002304077, | |
| "rewards/rejected": 0.1314982771873474, | |
| "step": 126, | |
| "tokens/chosen_response_len_mean": 458.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 332.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 2.692910890812567, | |
| "learning_rate": 4.99996058981844e-07, | |
| "logits/policy_chosen": -2.5161116123199463, | |
| "logits/policy_rejected": -2.3703317642211914, | |
| "logits/reference_chosen": -2.4154701232910156, | |
| "logits/reference_rejected": -2.4023566246032715, | |
| "logps/policy_chosen": -72.83512115478516, | |
| "logps/policy_rejected": -136.28607177734375, | |
| "logps/reference_chosen": -82.63104248046875, | |
| "logps/reference_rejected": -140.04986572265625, | |
| "loss": 0.6369, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.09795919060707092, | |
| "rewards/margins": 0.06032119691371918, | |
| "rewards/rejected": 0.037637993693351746, | |
| "step": 127, | |
| "tokens/chosen_response_len_mean": 416.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 410.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 2.859419025019686, | |
| "learning_rate": 4.999911327382711e-07, | |
| "logits/policy_chosen": -2.0673351287841797, | |
| "logits/policy_rejected": -1.9885450601577759, | |
| "logits/reference_chosen": -2.1499083042144775, | |
| "logits/reference_rejected": -1.6471858024597168, | |
| "logps/policy_chosen": -57.06989669799805, | |
| "logps/policy_rejected": -87.03865814208984, | |
| "logps/reference_chosen": -69.13873291015625, | |
| "logps/reference_rejected": -83.46507263183594, | |
| "loss": 0.5998, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.12068840861320496, | |
| "rewards/margins": 0.15642426908016205, | |
| "rewards/rejected": -0.035735856741666794, | |
| "step": 128, | |
| "tokens/chosen_response_len_mean": 343.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 194.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 2.8759886658807288, | |
| "learning_rate": 4.999842360516294e-07, | |
| "logits/policy_chosen": -3.1045093536376953, | |
| "logits/policy_rejected": -3.1223738193511963, | |
| "logits/reference_chosen": -3.09148907661438, | |
| "logits/reference_rejected": -3.091776132583618, | |
| "logps/policy_chosen": -40.253719329833984, | |
| "logps/policy_rejected": -93.64241027832031, | |
| "logps/reference_chosen": -43.66665267944336, | |
| "logps/reference_rejected": -92.32077026367188, | |
| "loss": 0.5883, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.03412933275103569, | |
| "rewards/margins": 0.047345712780952454, | |
| "rewards/rejected": -0.01321637723594904, | |
| "step": 129, | |
| "tokens/chosen_response_len_mean": 134.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 142.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 2.396608033911008, | |
| "learning_rate": 4.999753689762792e-07, | |
| "logits/policy_chosen": -2.7669222354888916, | |
| "logits/policy_rejected": -3.0244264602661133, | |
| "logits/reference_chosen": -2.7182681560516357, | |
| "logits/reference_rejected": -3.016852855682373, | |
| "logps/policy_chosen": -49.25522994995117, | |
| "logps/policy_rejected": -209.12026977539062, | |
| "logps/reference_chosen": -58.14155197143555, | |
| "logps/reference_rejected": -213.75338745117188, | |
| "loss": 0.6532, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.08886321634054184, | |
| "rewards/margins": 0.04253204166889191, | |
| "rewards/rejected": 0.04633117839694023, | |
| "step": 130, | |
| "tokens/chosen_response_len_mean": 140.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 397.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 2.8338085411833127, | |
| "learning_rate": 4.999645315821111e-07, | |
| "logits/policy_chosen": -2.177436590194702, | |
| "logits/policy_rejected": -2.1923160552978516, | |
| "logits/reference_chosen": -2.23976993560791, | |
| "logits/reference_rejected": -2.227308511734009, | |
| "logps/policy_chosen": -92.42753601074219, | |
| "logps/policy_rejected": -104.93522644042969, | |
| "logps/reference_chosen": -111.59609985351562, | |
| "logps/reference_rejected": -120.66436004638672, | |
| "loss": 0.6227, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.19168560206890106, | |
| "rewards/margins": 0.03439423814415932, | |
| "rewards/rejected": 0.15729136765003204, | |
| "step": 131, | |
| "tokens/chosen_response_len_mean": 428.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 393.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 2.537615272647266, | |
| "learning_rate": 4.99951723954546e-07, | |
| "logits/policy_chosen": -2.6528267860412598, | |
| "logits/policy_rejected": -2.7896065711975098, | |
| "logits/reference_chosen": -2.3186991214752197, | |
| "logits/reference_rejected": -2.7976412773132324, | |
| "logps/policy_chosen": -35.68031311035156, | |
| "logps/policy_rejected": -72.86119079589844, | |
| "logps/reference_chosen": -42.128787994384766, | |
| "logps/reference_rejected": -76.47801208496094, | |
| "loss": 0.6365, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.06448473036289215, | |
| "rewards/margins": 0.028316427022218704, | |
| "rewards/rejected": 0.03616830334067345, | |
| "step": 132, | |
| "tokens/chosen_response_len_mean": 226.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 273.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 2.952022770646515, | |
| "learning_rate": 4.999369461945343e-07, | |
| "logits/policy_chosen": -3.1463866233825684, | |
| "logits/policy_rejected": -2.8932509422302246, | |
| "logits/reference_chosen": -3.141604423522949, | |
| "logits/reference_rejected": -2.444639205932617, | |
| "logps/policy_chosen": -83.71105194091797, | |
| "logps/policy_rejected": -113.13372802734375, | |
| "logps/reference_chosen": -94.0230941772461, | |
| "logps/reference_rejected": -108.04383850097656, | |
| "loss": 0.5892, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.10312040150165558, | |
| "rewards/margins": 0.15401925146579742, | |
| "rewards/rejected": -0.05089886486530304, | |
| "step": 133, | |
| "tokens/chosen_response_len_mean": 249.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 178.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 3.540843119893174, | |
| "learning_rate": 4.999201984185552e-07, | |
| "logits/policy_chosen": -3.0958125591278076, | |
| "logits/policy_rejected": -3.042353868484497, | |
| "logits/reference_chosen": -3.133866310119629, | |
| "logits/reference_rejected": -3.0950450897216797, | |
| "logps/policy_chosen": -236.39605712890625, | |
| "logps/policy_rejected": -330.2846374511719, | |
| "logps/reference_chosen": -259.0321350097656, | |
| "logps/reference_rejected": -336.391357421875, | |
| "loss": 0.5663, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.22636055946350098, | |
| "rewards/margins": 0.16529357433319092, | |
| "rewards/rejected": 0.06106698513031006, | |
| "step": 134, | |
| "tokens/chosen_response_len_mean": 717.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 639.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 2.981444512825392, | |
| "learning_rate": 4.999014807586154e-07, | |
| "logits/policy_chosen": -2.3856396675109863, | |
| "logits/policy_rejected": -2.1970813274383545, | |
| "logits/reference_chosen": -2.4693095684051514, | |
| "logits/reference_rejected": -1.5482537746429443, | |
| "logps/policy_chosen": -142.9355926513672, | |
| "logps/policy_rejected": -189.60543823242188, | |
| "logps/reference_chosen": -164.72271728515625, | |
| "logps/reference_rejected": -189.248046875, | |
| "loss": 0.6065, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2178712785243988, | |
| "rewards/margins": 0.22144514322280884, | |
| "rewards/rejected": -0.003573855385184288, | |
| "step": 135, | |
| "tokens/chosen_response_len_mean": 339.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 276.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 2.4024504350993654, | |
| "learning_rate": 4.998807933622484e-07, | |
| "logits/policy_chosen": -3.1550679206848145, | |
| "logits/policy_rejected": -3.1557650566101074, | |
| "logits/reference_chosen": -3.1761221885681152, | |
| "logits/reference_rejected": -3.135535955429077, | |
| "logps/policy_chosen": -118.20024871826172, | |
| "logps/policy_rejected": -264.43560791015625, | |
| "logps/reference_chosen": -130.25527954101562, | |
| "logps/reference_rejected": -264.16741943359375, | |
| "loss": 0.6336, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.12055033445358276, | |
| "rewards/margins": 0.12323223054409027, | |
| "rewards/rejected": -0.002681899815797806, | |
| "step": 136, | |
| "tokens/chosen_response_len_mean": 354.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 386.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 2.8382305196380684, | |
| "learning_rate": 4.998581363925135e-07, | |
| "logits/policy_chosen": -2.2047266960144043, | |
| "logits/policy_rejected": -2.433465003967285, | |
| "logits/reference_chosen": -2.275531530380249, | |
| "logits/reference_rejected": -2.245540142059326, | |
| "logps/policy_chosen": -378.1270751953125, | |
| "logps/policy_rejected": -344.7900085449219, | |
| "logps/reference_chosen": -416.6529541015625, | |
| "logps/reference_rejected": -347.785888671875, | |
| "loss": 0.5804, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3852587938308716, | |
| "rewards/margins": 0.3552997410297394, | |
| "rewards/rejected": 0.02995903044939041, | |
| "step": 137, | |
| "tokens/chosen_response_len_mean": 584.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 427.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 2.1385240477591636, | |
| "learning_rate": 4.998335100279941e-07, | |
| "logits/policy_chosen": -2.6462714672088623, | |
| "logits/policy_rejected": -2.658045768737793, | |
| "logits/reference_chosen": -2.729498863220215, | |
| "logits/reference_rejected": -2.694502830505371, | |
| "logps/policy_chosen": -93.70889282226562, | |
| "logps/policy_rejected": -182.60643005371094, | |
| "logps/reference_chosen": -114.53910827636719, | |
| "logps/reference_rejected": -185.69065856933594, | |
| "loss": 0.6218, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2083020955324173, | |
| "rewards/margins": 0.17745977640151978, | |
| "rewards/rejected": 0.030842319130897522, | |
| "step": 138, | |
| "tokens/chosen_response_len_mean": 282.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 247.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 2.116685308038921, | |
| "learning_rate": 4.998069144627964e-07, | |
| "logits/policy_chosen": -2.870461940765381, | |
| "logits/policy_rejected": -2.6627469062805176, | |
| "logits/reference_chosen": -2.924255132675171, | |
| "logits/reference_rejected": -2.6669623851776123, | |
| "logps/policy_chosen": -149.49957275390625, | |
| "logps/policy_rejected": -133.25009155273438, | |
| "logps/reference_chosen": -191.32994079589844, | |
| "logps/reference_rejected": -157.3734588623047, | |
| "loss": 0.6358, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.4183034896850586, | |
| "rewards/margins": 0.1770700067281723, | |
| "rewards/rejected": 0.2412334829568863, | |
| "step": 139, | |
| "tokens/chosen_response_len_mean": 457.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 308.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 2.5870338720368387, | |
| "learning_rate": 4.99778349906548e-07, | |
| "logits/policy_chosen": -2.9523699283599854, | |
| "logits/policy_rejected": -2.9597270488739014, | |
| "logits/reference_chosen": -2.9815268516540527, | |
| "logits/reference_rejected": -2.9721755981445312, | |
| "logps/policy_chosen": -45.849361419677734, | |
| "logps/policy_rejected": -106.37723541259766, | |
| "logps/reference_chosen": -56.053375244140625, | |
| "logps/reference_rejected": -102.900390625, | |
| "loss": 0.6334, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.10204015672206879, | |
| "rewards/margins": 0.13680851459503174, | |
| "rewards/rejected": -0.03476836904883385, | |
| "step": 140, | |
| "tokens/chosen_response_len_mean": 174.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 184.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 3.1171856643556626, | |
| "learning_rate": 4.997478165843964e-07, | |
| "logits/policy_chosen": -2.8171520233154297, | |
| "logits/policy_rejected": -2.8751494884490967, | |
| "logits/reference_chosen": -2.9145286083221436, | |
| "logits/reference_rejected": -2.9242196083068848, | |
| "logps/policy_chosen": -170.06459045410156, | |
| "logps/policy_rejected": -296.7114562988281, | |
| "logps/reference_chosen": -197.37327575683594, | |
| "logps/reference_rejected": -302.43438720703125, | |
| "loss": 0.5862, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.27308669686317444, | |
| "rewards/margins": 0.21585765480995178, | |
| "rewards/rejected": 0.057229042053222656, | |
| "step": 141, | |
| "tokens/chosen_response_len_mean": 416.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 422.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 3.0084655845363835, | |
| "learning_rate": 4.997153147370065e-07, | |
| "logits/policy_chosen": -2.85317325592041, | |
| "logits/policy_rejected": -2.8581879138946533, | |
| "logits/reference_chosen": -2.7449471950531006, | |
| "logits/reference_rejected": -2.9342892169952393, | |
| "logps/policy_chosen": -95.86734008789062, | |
| "logps/policy_rejected": -255.03668212890625, | |
| "logps/reference_chosen": -114.49011993408203, | |
| "logps/reference_rejected": -234.34010314941406, | |
| "loss": 0.5691, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.18622779846191406, | |
| "rewards/margins": 0.39319345355033875, | |
| "rewards/rejected": -0.20696566998958588, | |
| "step": 142, | |
| "tokens/chosen_response_len_mean": 348.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 233.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 2.8051570403665753, | |
| "learning_rate": 4.996808446205598e-07, | |
| "logits/policy_chosen": -2.9738688468933105, | |
| "logits/policy_rejected": -2.8889176845550537, | |
| "logits/reference_chosen": -3.072221517562866, | |
| "logits/reference_rejected": -2.744046449661255, | |
| "logps/policy_chosen": -106.87481689453125, | |
| "logps/policy_rejected": -214.15478515625, | |
| "logps/reference_chosen": -126.41868591308594, | |
| "logps/reference_rejected": -216.4077606201172, | |
| "loss": 0.6244, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.19543880224227905, | |
| "rewards/margins": 0.17290906608104706, | |
| "rewards/rejected": 0.022529736161231995, | |
| "step": 143, | |
| "tokens/chosen_response_len_mean": 340.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 310.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 3.1099351541314864, | |
| "learning_rate": 4.996444065067515e-07, | |
| "logits/policy_chosen": -2.849621534347534, | |
| "logits/policy_rejected": -2.8288486003875732, | |
| "logits/reference_chosen": -2.9862475395202637, | |
| "logits/reference_rejected": -2.9479100704193115, | |
| "logps/policy_chosen": -331.7215576171875, | |
| "logps/policy_rejected": -571.941650390625, | |
| "logps/reference_chosen": -382.40533447265625, | |
| "logps/reference_rejected": -549.6744995117188, | |
| "loss": 0.5803, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.5068381428718567, | |
| "rewards/margins": 0.7295098304748535, | |
| "rewards/rejected": -0.2226717472076416, | |
| "step": 144, | |
| "tokens/chosen_response_len_mean": 687.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 550.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 2.712783580670268, | |
| "learning_rate": 4.996060006827887e-07, | |
| "logits/policy_chosen": -2.882990837097168, | |
| "logits/policy_rejected": -2.8665995597839355, | |
| "logits/reference_chosen": -2.9348299503326416, | |
| "logits/reference_rejected": -2.9280238151550293, | |
| "logps/policy_chosen": -75.93386840820312, | |
| "logps/policy_rejected": -159.058837890625, | |
| "logps/reference_chosen": -91.52436828613281, | |
| "logps/reference_rejected": -146.97247314453125, | |
| "loss": 0.5965, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.15590497851371765, | |
| "rewards/margins": 0.2767685055732727, | |
| "rewards/rejected": -0.12086354941129684, | |
| "step": 145, | |
| "tokens/chosen_response_len_mean": 248.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 191.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 2.364187725655967, | |
| "learning_rate": 4.995656274513881e-07, | |
| "logits/policy_chosen": -3.0416619777679443, | |
| "logits/policy_rejected": -2.995178699493408, | |
| "logits/reference_chosen": -3.053309440612793, | |
| "logits/reference_rejected": -3.0008325576782227, | |
| "logps/policy_chosen": -90.75337982177734, | |
| "logps/policy_rejected": -127.47672271728516, | |
| "logps/reference_chosen": -102.95506286621094, | |
| "logps/reference_rejected": -132.29486083984375, | |
| "loss": 0.6275, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.1220167949795723, | |
| "rewards/margins": 0.07383548468351364, | |
| "rewards/rejected": 0.04818130284547806, | |
| "step": 146, | |
| "tokens/chosen_response_len_mean": 403.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 352.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 3.163944044368287, | |
| "learning_rate": 4.995232871307736e-07, | |
| "logits/policy_chosen": -3.0712456703186035, | |
| "logits/policy_rejected": -2.916374921798706, | |
| "logits/reference_chosen": -3.115262746810913, | |
| "logits/reference_rejected": -2.6205530166625977, | |
| "logps/policy_chosen": -104.03935241699219, | |
| "logps/policy_rejected": -156.13577270507812, | |
| "logps/reference_chosen": -121.57315063476562, | |
| "logps/reference_rejected": -153.22605895996094, | |
| "loss": 0.583, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.17533791065216064, | |
| "rewards/margins": 0.2044350802898407, | |
| "rewards/rejected": -0.029097169637680054, | |
| "step": 147, | |
| "tokens/chosen_response_len_mean": 273.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 197.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 2.8981364214624588, | |
| "learning_rate": 4.994789800546737e-07, | |
| "logits/policy_chosen": -2.519289970397949, | |
| "logits/policy_rejected": -2.3818559646606445, | |
| "logits/reference_chosen": -2.5594940185546875, | |
| "logits/reference_rejected": -2.2251200675964355, | |
| "logps/policy_chosen": -64.7489242553711, | |
| "logps/policy_rejected": -90.21540832519531, | |
| "logps/reference_chosen": -80.6169204711914, | |
| "logps/reference_rejected": -92.92266845703125, | |
| "loss": 0.5887, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.15867996215820312, | |
| "rewards/margins": 0.13160738348960876, | |
| "rewards/rejected": 0.027072593569755554, | |
| "step": 148, | |
| "tokens/chosen_response_len_mean": 252.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 195.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 2.7118945452462073, | |
| "learning_rate": 4.994327065723192e-07, | |
| "logits/policy_chosen": -2.6516730785369873, | |
| "logits/policy_rejected": -2.5235743522644043, | |
| "logits/reference_chosen": -2.629504442214966, | |
| "logits/reference_rejected": -2.4736945629119873, | |
| "logps/policy_chosen": -80.69113159179688, | |
| "logps/policy_rejected": -134.16526794433594, | |
| "logps/reference_chosen": -103.33537292480469, | |
| "logps/reference_rejected": -144.43687438964844, | |
| "loss": 0.5924, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2264423370361328, | |
| "rewards/margins": 0.12372627854347229, | |
| "rewards/rejected": 0.10271605104207993, | |
| "step": 149, | |
| "tokens/chosen_response_len_mean": 244.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 244.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 2.5381060398035205, | |
| "learning_rate": 4.9938446704844e-07, | |
| "logits/policy_chosen": -2.8819587230682373, | |
| "logits/policy_rejected": -2.9721384048461914, | |
| "logits/reference_chosen": -2.7422800064086914, | |
| "logits/reference_rejected": -2.9950976371765137, | |
| "logps/policy_chosen": -57.52130126953125, | |
| "logps/policy_rejected": -120.23619079589844, | |
| "logps/reference_chosen": -59.788352966308594, | |
| "logps/reference_rejected": -115.03448486328125, | |
| "loss": 0.619, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.022670526057481766, | |
| "rewards/margins": 0.07468754798173904, | |
| "rewards/rejected": -0.05201702564954758, | |
| "step": 150, | |
| "tokens/chosen_response_len_mean": 145.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 157.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 2.8034120172472052, | |
| "learning_rate": 4.993342618632626e-07, | |
| "logits/policy_chosen": -3.0852291584014893, | |
| "logits/policy_rejected": -2.92305850982666, | |
| "logits/reference_chosen": -3.162454128265381, | |
| "logits/reference_rejected": -2.4117138385772705, | |
| "logps/policy_chosen": -101.7428207397461, | |
| "logps/policy_rejected": -242.9844207763672, | |
| "logps/reference_chosen": -119.85108184814453, | |
| "logps/reference_rejected": -215.4344940185547, | |
| "loss": 0.5702, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.18108254671096802, | |
| "rewards/margins": 0.45658186078071594, | |
| "rewards/rejected": -0.2754993140697479, | |
| "step": 151, | |
| "tokens/chosen_response_len_mean": 318.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 234.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 2.9459543515337776, | |
| "learning_rate": 4.992820914125069e-07, | |
| "logits/policy_chosen": -2.8646183013916016, | |
| "logits/policy_rejected": -2.754851818084717, | |
| "logits/reference_chosen": -3.0292770862579346, | |
| "logits/reference_rejected": -2.5228896141052246, | |
| "logps/policy_chosen": -150.56875610351562, | |
| "logps/policy_rejected": -277.60125732421875, | |
| "logps/reference_chosen": -179.77772521972656, | |
| "logps/reference_rejected": -275.3694152832031, | |
| "loss": 0.586, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.2920897901058197, | |
| "rewards/margins": 0.31440818309783936, | |
| "rewards/rejected": -0.022318381816148758, | |
| "step": 152, | |
| "tokens/chosen_response_len_mean": 543.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 491.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 2.9972427731212954, | |
| "learning_rate": 4.99227956107383e-07, | |
| "logits/policy_chosen": -2.133497714996338, | |
| "logits/policy_rejected": -2.275707721710205, | |
| "logits/reference_chosen": -2.4100184440612793, | |
| "logits/reference_rejected": -2.27435040473938, | |
| "logps/policy_chosen": -149.89268493652344, | |
| "logps/policy_rejected": -225.88043212890625, | |
| "logps/reference_chosen": -195.48631286621094, | |
| "logps/reference_rejected": -232.33531188964844, | |
| "loss": 0.5712, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.4559362828731537, | |
| "rewards/margins": 0.3913875222206116, | |
| "rewards/rejected": 0.06454875320196152, | |
| "step": 153, | |
| "tokens/chosen_response_len_mean": 551.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 501.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 2.6455429598648053, | |
| "learning_rate": 4.991718563745882e-07, | |
| "logits/policy_chosen": -3.022681951522827, | |
| "logits/policy_rejected": -3.033928155899048, | |
| "logits/reference_chosen": -3.0507426261901855, | |
| "logits/reference_rejected": -3.0070152282714844, | |
| "logps/policy_chosen": -17.34947395324707, | |
| "logps/policy_rejected": -55.760353088378906, | |
| "logps/reference_chosen": -20.67547607421875, | |
| "logps/reference_rejected": -50.72275924682617, | |
| "loss": 0.5857, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.03326002135872841, | |
| "rewards/margins": 0.08363597095012665, | |
| "rewards/rejected": -0.05037594214081764, | |
| "step": 154, | |
| "tokens/chosen_response_len_mean": 28.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 28.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 2.188503580126624, | |
| "learning_rate": 4.991137926563035e-07, | |
| "logits/policy_chosen": -2.8026483058929443, | |
| "logits/policy_rejected": -2.840829372406006, | |
| "logits/reference_chosen": -2.866769790649414, | |
| "logits/reference_rejected": -2.868910789489746, | |
| "logps/policy_chosen": -45.941871643066406, | |
| "logps/policy_rejected": -90.47376251220703, | |
| "logps/reference_chosen": -53.47924041748047, | |
| "logps/reference_rejected": -84.99751281738281, | |
| "loss": 0.6102, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.07537368685007095, | |
| "rewards/margins": 0.1301361620426178, | |
| "rewards/rejected": -0.054762475192546844, | |
| "step": 155, | |
| "tokens/chosen_response_len_mean": 173.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 133.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 2.5667127080808974, | |
| "learning_rate": 4.990537654101903e-07, | |
| "logits/policy_chosen": -2.980593681335449, | |
| "logits/policy_rejected": -3.0731048583984375, | |
| "logits/reference_chosen": -3.066528797149658, | |
| "logits/reference_rejected": -3.1092565059661865, | |
| "logps/policy_chosen": -92.3929672241211, | |
| "logps/policy_rejected": -223.2591552734375, | |
| "logps/reference_chosen": -117.49671936035156, | |
| "logps/reference_rejected": -224.93527221679688, | |
| "loss": 0.5677, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.25103750824928284, | |
| "rewards/margins": 0.2342762053012848, | |
| "rewards/rejected": 0.016761288046836853, | |
| "step": 156, | |
| "tokens/chosen_response_len_mean": 291.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 321.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 2.742556033679143, | |
| "learning_rate": 4.989917751093862e-07, | |
| "logits/policy_chosen": -2.9195590019226074, | |
| "logits/policy_rejected": -2.7180957794189453, | |
| "logits/reference_chosen": -3.0530753135681152, | |
| "logits/reference_rejected": -2.1471331119537354, | |
| "logps/policy_chosen": -86.05413818359375, | |
| "logps/policy_rejected": -171.83856201171875, | |
| "logps/reference_chosen": -108.33873748779297, | |
| "logps/reference_rejected": -158.24143981933594, | |
| "loss": 0.5624, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.22284595668315887, | |
| "rewards/margins": 0.35881713032722473, | |
| "rewards/rejected": -0.13597117364406586, | |
| "step": 157, | |
| "tokens/chosen_response_len_mean": 300.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 193.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 2.320383809339318, | |
| "learning_rate": 4.989278222425021e-07, | |
| "logits/policy_chosen": -2.9507193565368652, | |
| "logits/policy_rejected": -3.0587496757507324, | |
| "logits/reference_chosen": -3.0194993019104004, | |
| "logits/reference_rejected": -3.0736992359161377, | |
| "logps/policy_chosen": -103.41101837158203, | |
| "logps/policy_rejected": -312.6053466796875, | |
| "logps/reference_chosen": -126.41258239746094, | |
| "logps/reference_rejected": -292.84515380859375, | |
| "loss": 0.5961, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2300155907869339, | |
| "rewards/margins": 0.4276174306869507, | |
| "rewards/rejected": -0.19760185480117798, | |
| "step": 158, | |
| "tokens/chosen_response_len_mean": 338.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 361.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 2.516259723460725, | |
| "learning_rate": 4.988619073136179e-07, | |
| "logits/policy_chosen": -2.578339099884033, | |
| "logits/policy_rejected": -2.6243929862976074, | |
| "logits/reference_chosen": -2.651563882827759, | |
| "logits/reference_rejected": -2.7075228691101074, | |
| "logps/policy_chosen": -60.65949249267578, | |
| "logps/policy_rejected": -124.75245666503906, | |
| "logps/reference_chosen": -77.07334899902344, | |
| "logps/reference_rejected": -121.59002685546875, | |
| "loss": 0.5969, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.16413861513137817, | |
| "rewards/margins": 0.19576290249824524, | |
| "rewards/rejected": -0.031624287366867065, | |
| "step": 159, | |
| "tokens/chosen_response_len_mean": 344.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 347.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 2.715123355655463, | |
| "learning_rate": 4.987940308422783e-07, | |
| "logits/policy_chosen": -2.0114169120788574, | |
| "logits/policy_rejected": -2.0045886039733887, | |
| "logits/reference_chosen": -2.092721700668335, | |
| "logits/reference_rejected": -1.7612866163253784, | |
| "logps/policy_chosen": -84.94512176513672, | |
| "logps/policy_rejected": -133.75323486328125, | |
| "logps/reference_chosen": -107.3111801147461, | |
| "logps/reference_rejected": -128.45538330078125, | |
| "loss": 0.5549, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.22366058826446533, | |
| "rewards/margins": 0.27663907408714294, | |
| "rewards/rejected": -0.05297844856977463, | |
| "step": 160, | |
| "tokens/chosen_response_len_mean": 425.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 374.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 2.6648898364230735, | |
| "learning_rate": 4.987241933634892e-07, | |
| "logits/policy_chosen": -2.6987576484680176, | |
| "logits/policy_rejected": -2.7210419178009033, | |
| "logits/reference_chosen": -2.804107189178467, | |
| "logits/reference_rejected": -2.7485485076904297, | |
| "logps/policy_chosen": -101.68658447265625, | |
| "logps/policy_rejected": -234.5952606201172, | |
| "logps/reference_chosen": -123.14704132080078, | |
| "logps/reference_rejected": -228.36183166503906, | |
| "loss": 0.5769, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.21460454165935516, | |
| "rewards/margins": 0.276938796043396, | |
| "rewards/rejected": -0.06233424320816994, | |
| "step": 161, | |
| "tokens/chosen_response_len_mean": 409.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 407.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 2.82572590776649, | |
| "learning_rate": 4.986523954277134e-07, | |
| "logits/policy_chosen": -2.5539088249206543, | |
| "logits/policy_rejected": -2.5690407752990723, | |
| "logits/reference_chosen": -2.634115695953369, | |
| "logits/reference_rejected": -2.615056037902832, | |
| "logps/policy_chosen": -37.4378662109375, | |
| "logps/policy_rejected": -93.33854675292969, | |
| "logps/reference_chosen": -45.8214225769043, | |
| "logps/reference_rejected": -89.81768798828125, | |
| "loss": 0.5852, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.08383554965257645, | |
| "rewards/margins": 0.11904413998126984, | |
| "rewards/rejected": -0.03520859405398369, | |
| "step": 162, | |
| "tokens/chosen_response_len_mean": 228.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 240.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 2.7170854054640863, | |
| "learning_rate": 4.985786376008658e-07, | |
| "logits/policy_chosen": -2.820416212081909, | |
| "logits/policy_rejected": -2.79477858543396, | |
| "logits/reference_chosen": -2.9494552612304688, | |
| "logits/reference_rejected": -2.837876081466675, | |
| "logps/policy_chosen": -71.54603576660156, | |
| "logps/policy_rejected": -156.01953125, | |
| "logps/reference_chosen": -90.15393829345703, | |
| "logps/reference_rejected": -154.65646362304688, | |
| "loss": 0.5657, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.1860790103673935, | |
| "rewards/margins": 0.1997096687555313, | |
| "rewards/rejected": -0.01363067515194416, | |
| "step": 163, | |
| "tokens/chosen_response_len_mean": 302.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 296.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 2.3740765633517396, | |
| "learning_rate": 4.985029204643094e-07, | |
| "logits/policy_chosen": -2.747786045074463, | |
| "logits/policy_rejected": -2.7663800716400146, | |
| "logits/reference_chosen": -2.8246467113494873, | |
| "logits/reference_rejected": -2.6436727046966553, | |
| "logps/policy_chosen": -58.324737548828125, | |
| "logps/policy_rejected": -223.62765502929688, | |
| "logps/reference_chosen": -85.9714126586914, | |
| "logps/reference_rejected": -203.6089630126953, | |
| "loss": 0.6018, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.27646681666374207, | |
| "rewards/margins": 0.4766535758972168, | |
| "rewards/rejected": -0.20018675923347473, | |
| "step": 164, | |
| "tokens/chosen_response_len_mean": 223.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 216.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 2.188806416183682, | |
| "learning_rate": 4.984252446148508e-07, | |
| "logits/policy_chosen": -2.701019287109375, | |
| "logits/policy_rejected": -2.729661226272583, | |
| "logits/reference_chosen": -2.8306493759155273, | |
| "logits/reference_rejected": -2.8457367420196533, | |
| "logps/policy_chosen": -154.78518676757812, | |
| "logps/policy_rejected": -214.3829345703125, | |
| "logps/reference_chosen": -194.75637817382812, | |
| "logps/reference_rejected": -246.5928497314453, | |
| "loss": 0.6206, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.39971184730529785, | |
| "rewards/margins": 0.07761266827583313, | |
| "rewards/rejected": 0.3220991790294647, | |
| "step": 165, | |
| "tokens/chosen_response_len_mean": 429.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 445.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 2.8889287147059104, | |
| "learning_rate": 4.983456106647348e-07, | |
| "logits/policy_chosen": -2.861962080001831, | |
| "logits/policy_rejected": -2.8529515266418457, | |
| "logits/reference_chosen": -3.064021587371826, | |
| "logits/reference_rejected": -2.927248001098633, | |
| "logps/policy_chosen": -146.7970428466797, | |
| "logps/policy_rejected": -283.18389892578125, | |
| "logps/reference_chosen": -181.90396118164062, | |
| "logps/reference_rejected": -277.1955871582031, | |
| "loss": 0.5642, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.35106930136680603, | |
| "rewards/margins": 0.4109525978565216, | |
| "rewards/rejected": -0.05988328531384468, | |
| "step": 166, | |
| "tokens/chosen_response_len_mean": 468.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 421.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 2.9841019270826292, | |
| "learning_rate": 4.982640192416404e-07, | |
| "logits/policy_chosen": -2.6893157958984375, | |
| "logits/policy_rejected": -2.681230068206787, | |
| "logits/reference_chosen": -2.948234796524048, | |
| "logits/reference_rejected": -2.248135566711426, | |
| "logps/policy_chosen": -122.96820831298828, | |
| "logps/policy_rejected": -197.66404724121094, | |
| "logps/reference_chosen": -155.029541015625, | |
| "logps/reference_rejected": -194.3126678466797, | |
| "loss": 0.5642, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.32061344385147095, | |
| "rewards/margins": 0.35412728786468506, | |
| "rewards/rejected": -0.0335138775408268, | |
| "step": 167, | |
| "tokens/chosen_response_len_mean": 452.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 366.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 2.766687221527344, | |
| "learning_rate": 4.981804709886755e-07, | |
| "logits/policy_chosen": -2.4212305545806885, | |
| "logits/policy_rejected": -2.4506514072418213, | |
| "logits/reference_chosen": -2.6806509494781494, | |
| "logits/reference_rejected": -2.608292579650879, | |
| "logps/policy_chosen": -60.471797943115234, | |
| "logps/policy_rejected": -120.82171630859375, | |
| "logps/reference_chosen": -92.31079864501953, | |
| "logps/reference_rejected": -130.82656860351562, | |
| "loss": 0.54, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.31838998198509216, | |
| "rewards/margins": 0.21834133565425873, | |
| "rewards/rejected": 0.10004864633083344, | |
| "step": 168, | |
| "tokens/chosen_response_len_mean": 324.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 310.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 2.1828651372724637, | |
| "learning_rate": 4.980949665643719e-07, | |
| "logits/policy_chosen": -2.7441906929016113, | |
| "logits/policy_rejected": -2.7218408584594727, | |
| "logits/reference_chosen": -2.8971827030181885, | |
| "logits/reference_rejected": -2.860405445098877, | |
| "logps/policy_chosen": -92.7531509399414, | |
| "logps/policy_rejected": -210.39797973632812, | |
| "logps/reference_chosen": -123.75934600830078, | |
| "logps/reference_rejected": -200.2626190185547, | |
| "loss": 0.5755, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.3100619614124298, | |
| "rewards/margins": 0.41141554713249207, | |
| "rewards/rejected": -0.10135360062122345, | |
| "step": 169, | |
| "tokens/chosen_response_len_mean": 517.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 465.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 2.9989447327164425, | |
| "learning_rate": 4.980075066426795e-07, | |
| "logits/policy_chosen": -1.6698166131973267, | |
| "logits/policy_rejected": -1.7434771060943604, | |
| "logits/reference_chosen": -1.803000569343567, | |
| "logits/reference_rejected": -1.8658090829849243, | |
| "logps/policy_chosen": -61.05796813964844, | |
| "logps/policy_rejected": -89.74996948242188, | |
| "logps/reference_chosen": -77.35304260253906, | |
| "logps/reference_rejected": -98.12295532226562, | |
| "loss": 0.5561, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.1629507839679718, | |
| "rewards/margins": 0.07922087609767914, | |
| "rewards/rejected": 0.08372990787029266, | |
| "step": 170, | |
| "tokens/chosen_response_len_mean": 243.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 245.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 2.5554431416025243, | |
| "learning_rate": 4.979180919129624e-07, | |
| "logits/policy_chosen": -2.85784912109375, | |
| "logits/policy_rejected": -2.838977098464966, | |
| "logits/reference_chosen": -2.8757684230804443, | |
| "logits/reference_rejected": -2.8608899116516113, | |
| "logps/policy_chosen": -53.759185791015625, | |
| "logps/policy_rejected": -90.618408203125, | |
| "logps/reference_chosen": -62.647789001464844, | |
| "logps/reference_rejected": -86.81078338623047, | |
| "loss": 0.5702, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.08888600766658783, | |
| "rewards/margins": 0.1269623190164566, | |
| "rewards/rejected": -0.03807631880044937, | |
| "step": 171, | |
| "tokens/chosen_response_len_mean": 207.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 209.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 2.7869612955855776, | |
| "learning_rate": 4.978267230799918e-07, | |
| "logits/policy_chosen": -2.0658278465270996, | |
| "logits/policy_rejected": -2.2496557235717773, | |
| "logits/reference_chosen": -2.1524839401245117, | |
| "logits/reference_rejected": -2.3013737201690674, | |
| "logps/policy_chosen": -86.49990844726562, | |
| "logps/policy_rejected": -197.741943359375, | |
| "logps/reference_chosen": -109.14804077148438, | |
| "logps/reference_rejected": -182.33616638183594, | |
| "loss": 0.5549, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.22648128867149353, | |
| "rewards/margins": 0.380539208650589, | |
| "rewards/rejected": -0.15405790507793427, | |
| "step": 172, | |
| "tokens/chosen_response_len_mean": 443.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 439.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 2.9395952260087506, | |
| "learning_rate": 4.977334008639417e-07, | |
| "logits/policy_chosen": -2.605189800262451, | |
| "logits/policy_rejected": -2.5856776237487793, | |
| "logits/reference_chosen": -2.778407335281372, | |
| "logits/reference_rejected": -2.793548583984375, | |
| "logps/policy_chosen": -112.27513122558594, | |
| "logps/policy_rejected": -266.2890625, | |
| "logps/reference_chosen": -140.7950439453125, | |
| "logps/reference_rejected": -241.72708129882812, | |
| "loss": 0.5051, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.28519904613494873, | |
| "rewards/margins": 0.5308188796043396, | |
| "rewards/rejected": -0.24561986327171326, | |
| "step": 173, | |
| "tokens/chosen_response_len_mean": 462.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 293.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 2.7423154795634153, | |
| "learning_rate": 4.976381260003826e-07, | |
| "logits/policy_chosen": -2.7179572582244873, | |
| "logits/policy_rejected": -2.756556510925293, | |
| "logits/reference_chosen": -2.8246583938598633, | |
| "logits/reference_rejected": -2.793865203857422, | |
| "logps/policy_chosen": -80.221923828125, | |
| "logps/policy_rejected": -204.06829833984375, | |
| "logps/reference_chosen": -96.3402328491211, | |
| "logps/reference_rejected": -181.1507568359375, | |
| "loss": 0.5916, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.16118302941322327, | |
| "rewards/margins": 0.3903582692146301, | |
| "rewards/rejected": -0.22917523980140686, | |
| "step": 174, | |
| "tokens/chosen_response_len_mean": 307.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 307.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 2.8390846969584946, | |
| "learning_rate": 4.97540899240276e-07, | |
| "logits/policy_chosen": -2.6149544715881348, | |
| "logits/policy_rejected": -2.7186009883880615, | |
| "logits/reference_chosen": -2.755988597869873, | |
| "logits/reference_rejected": -2.8059897422790527, | |
| "logps/policy_chosen": -83.78113555908203, | |
| "logps/policy_rejected": -158.60336303710938, | |
| "logps/reference_chosen": -105.53045654296875, | |
| "logps/reference_rejected": -161.6735076904297, | |
| "loss": 0.5707, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2174932360649109, | |
| "rewards/margins": 0.18679174780845642, | |
| "rewards/rejected": 0.030701473355293274, | |
| "step": 175, | |
| "tokens/chosen_response_len_mean": 227.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 221.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 1.928131402745428, | |
| "learning_rate": 4.974417213499681e-07, | |
| "logits/policy_chosen": -2.97554874420166, | |
| "logits/policy_rejected": -3.0169177055358887, | |
| "logits/reference_chosen": -3.1351237297058105, | |
| "logits/reference_rejected": -3.054442882537842, | |
| "logps/policy_chosen": -155.18731689453125, | |
| "logps/policy_rejected": -306.79449462890625, | |
| "logps/reference_chosen": -198.6287841796875, | |
| "logps/reference_rejected": -296.349609375, | |
| "loss": 0.5884, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.4344147741794586, | |
| "rewards/margins": 0.5388633608818054, | |
| "rewards/rejected": -0.1044485792517662, | |
| "step": 176, | |
| "tokens/chosen_response_len_mean": 399.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 380.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 2.5891349823773604, | |
| "learning_rate": 4.973405931111844e-07, | |
| "logits/policy_chosen": -2.475595474243164, | |
| "logits/policy_rejected": -2.5474050045013428, | |
| "logits/reference_chosen": -2.574971914291382, | |
| "logits/reference_rejected": -2.7303850650787354, | |
| "logps/policy_chosen": -60.937068939208984, | |
| "logps/policy_rejected": -196.1728515625, | |
| "logps/reference_chosen": -93.98703002929688, | |
| "logps/reference_rejected": -193.26068115234375, | |
| "loss": 0.5721, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.330499529838562, | |
| "rewards/margins": 0.3596211075782776, | |
| "rewards/rejected": -0.029121577739715576, | |
| "step": 177, | |
| "tokens/chosen_response_len_mean": 277.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 290.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 3.186008328730442, | |
| "learning_rate": 4.972375153210227e-07, | |
| "logits/policy_chosen": -2.73168683052063, | |
| "logits/policy_rejected": -2.80218505859375, | |
| "logits/reference_chosen": -2.8586461544036865, | |
| "logits/reference_rejected": -2.856407642364502, | |
| "logps/policy_chosen": -41.751312255859375, | |
| "logps/policy_rejected": -121.0931396484375, | |
| "logps/reference_chosen": -52.108253479003906, | |
| "logps/reference_rejected": -104.97093200683594, | |
| "loss": 0.5198, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.1035693883895874, | |
| "rewards/margins": 0.26479148864746094, | |
| "rewards/rejected": -0.16122210025787354, | |
| "step": 178, | |
| "tokens/chosen_response_len_mean": 131.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 131.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 2.4038166772062186, | |
| "learning_rate": 4.971324887919477e-07, | |
| "logits/policy_chosen": -2.768160820007324, | |
| "logits/policy_rejected": -2.2585337162017822, | |
| "logits/reference_chosen": -2.8858625888824463, | |
| "logits/reference_rejected": -2.4065427780151367, | |
| "logps/policy_chosen": -67.46320343017578, | |
| "logps/policy_rejected": -281.3670654296875, | |
| "logps/reference_chosen": -94.11170196533203, | |
| "logps/reference_rejected": -263.22698974609375, | |
| "loss": 0.5596, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.26648497581481934, | |
| "rewards/margins": 0.4478859305381775, | |
| "rewards/rejected": -0.18140093982219696, | |
| "step": 179, | |
| "tokens/chosen_response_len_mean": 239.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 333.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 2.530913530757402, | |
| "learning_rate": 4.970255143517838e-07, | |
| "logits/policy_chosen": -2.6580893993377686, | |
| "logits/policy_rejected": -2.717331886291504, | |
| "logits/reference_chosen": -2.90189790725708, | |
| "logits/reference_rejected": -2.8899142742156982, | |
| "logps/policy_chosen": -129.58033752441406, | |
| "logps/policy_rejected": -165.34884643554688, | |
| "logps/reference_chosen": -179.34756469726562, | |
| "logps/reference_rejected": -170.3519287109375, | |
| "loss": 0.5522, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.4976721405982971, | |
| "rewards/margins": 0.4476412534713745, | |
| "rewards/rejected": 0.050030872225761414, | |
| "step": 180, | |
| "tokens/chosen_response_len_mean": 500.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 318.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 2.633909183828016, | |
| "learning_rate": 4.969165928437091e-07, | |
| "logits/policy_chosen": -2.570159912109375, | |
| "logits/policy_rejected": -2.5334911346435547, | |
| "logits/reference_chosen": -2.760227680206299, | |
| "logits/reference_rejected": -2.3865246772766113, | |
| "logps/policy_chosen": -55.27775955200195, | |
| "logps/policy_rejected": -119.49053955078125, | |
| "logps/reference_chosen": -72.7205581665039, | |
| "logps/reference_rejected": -111.53352355957031, | |
| "loss": 0.5199, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.17442797124385834, | |
| "rewards/margins": 0.2539982497692108, | |
| "rewards/rejected": -0.07957027852535248, | |
| "step": 181, | |
| "tokens/chosen_response_len_mean": 307.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 267.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 2.3888599137710167, | |
| "learning_rate": 4.968057251262486e-07, | |
| "logits/policy_chosen": -2.6059675216674805, | |
| "logits/policy_rejected": -2.395353078842163, | |
| "logits/reference_chosen": -2.9607644081115723, | |
| "logits/reference_rejected": -1.8757669925689697, | |
| "logps/policy_chosen": -74.94976043701172, | |
| "logps/policy_rejected": -134.52859497070312, | |
| "logps/reference_chosen": -111.65763092041016, | |
| "logps/reference_rejected": -96.27473449707031, | |
| "loss": 0.5536, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.36707866191864014, | |
| "rewards/margins": 0.7496173977851868, | |
| "rewards/rejected": -0.38253867626190186, | |
| "step": 182, | |
| "tokens/chosen_response_len_mean": 351.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 148.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 2.826021624132104, | |
| "learning_rate": 4.966929120732675e-07, | |
| "logits/policy_chosen": -2.3233590126037598, | |
| "logits/policy_rejected": -2.4008846282958984, | |
| "logits/reference_chosen": -2.5622141361236572, | |
| "logits/reference_rejected": -2.381047248840332, | |
| "logps/policy_chosen": -185.09976196289062, | |
| "logps/policy_rejected": -336.95941162109375, | |
| "logps/reference_chosen": -236.7534637451172, | |
| "logps/reference_rejected": -320.8805847167969, | |
| "loss": 0.5087, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.516537070274353, | |
| "rewards/margins": 0.6773253679275513, | |
| "rewards/rejected": -0.16078826785087585, | |
| "step": 183, | |
| "tokens/chosen_response_len_mean": 441.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 384.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 2.26864738127188, | |
| "learning_rate": 4.965781545739638e-07, | |
| "logits/policy_chosen": -2.7422099113464355, | |
| "logits/policy_rejected": -2.7613723278045654, | |
| "logits/reference_chosen": -2.9637043476104736, | |
| "logits/reference_rejected": -2.921086549758911, | |
| "logps/policy_chosen": -45.85551452636719, | |
| "logps/policy_rejected": -118.8583984375, | |
| "logps/reference_chosen": -65.04806518554688, | |
| "logps/reference_rejected": -119.62251281738281, | |
| "loss": 0.5676, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.191925510764122, | |
| "rewards/margins": 0.1842842549085617, | |
| "rewards/rejected": 0.007641255855560303, | |
| "step": 184, | |
| "tokens/chosen_response_len_mean": 266.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 242.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 2.8182732561665023, | |
| "learning_rate": 4.964614535328625e-07, | |
| "logits/policy_chosen": -2.523998975753784, | |
| "logits/policy_rejected": -2.4280190467834473, | |
| "logits/reference_chosen": -2.717611789703369, | |
| "logits/reference_rejected": -2.5818982124328613, | |
| "logps/policy_chosen": -250.7300262451172, | |
| "logps/policy_rejected": -131.51202392578125, | |
| "logps/reference_chosen": -304.0659484863281, | |
| "logps/reference_rejected": -136.93946838378906, | |
| "loss": 0.5284, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.533359169960022, | |
| "rewards/margins": 0.47908464074134827, | |
| "rewards/rejected": 0.0542745515704155, | |
| "step": 185, | |
| "tokens/chosen_response_len_mean": 653.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 300.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 2.658487252908494, | |
| "learning_rate": 4.963428098698069e-07, | |
| "logits/policy_chosen": -2.8848061561584473, | |
| "logits/policy_rejected": -2.887396812438965, | |
| "logits/reference_chosen": -3.0770082473754883, | |
| "logits/reference_rejected": -3.012545347213745, | |
| "logps/policy_chosen": -26.844032287597656, | |
| "logps/policy_rejected": -64.93582916259766, | |
| "logps/reference_chosen": -37.33853530883789, | |
| "logps/reference_rejected": -57.781166076660156, | |
| "loss": 0.5617, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.10494504123926163, | |
| "rewards/margins": 0.17649173736572266, | |
| "rewards/rejected": -0.07154669612646103, | |
| "step": 186, | |
| "tokens/chosen_response_len_mean": 101.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 99.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 3.192921534395214, | |
| "learning_rate": 4.962222245199527e-07, | |
| "logits/policy_chosen": -2.5711615085601807, | |
| "logits/policy_rejected": -2.556881904602051, | |
| "logits/reference_chosen": -2.9153711795806885, | |
| "logits/reference_rejected": -2.4689133167266846, | |
| "logps/policy_chosen": -79.51646423339844, | |
| "logps/policy_rejected": -212.90431213378906, | |
| "logps/reference_chosen": -108.06434631347656, | |
| "logps/reference_rejected": -189.78848266601562, | |
| "loss": 0.5117, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.285478800535202, | |
| "rewards/margins": 0.5166370868682861, | |
| "rewards/rejected": -0.2311583161354065, | |
| "step": 187, | |
| "tokens/chosen_response_len_mean": 368.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 330.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 3.185300778929063, | |
| "learning_rate": 4.960996984337598e-07, | |
| "logits/policy_chosen": -2.322800397872925, | |
| "logits/policy_rejected": -2.180086851119995, | |
| "logits/reference_chosen": -2.647416830062866, | |
| "logits/reference_rejected": -2.404383420944214, | |
| "logps/policy_chosen": -88.6377944946289, | |
| "logps/policy_rejected": -154.1780242919922, | |
| "logps/reference_chosen": -146.73565673828125, | |
| "logps/reference_rejected": -158.4337921142578, | |
| "loss": 0.4879, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.5809786319732666, | |
| "rewards/margins": 0.5384209156036377, | |
| "rewards/rejected": 0.042557742446660995, | |
| "step": 188, | |
| "tokens/chosen_response_len_mean": 496.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 375.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 3.018235573859185, | |
| "learning_rate": 4.959752325769852e-07, | |
| "logits/policy_chosen": -2.469785213470459, | |
| "logits/policy_rejected": -2.580235481262207, | |
| "logits/reference_chosen": -2.680108070373535, | |
| "logits/reference_rejected": -2.8124053478240967, | |
| "logps/policy_chosen": -57.80915069580078, | |
| "logps/policy_rejected": -210.04135131835938, | |
| "logps/reference_chosen": -90.7098617553711, | |
| "logps/reference_rejected": -191.3103485107422, | |
| "loss": 0.5062, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.329007089138031, | |
| "rewards/margins": 0.5163170099258423, | |
| "rewards/rejected": -0.18730990588665009, | |
| "step": 189, | |
| "tokens/chosen_response_len_mean": 332.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 348.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 3.1505949518170437, | |
| "learning_rate": 4.958488279306753e-07, | |
| "logits/policy_chosen": -2.3762245178222656, | |
| "logits/policy_rejected": -2.466752767562866, | |
| "logits/reference_chosen": -2.847898483276367, | |
| "logits/reference_rejected": -2.2204389572143555, | |
| "logps/policy_chosen": -104.57975006103516, | |
| "logps/policy_rejected": -250.35414123535156, | |
| "logps/reference_chosen": -159.56100463867188, | |
| "logps/reference_rejected": -225.86146545410156, | |
| "loss": 0.522, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.5498126149177551, | |
| "rewards/margins": 0.7947396039962769, | |
| "rewards/rejected": -0.24492698907852173, | |
| "step": 190, | |
| "tokens/chosen_response_len_mean": 512.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 435.75, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 3.056135297935867, | |
| "learning_rate": 4.957204854911578e-07, | |
| "logits/policy_chosen": -2.485416889190674, | |
| "logits/policy_rejected": -2.3778135776519775, | |
| "logits/reference_chosen": -2.856623411178589, | |
| "logits/reference_rejected": -2.3928959369659424, | |
| "logps/policy_chosen": -83.4710693359375, | |
| "logps/policy_rejected": -267.17791748046875, | |
| "logps/reference_chosen": -124.20071411132812, | |
| "logps/reference_rejected": -209.21670532226562, | |
| "loss": 0.5036, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.40729638934135437, | |
| "rewards/margins": 0.9869081377983093, | |
| "rewards/rejected": -0.5796117782592773, | |
| "step": 191, | |
| "tokens/chosen_response_len_mean": 528.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 410.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 2.503765131980355, | |
| "learning_rate": 4.955902062700349e-07, | |
| "logits/policy_chosen": -2.3622069358825684, | |
| "logits/policy_rejected": -2.4439258575439453, | |
| "logits/reference_chosen": -2.7395248413085938, | |
| "logits/reference_rejected": -2.6338257789611816, | |
| "logps/policy_chosen": -76.7771224975586, | |
| "logps/policy_rejected": -262.30841064453125, | |
| "logps/reference_chosen": -117.23094940185547, | |
| "logps/reference_rejected": -224.7442626953125, | |
| "loss": 0.5445, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.40453827381134033, | |
| "rewards/margins": 0.7801797986030579, | |
| "rewards/rejected": -0.37564149498939514, | |
| "step": 192, | |
| "tokens/chosen_response_len_mean": 359.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 367.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 2.9703660280881734, | |
| "learning_rate": 4.954579912941737e-07, | |
| "logits/policy_chosen": -2.622279644012451, | |
| "logits/policy_rejected": -2.6470251083374023, | |
| "logits/reference_chosen": -2.853696823120117, | |
| "logits/reference_rejected": -2.840925931930542, | |
| "logps/policy_chosen": -38.201744079589844, | |
| "logps/policy_rejected": -138.85470581054688, | |
| "logps/reference_chosen": -63.35798645019531, | |
| "logps/reference_rejected": -113.46359252929688, | |
| "loss": 0.4864, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2515623867511749, | |
| "rewards/margins": 0.5054734349250793, | |
| "rewards/rejected": -0.25391101837158203, | |
| "step": 193, | |
| "tokens/chosen_response_len_mean": 157.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 169.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 2.9246186619159347, | |
| "learning_rate": 4.953238416057e-07, | |
| "logits/policy_chosen": -2.605665683746338, | |
| "logits/policy_rejected": -2.5566539764404297, | |
| "logits/reference_chosen": -2.9360122680664062, | |
| "logits/reference_rejected": -2.068471908569336, | |
| "logps/policy_chosen": -84.89468383789062, | |
| "logps/policy_rejected": -218.09048461914062, | |
| "logps/reference_chosen": -133.76991271972656, | |
| "logps/reference_rejected": -183.8456573486328, | |
| "loss": 0.4881, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.48875218629837036, | |
| "rewards/margins": 0.8312005996704102, | |
| "rewards/rejected": -0.3424483835697174, | |
| "step": 194, | |
| "tokens/chosen_response_len_mean": 360.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 238.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 2.80153859644754, | |
| "learning_rate": 4.951877582619881e-07, | |
| "logits/policy_chosen": -2.8731517791748047, | |
| "logits/policy_rejected": -2.975919246673584, | |
| "logits/reference_chosen": -3.0668981075286865, | |
| "logits/reference_rejected": -3.1014294624328613, | |
| "logps/policy_chosen": -24.165613174438477, | |
| "logps/policy_rejected": -61.2584342956543, | |
| "logps/reference_chosen": -23.939205169677734, | |
| "logps/reference_rejected": -43.215232849121094, | |
| "loss": 0.5741, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.002264055423438549, | |
| "rewards/margins": 0.1781679093837738, | |
| "rewards/rejected": -0.18043196201324463, | |
| "step": 195, | |
| "tokens/chosen_response_len_mean": 62.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 63.5, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 2.7527879974178955, | |
| "learning_rate": 4.950497423356542e-07, | |
| "logits/policy_chosen": -2.9940450191497803, | |
| "logits/policy_rejected": -2.710885763168335, | |
| "logits/reference_chosen": -3.169459581375122, | |
| "logits/reference_rejected": -2.7128515243530273, | |
| "logps/policy_chosen": -120.05692291259766, | |
| "logps/policy_rejected": -105.24632263183594, | |
| "logps/reference_chosen": -160.91629028320312, | |
| "logps/reference_rejected": -78.08905029296875, | |
| "loss": 0.5115, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.4085935354232788, | |
| "rewards/margins": 0.6801662445068359, | |
| "rewards/rejected": -0.27157267928123474, | |
| "step": 196, | |
| "tokens/chosen_response_len_mean": 332.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 53.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 2.473954328567208, | |
| "learning_rate": 4.949097949145471e-07, | |
| "logits/policy_chosen": -2.994095802307129, | |
| "logits/policy_rejected": -2.9911117553710938, | |
| "logits/reference_chosen": -3.0934770107269287, | |
| "logits/reference_rejected": -3.05942440032959, | |
| "logps/policy_chosen": -54.1805419921875, | |
| "logps/policy_rejected": -158.96043395996094, | |
| "logps/reference_chosen": -74.52861785888672, | |
| "logps/reference_rejected": -128.6375732421875, | |
| "loss": 0.4544, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2034807801246643, | |
| "rewards/margins": 0.5067094564437866, | |
| "rewards/rejected": -0.3032286465167999, | |
| "step": 197, | |
| "tokens/chosen_response_len_mean": 182.25, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 165.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 2.6028329307586944, | |
| "learning_rate": 4.947679171017394e-07, | |
| "logits/policy_chosen": -2.7241291999816895, | |
| "logits/policy_rejected": -2.8773388862609863, | |
| "logits/reference_chosen": -2.9862842559814453, | |
| "logits/reference_rejected": -2.9953157901763916, | |
| "logps/policy_chosen": -56.624752044677734, | |
| "logps/policy_rejected": -255.8826141357422, | |
| "logps/reference_chosen": -101.90193176269531, | |
| "logps/reference_rejected": -203.60231018066406, | |
| "loss": 0.491, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.4527718722820282, | |
| "rewards/margins": 0.9755749106407166, | |
| "rewards/rejected": -0.5228029489517212, | |
| "step": 198, | |
| "tokens/chosen_response_len_mean": 225.5, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 217.25, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 2.8264702439738474, | |
| "learning_rate": 4.946241100155197e-07, | |
| "logits/policy_chosen": -2.6785213947296143, | |
| "logits/policy_rejected": -2.800018310546875, | |
| "logits/reference_chosen": -2.9413294792175293, | |
| "logits/reference_rejected": -2.913431406021118, | |
| "logps/policy_chosen": -38.42972183227539, | |
| "logps/policy_rejected": -164.37850952148438, | |
| "logps/reference_chosen": -65.64104461669922, | |
| "logps/reference_rejected": -128.12234497070312, | |
| "loss": 0.4973, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.27211323380470276, | |
| "rewards/margins": 0.6346749067306519, | |
| "rewards/rejected": -0.3625616431236267, | |
| "step": 199, | |
| "tokens/chosen_response_len_mean": 312.0, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 278.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 3.0030443321611644, | |
| "learning_rate": 4.944783747893825e-07, | |
| "logits/policy_chosen": -2.727503538131714, | |
| "logits/policy_rejected": -2.6386661529541016, | |
| "logits/reference_chosen": -2.9862632751464844, | |
| "logits/reference_rejected": -2.1630351543426514, | |
| "logps/policy_chosen": -84.61045837402344, | |
| "logps/policy_rejected": -191.14212036132812, | |
| "logps/reference_chosen": -125.34469604492188, | |
| "logps/reference_rejected": -150.1243896484375, | |
| "loss": 0.4714, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.4073423147201538, | |
| "rewards/margins": 0.8175195455551147, | |
| "rewards/rejected": -0.41017720103263855, | |
| "step": 200, | |
| "tokens/chosen_response_len_mean": 264.75, | |
| "tokens/chosen_response_len_std": 0.0, | |
| "tokens/rejected_response_len_mean": 164.0, | |
| "tokens/rejected_response_len_std": 0.0 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 1244, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 100, | |
| "total_flos": 0.0, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |