diff --git "a/checkpoint-800/trainer_state.json" "b/checkpoint-800/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-800/trainer_state.json" @@ -0,0 +1,11221 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8727074384673076, + "eval_steps": 500, + "global_step": 800, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 9.782608695652175e-07, + "logits/chosen": -1.8868483304977417, + "logits/rejected": -2.3036646842956543, + "logps/chosen": -466.7117004394531, + "logps/rejected": -99.3152084350586, + "loss": 0.6931, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 1 + }, + { + "epoch": 0.0, + "learning_rate": 1.956521739130435e-06, + "logits/chosen": -2.2296698093414307, + "logits/rejected": -2.469517469406128, + "logps/chosen": -335.50140380859375, + "logps/rejected": -97.97496032714844, + "loss": 0.6931, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 2 + }, + { + "epoch": 0.0, + "learning_rate": 2.9347826086956523e-06, + "logits/chosen": -2.125047206878662, + "logits/rejected": -2.445204019546509, + "logps/chosen": -484.7939758300781, + "logps/rejected": -125.93072509765625, + "loss": 1.7268, + "rewards/accuracies": 0.4375, + "rewards/chosen": -1.0339633226394653, + "rewards/margins": -0.9356753826141357, + "rewards/rejected": -0.09828801453113556, + "step": 3 + }, + { + "epoch": 0.0, + "learning_rate": 3.91304347826087e-06, + "logits/chosen": -1.7763867378234863, + "logits/rejected": -2.1961894035339355, + "logps/chosen": -386.8011779785156, + "logps/rejected": -106.04309844970703, + "loss": 2.3806, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.9238373041152954, + "rewards/margins": -1.2902063131332397, + "rewards/rejected": 0.36636874079704285, + "step": 4 + }, + { + "epoch": 0.01, + "learning_rate": 4.891304347826087e-06, + "logits/chosen": -1.7257812023162842, + "logits/rejected": -2.353095769882202, + "logps/chosen": -524.7261962890625, + "logps/rejected": -70.26946258544922, + "loss": 0.7488, + "rewards/accuracies": 0.625, + "rewards/chosen": 2.197690010070801, + "rewards/margins": 2.0008764266967773, + "rewards/rejected": 0.19681359827518463, + "step": 5 + }, + { + "epoch": 0.01, + "learning_rate": 5.869565217391305e-06, + "logits/chosen": -2.1483359336853027, + "logits/rejected": -2.4817819595336914, + "logps/chosen": -283.6588134765625, + "logps/rejected": -58.25455093383789, + "loss": 0.8712, + "rewards/accuracies": 0.5625, + "rewards/chosen": 0.4915351867675781, + "rewards/margins": 0.2005542814731598, + "rewards/rejected": 0.2909809350967407, + "step": 6 + }, + { + "epoch": 0.01, + "learning_rate": 6.847826086956523e-06, + "logits/chosen": -2.1019060611724854, + "logits/rejected": -2.300177812576294, + "logps/chosen": -297.35235595703125, + "logps/rejected": -175.5060272216797, + "loss": 1.1392, + "rewards/accuracies": 0.5625, + "rewards/chosen": 0.6018204689025879, + "rewards/margins": -0.036775827407836914, + "rewards/rejected": 0.6385962963104248, + "step": 7 + }, + { + "epoch": 0.01, + "learning_rate": 7.82608695652174e-06, + "logits/chosen": -2.0169122219085693, + "logits/rejected": -2.201702117919922, + "logps/chosen": -342.04852294921875, + "logps/rejected": -117.9761962890625, + "loss": 1.1799, + "rewards/accuracies": 0.5625, + "rewards/chosen": 1.111680507659912, + "rewards/margins": 0.17297500371932983, + "rewards/rejected": 0.9387054443359375, + "step": 8 + }, + { + "epoch": 0.01, + "learning_rate": 8.804347826086957e-06, + "logits/chosen": -1.6254596710205078, + "logits/rejected": -2.0724740028381348, + "logps/chosen": -480.1983642578125, + "logps/rejected": -108.05937194824219, + "loss": 0.5203, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.161550283432007, + "rewards/margins": 1.3515225648880005, + "rewards/rejected": 0.8100277185440063, + "step": 9 + }, + { + "epoch": 0.01, + "learning_rate": 9.782608695652175e-06, + "logits/chosen": -1.4482485055923462, + "logits/rejected": -2.1116294860839844, + "logps/chosen": -451.5759582519531, + "logps/rejected": -67.73944854736328, + "loss": 0.355, + "rewards/accuracies": 0.8125, + "rewards/chosen": 6.18800163269043, + "rewards/margins": 5.184471130371094, + "rewards/rejected": 1.0035302639007568, + "step": 10 + }, + { + "epoch": 0.01, + "learning_rate": 1.0760869565217392e-05, + "logits/chosen": -1.2660062313079834, + "logits/rejected": -1.8649226427078247, + "logps/chosen": -265.8935546875, + "logps/rejected": -57.866336822509766, + "loss": 0.3178, + "rewards/accuracies": 0.8125, + "rewards/chosen": 5.78374719619751, + "rewards/margins": 4.856299877166748, + "rewards/rejected": 0.9274475574493408, + "step": 11 + }, + { + "epoch": 0.01, + "learning_rate": 1.173913043478261e-05, + "logits/chosen": -1.1804625988006592, + "logits/rejected": -1.7009960412979126, + "logps/chosen": -247.2069854736328, + "logps/rejected": -79.96739196777344, + "loss": 0.6617, + "rewards/accuracies": 0.75, + "rewards/chosen": 6.5535197257995605, + "rewards/margins": 4.647799015045166, + "rewards/rejected": 1.9057204723358154, + "step": 12 + }, + { + "epoch": 0.01, + "learning_rate": 1.2717391304347827e-05, + "logits/chosen": -0.9693112373352051, + "logits/rejected": -1.6497248411178589, + "logps/chosen": -372.8963623046875, + "logps/rejected": -70.47633361816406, + "loss": 0.2498, + "rewards/accuracies": 0.8125, + "rewards/chosen": 9.02272891998291, + "rewards/margins": 7.325143814086914, + "rewards/rejected": 1.6975862979888916, + "step": 13 + }, + { + "epoch": 0.02, + "learning_rate": 1.3695652173913046e-05, + "logits/chosen": -0.7723280787467957, + "logits/rejected": -1.4176554679870605, + "logps/chosen": -318.0392150878906, + "logps/rejected": -79.94412994384766, + "loss": 0.6401, + "rewards/accuracies": 0.6875, + "rewards/chosen": 6.698611259460449, + "rewards/margins": 4.517416477203369, + "rewards/rejected": 2.18119478225708, + "step": 14 + }, + { + "epoch": 0.02, + "learning_rate": 1.4673913043478261e-05, + "logits/chosen": -0.8084051609039307, + "logits/rejected": -1.4428132772445679, + "logps/chosen": -363.15869140625, + "logps/rejected": -70.70079803466797, + "loss": 0.4963, + "rewards/accuracies": 0.75, + "rewards/chosen": 7.02784538269043, + "rewards/margins": 5.400259971618652, + "rewards/rejected": 1.6275854110717773, + "step": 15 + }, + { + "epoch": 0.02, + "learning_rate": 1.565217391304348e-05, + "logits/chosen": -0.8056433200836182, + "logits/rejected": -1.4629294872283936, + "logps/chosen": -328.90435791015625, + "logps/rejected": -77.36820983886719, + "loss": 0.544, + "rewards/accuracies": 0.75, + "rewards/chosen": 9.417024612426758, + "rewards/margins": 7.023505687713623, + "rewards/rejected": 2.393519163131714, + "step": 16 + }, + { + "epoch": 0.02, + "learning_rate": 1.6630434782608694e-05, + "logits/chosen": -1.0918309688568115, + "logits/rejected": -1.6613047122955322, + "logps/chosen": -277.37982177734375, + "logps/rejected": -56.75635528564453, + "loss": 0.8958, + "rewards/accuracies": 0.8125, + "rewards/chosen": 6.844330310821533, + "rewards/margins": 4.82295560836792, + "rewards/rejected": 2.0213749408721924, + "step": 17 + }, + { + "epoch": 0.02, + "learning_rate": 1.7608695652173915e-05, + "logits/chosen": -0.9810999631881714, + "logits/rejected": -1.975247859954834, + "logps/chosen": -465.6856689453125, + "logps/rejected": -56.440269470214844, + "loss": 0.2275, + "rewards/accuracies": 0.875, + "rewards/chosen": 7.767542362213135, + "rewards/margins": 6.572459697723389, + "rewards/rejected": 1.1950825452804565, + "step": 18 + }, + { + "epoch": 0.02, + "learning_rate": 1.8586956521739132e-05, + "logits/chosen": -1.2428462505340576, + "logits/rejected": -1.8064367771148682, + "logps/chosen": -322.7100830078125, + "logps/rejected": -108.71321105957031, + "loss": 0.3006, + "rewards/accuracies": 0.875, + "rewards/chosen": 6.459216594696045, + "rewards/margins": 3.9207711219787598, + "rewards/rejected": 2.5384464263916016, + "step": 19 + }, + { + "epoch": 0.02, + "learning_rate": 1.956521739130435e-05, + "logits/chosen": -1.7773098945617676, + "logits/rejected": -2.178387403488159, + "logps/chosen": -357.8656921386719, + "logps/rejected": -112.65576934814453, + "loss": 0.3837, + "rewards/accuracies": 0.9375, + "rewards/chosen": 4.684590816497803, + "rewards/margins": 4.16137170791626, + "rewards/rejected": 0.5232191681861877, + "step": 20 + }, + { + "epoch": 0.02, + "learning_rate": 2.0543478260869567e-05, + "logits/chosen": -1.6802027225494385, + "logits/rejected": -1.943274974822998, + "logps/chosen": -327.85693359375, + "logps/rejected": -87.53279113769531, + "loss": 0.4428, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.681103229522705, + "rewards/margins": 2.3518502712249756, + "rewards/rejected": 0.32925307750701904, + "step": 21 + }, + { + "epoch": 0.02, + "learning_rate": 2.1521739130434784e-05, + "logits/chosen": -1.5789223909378052, + "logits/rejected": -2.4593281745910645, + "logps/chosen": -590.5732421875, + "logps/rejected": -86.78819274902344, + "loss": 0.252, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.6503655910491943, + "rewards/margins": 3.621542453765869, + "rewards/rejected": 0.028822531923651695, + "step": 22 + }, + { + "epoch": 0.03, + "learning_rate": 2.25e-05, + "logits/chosen": -1.8142305612564087, + "logits/rejected": -2.3507237434387207, + "logps/chosen": -296.42266845703125, + "logps/rejected": -60.37131118774414, + "loss": 0.5602, + "rewards/accuracies": 0.75, + "rewards/chosen": 2.170171022415161, + "rewards/margins": 2.5353941917419434, + "rewards/rejected": -0.36522313952445984, + "step": 23 + }, + { + "epoch": 0.03, + "learning_rate": 2.347826086956522e-05, + "logits/chosen": -1.7050156593322754, + "logits/rejected": -2.158708333969116, + "logps/chosen": -523.41748046875, + "logps/rejected": -81.6954345703125, + "loss": 0.7359, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.3799068927764893, + "rewards/margins": 2.963341236114502, + "rewards/rejected": -0.583433985710144, + "step": 24 + }, + { + "epoch": 0.03, + "learning_rate": 2.4456521739130436e-05, + "logits/chosen": -1.704190731048584, + "logits/rejected": -2.215942859649658, + "logps/chosen": -457.6519775390625, + "logps/rejected": -91.778076171875, + "loss": 0.0865, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.57167911529541, + "rewards/margins": 4.6597795486450195, + "rewards/rejected": -0.08810039609670639, + "step": 25 + }, + { + "epoch": 0.03, + "learning_rate": 2.5434782608695653e-05, + "logits/chosen": -1.4195502996444702, + "logits/rejected": -2.109910488128662, + "logps/chosen": -445.05908203125, + "logps/rejected": -80.95398712158203, + "loss": 0.1265, + "rewards/accuracies": 0.9375, + "rewards/chosen": 5.362960338592529, + "rewards/margins": 5.412578582763672, + "rewards/rejected": -0.049618080258369446, + "step": 26 + }, + { + "epoch": 0.03, + "learning_rate": 2.6413043478260874e-05, + "logits/chosen": -1.0274971723556519, + "logits/rejected": -1.7681918144226074, + "logps/chosen": -393.85870361328125, + "logps/rejected": -77.79011535644531, + "loss": 0.1702, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.572484016418457, + "rewards/margins": 4.946743488311768, + "rewards/rejected": -0.374259889125824, + "step": 27 + }, + { + "epoch": 0.03, + "learning_rate": 2.739130434782609e-05, + "logits/chosen": -0.7365273237228394, + "logits/rejected": -1.9988534450531006, + "logps/chosen": -569.8936767578125, + "logps/rejected": -79.71562194824219, + "loss": 0.0465, + "rewards/accuracies": 1.0, + "rewards/chosen": 12.751686096191406, + "rewards/margins": 12.474647521972656, + "rewards/rejected": 0.2770393192768097, + "step": 28 + }, + { + "epoch": 0.03, + "learning_rate": 2.836956521739131e-05, + "logits/chosen": -0.6424872279167175, + "logits/rejected": -1.6069284677505493, + "logps/chosen": -368.68292236328125, + "logps/rejected": -103.9141845703125, + "loss": 0.0351, + "rewards/accuracies": 1.0, + "rewards/chosen": 10.144453048706055, + "rewards/margins": 10.381340980529785, + "rewards/rejected": -0.23688830435276031, + "step": 29 + }, + { + "epoch": 0.03, + "learning_rate": 2.9347826086956523e-05, + "logits/chosen": -0.6061868071556091, + "logits/rejected": -1.4256294965744019, + "logps/chosen": -360.4029846191406, + "logps/rejected": -130.27224731445312, + "loss": 0.0414, + "rewards/accuracies": 1.0, + "rewards/chosen": 12.649539947509766, + "rewards/margins": 12.36604118347168, + "rewards/rejected": 0.2834990620613098, + "step": 30 + }, + { + "epoch": 0.03, + "learning_rate": 3.032608695652174e-05, + "logits/chosen": -1.0075111389160156, + "logits/rejected": -1.3918204307556152, + "logps/chosen": -227.05450439453125, + "logps/rejected": -172.67947387695312, + "loss": 1.1145, + "rewards/accuracies": 0.875, + "rewards/chosen": 4.628115653991699, + "rewards/margins": 3.9521055221557617, + "rewards/rejected": 0.6760098934173584, + "step": 31 + }, + { + "epoch": 0.03, + "learning_rate": 3.130434782608696e-05, + "logits/chosen": -0.9060839414596558, + "logits/rejected": -2.0123891830444336, + "logps/chosen": -403.06781005859375, + "logps/rejected": -75.35684967041016, + "loss": 0.0319, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.352475643157959, + "rewards/margins": 8.778717041015625, + "rewards/rejected": -1.4262410402297974, + "step": 32 + }, + { + "epoch": 0.04, + "learning_rate": 3.228260869565217e-05, + "logits/chosen": -1.2984271049499512, + "logits/rejected": -1.9507720470428467, + "logps/chosen": -306.7958984375, + "logps/rejected": -107.01404571533203, + "loss": 0.0795, + "rewards/accuracies": 0.9375, + "rewards/chosen": 5.223363399505615, + "rewards/margins": 7.012631416320801, + "rewards/rejected": -1.7892680168151855, + "step": 33 + }, + { + "epoch": 0.04, + "learning_rate": 3.326086956521739e-05, + "logits/chosen": -1.1749062538146973, + "logits/rejected": -2.004805564880371, + "logps/chosen": -437.9205322265625, + "logps/rejected": -135.21376037597656, + "loss": 0.0021, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.3555908203125, + "rewards/margins": 12.285521507263184, + "rewards/rejected": -2.9299299716949463, + "step": 34 + }, + { + "epoch": 0.04, + "learning_rate": 3.423913043478261e-05, + "logits/chosen": -0.9383536577224731, + "logits/rejected": -2.0366413593292236, + "logps/chosen": -457.8018493652344, + "logps/rejected": -87.7723617553711, + "loss": 0.041, + "rewards/accuracies": 1.0, + "rewards/chosen": 6.836911678314209, + "rewards/margins": 9.43667984008789, + "rewards/rejected": -2.599769115447998, + "step": 35 + }, + { + "epoch": 0.04, + "learning_rate": 3.521739130434783e-05, + "logits/chosen": -1.120687484741211, + "logits/rejected": -1.9933563470840454, + "logps/chosen": -439.6186828613281, + "logps/rejected": -94.24763488769531, + "loss": 0.1906, + "rewards/accuracies": 0.9375, + "rewards/chosen": 6.565365314483643, + "rewards/margins": 9.163484573364258, + "rewards/rejected": -2.5981192588806152, + "step": 36 + }, + { + "epoch": 0.04, + "learning_rate": 3.619565217391305e-05, + "logits/chosen": -1.338189959526062, + "logits/rejected": -1.9118335247039795, + "logps/chosen": -286.96728515625, + "logps/rejected": -127.22909545898438, + "loss": 0.0279, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.9341468811035156, + "rewards/margins": 6.362301349639893, + "rewards/rejected": -3.4281537532806396, + "step": 37 + }, + { + "epoch": 0.04, + "learning_rate": 3.7173913043478264e-05, + "logits/chosen": -1.2371840476989746, + "logits/rejected": -1.7362310886383057, + "logps/chosen": -319.52911376953125, + "logps/rejected": -151.65084838867188, + "loss": 0.0939, + "rewards/accuracies": 0.9375, + "rewards/chosen": 6.216956615447998, + "rewards/margins": 9.073128700256348, + "rewards/rejected": -2.856172561645508, + "step": 38 + }, + { + "epoch": 0.04, + "learning_rate": 3.815217391304348e-05, + "logits/chosen": -0.8356418609619141, + "logits/rejected": -1.7252635955810547, + "logps/chosen": -391.63275146484375, + "logps/rejected": -154.64361572265625, + "loss": 0.2483, + "rewards/accuracies": 0.9375, + "rewards/chosen": 8.802470207214355, + "rewards/margins": 11.676048278808594, + "rewards/rejected": -2.873577356338501, + "step": 39 + }, + { + "epoch": 0.04, + "learning_rate": 3.91304347826087e-05, + "logits/chosen": -0.9311200380325317, + "logits/rejected": -1.8107151985168457, + "logps/chosen": -374.2525939941406, + "logps/rejected": -131.86715698242188, + "loss": 0.0058, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.586356163024902, + "rewards/margins": 11.15583324432373, + "rewards/rejected": -2.569479465484619, + "step": 40 + }, + { + "epoch": 0.04, + "learning_rate": 4.0108695652173916e-05, + "logits/chosen": -0.6861732006072998, + "logits/rejected": -1.6295411586761475, + "logps/chosen": -311.7944641113281, + "logps/rejected": -132.4564666748047, + "loss": 0.0124, + "rewards/accuracies": 1.0, + "rewards/chosen": 5.333560943603516, + "rewards/margins": 10.023443222045898, + "rewards/rejected": -4.689882755279541, + "step": 41 + }, + { + "epoch": 0.05, + "learning_rate": 4.1086956521739134e-05, + "logits/chosen": -1.1133558750152588, + "logits/rejected": -2.0819039344787598, + "logps/chosen": -282.22332763671875, + "logps/rejected": -98.7315673828125, + "loss": 0.0156, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.930261611938477, + "rewards/margins": 8.344033241271973, + "rewards/rejected": -3.413771629333496, + "step": 42 + }, + { + "epoch": 0.05, + "learning_rate": 4.206521739130435e-05, + "logits/chosen": -1.0311042070388794, + "logits/rejected": -2.0758776664733887, + "logps/chosen": -369.7279052734375, + "logps/rejected": -146.08364868164062, + "loss": 0.0127, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.7552289962768555, + "rewards/margins": 13.249262809753418, + "rewards/rejected": -5.494032859802246, + "step": 43 + }, + { + "epoch": 0.05, + "learning_rate": 4.304347826086957e-05, + "logits/chosen": -1.0191632509231567, + "logits/rejected": -1.995982050895691, + "logps/chosen": -367.692626953125, + "logps/rejected": -152.65135192871094, + "loss": 0.0004, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.9517822265625, + "rewards/margins": 13.869260787963867, + "rewards/rejected": -5.917478084564209, + "step": 44 + }, + { + "epoch": 0.05, + "learning_rate": 4.4021739130434786e-05, + "logits/chosen": -1.3430471420288086, + "logits/rejected": -2.0910849571228027, + "logps/chosen": -334.8968811035156, + "logps/rejected": -168.4519500732422, + "loss": 0.0057, + "rewards/accuracies": 1.0, + "rewards/chosen": 5.113959312438965, + "rewards/margins": 12.524876594543457, + "rewards/rejected": -7.41091775894165, + "step": 45 + }, + { + "epoch": 0.05, + "learning_rate": 4.5e-05, + "logits/chosen": -0.8516018986701965, + "logits/rejected": -1.9934579133987427, + "logps/chosen": -386.7164001464844, + "logps/rejected": -152.8460235595703, + "loss": 0.0253, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.0803022384643555, + "rewards/margins": 13.202106475830078, + "rewards/rejected": -6.121803283691406, + "step": 46 + }, + { + "epoch": 0.05, + "learning_rate": 4.494827586206897e-05, + "logits/chosen": -0.9460695385932922, + "logits/rejected": -1.9993284940719604, + "logps/chosen": -355.2873229980469, + "logps/rejected": -188.5989532470703, + "loss": 0.1008, + "rewards/accuracies": 0.9375, + "rewards/chosen": 6.138028144836426, + "rewards/margins": 13.93982982635498, + "rewards/rejected": -7.801800727844238, + "step": 47 + }, + { + "epoch": 0.05, + "learning_rate": 4.489655172413793e-05, + "logits/chosen": -1.212510108947754, + "logits/rejected": -2.0202791690826416, + "logps/chosen": -293.0318908691406, + "logps/rejected": -194.92649841308594, + "loss": 0.0058, + "rewards/accuracies": 1.0, + "rewards/chosen": 6.742659091949463, + "rewards/margins": 17.209505081176758, + "rewards/rejected": -10.466848373413086, + "step": 48 + }, + { + "epoch": 0.05, + "learning_rate": 4.48448275862069e-05, + "logits/chosen": -1.388157606124878, + "logits/rejected": -2.1885251998901367, + "logps/chosen": -330.6868591308594, + "logps/rejected": -214.35423278808594, + "loss": 0.0003, + "rewards/accuracies": 1.0, + "rewards/chosen": 5.653369903564453, + "rewards/margins": 18.257244110107422, + "rewards/rejected": -12.603873252868652, + "step": 49 + }, + { + "epoch": 0.05, + "learning_rate": 4.479310344827587e-05, + "logits/chosen": -1.020527958869934, + "logits/rejected": -1.9433815479278564, + "logps/chosen": -347.48394775390625, + "logps/rejected": -191.15147399902344, + "loss": 0.0001, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.30579137802124, + "rewards/margins": 18.734811782836914, + "rewards/rejected": -11.429021835327148, + "step": 50 + }, + { + "epoch": 0.06, + "learning_rate": 4.474137931034483e-05, + "logits/chosen": -1.2260757684707642, + "logits/rejected": -2.0841352939605713, + "logps/chosen": -243.5459747314453, + "logps/rejected": -188.00442504882812, + "loss": 0.0002, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.675260543823242, + "rewards/margins": 14.774760246276855, + "rewards/rejected": -11.099499702453613, + "step": 51 + }, + { + "epoch": 0.06, + "learning_rate": 4.46896551724138e-05, + "logits/chosen": -1.0378711223602295, + "logits/rejected": -2.0174639225006104, + "logps/chosen": -375.9520263671875, + "logps/rejected": -182.97463989257812, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.18237829208374, + "rewards/margins": 19.010208129882812, + "rewards/rejected": -11.82783317565918, + "step": 52 + }, + { + "epoch": 0.06, + "learning_rate": 4.4637931034482765e-05, + "logits/chosen": -1.1728301048278809, + "logits/rejected": -1.9667140245437622, + "logps/chosen": -309.5382995605469, + "logps/rejected": -241.13743591308594, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 6.681103706359863, + "rewards/margins": 21.542236328125, + "rewards/rejected": -14.861133575439453, + "step": 53 + }, + { + "epoch": 0.06, + "learning_rate": 4.4586206896551726e-05, + "logits/chosen": -1.1491725444793701, + "logits/rejected": -1.760243535041809, + "logps/chosen": -250.09036254882812, + "logps/rejected": -226.221435546875, + "loss": 0.0001, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.3173880577087402, + "rewards/margins": 16.417604446411133, + "rewards/rejected": -14.100215911865234, + "step": 54 + }, + { + "epoch": 0.06, + "learning_rate": 4.4534482758620694e-05, + "logits/chosen": -1.2637637853622437, + "logits/rejected": -1.8996949195861816, + "logps/chosen": -401.9977722167969, + "logps/rejected": -303.9198303222656, + "loss": 0.0011, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.595758438110352, + "rewards/margins": 26.598325729370117, + "rewards/rejected": -18.002567291259766, + "step": 55 + }, + { + "epoch": 0.06, + "learning_rate": 4.4482758620689656e-05, + "logits/chosen": -0.9855178594589233, + "logits/rejected": -2.0190346240997314, + "logps/chosen": -394.40625, + "logps/rejected": -253.97720336914062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 6.395060062408447, + "rewards/margins": 23.680639266967773, + "rewards/rejected": -17.28557777404785, + "step": 56 + }, + { + "epoch": 0.06, + "learning_rate": 4.4431034482758624e-05, + "logits/chosen": -1.049959421157837, + "logits/rejected": -1.6531357765197754, + "logps/chosen": -205.54058837890625, + "logps/rejected": -250.66958618164062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.874389171600342, + "rewards/margins": 20.083953857421875, + "rewards/rejected": -15.209564208984375, + "step": 57 + }, + { + "epoch": 0.06, + "learning_rate": 4.4379310344827585e-05, + "logits/chosen": -0.8982111811637878, + "logits/rejected": -1.6001371145248413, + "logps/chosen": -241.91790771484375, + "logps/rejected": -252.10299682617188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 6.044814109802246, + "rewards/margins": 21.77715301513672, + "rewards/rejected": -15.732340812683105, + "step": 58 + }, + { + "epoch": 0.06, + "learning_rate": 4.432758620689655e-05, + "logits/chosen": -0.753044068813324, + "logits/rejected": -1.6325318813323975, + "logps/chosen": -315.30230712890625, + "logps/rejected": -245.91976928710938, + "loss": 0.0078, + "rewards/accuracies": 1.0, + "rewards/chosen": 5.801758289337158, + "rewards/margins": 20.689924240112305, + "rewards/rejected": -14.888164520263672, + "step": 59 + }, + { + "epoch": 0.07, + "learning_rate": 4.427586206896552e-05, + "logits/chosen": -0.5443448424339294, + "logits/rejected": -1.7224345207214355, + "logps/chosen": -388.339599609375, + "logps/rejected": -211.5500946044922, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 11.5880708694458, + "rewards/margins": 24.889877319335938, + "rewards/rejected": -13.301810264587402, + "step": 60 + }, + { + "epoch": 0.07, + "learning_rate": 4.422413793103448e-05, + "logits/chosen": -0.6921290755271912, + "logits/rejected": -1.6139135360717773, + "logps/chosen": -242.47312927246094, + "logps/rejected": -214.73622131347656, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 11.329044342041016, + "rewards/margins": 21.45581817626953, + "rewards/rejected": -10.1267728805542, + "step": 61 + }, + { + "epoch": 0.07, + "learning_rate": 4.417241379310345e-05, + "logits/chosen": -0.4828271269798279, + "logits/rejected": -1.649349570274353, + "logps/chosen": -306.6004638671875, + "logps/rejected": -172.2523956298828, + "loss": 0.0003, + "rewards/accuracies": 1.0, + "rewards/chosen": 11.373737335205078, + "rewards/margins": 20.521434783935547, + "rewards/rejected": -9.147698402404785, + "step": 62 + }, + { + "epoch": 0.07, + "learning_rate": 4.412068965517242e-05, + "logits/chosen": -0.4676312506198883, + "logits/rejected": -1.6393827199935913, + "logps/chosen": -357.127685546875, + "logps/rejected": -199.84524536132812, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 16.730457305908203, + "rewards/margins": 26.409488677978516, + "rewards/rejected": -9.679031372070312, + "step": 63 + }, + { + "epoch": 0.07, + "learning_rate": 4.406896551724138e-05, + "logits/chosen": -0.7774850726127625, + "logits/rejected": -1.324657678604126, + "logps/chosen": -193.75833129882812, + "logps/rejected": -194.70233154296875, + "loss": 0.0015, + "rewards/accuracies": 1.0, + "rewards/chosen": 5.371121406555176, + "rewards/margins": 15.129293441772461, + "rewards/rejected": -9.758172988891602, + "step": 64 + }, + { + "epoch": 0.07, + "learning_rate": 4.401724137931035e-05, + "logits/chosen": -0.5502160787582397, + "logits/rejected": -1.466262936592102, + "logps/chosen": -240.96836853027344, + "logps/rejected": -174.6772003173828, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.808657169342041, + "rewards/margins": 17.520769119262695, + "rewards/rejected": -9.712109565734863, + "step": 65 + }, + { + "epoch": 0.07, + "learning_rate": 4.3965517241379315e-05, + "logits/chosen": -0.5615445375442505, + "logits/rejected": -1.3243337869644165, + "logps/chosen": -282.65484619140625, + "logps/rejected": -211.33326721191406, + "loss": 0.4403, + "rewards/accuracies": 0.9375, + "rewards/chosen": 9.213937759399414, + "rewards/margins": 18.93706512451172, + "rewards/rejected": -9.723125457763672, + "step": 66 + }, + { + "epoch": 0.07, + "learning_rate": 4.3913793103448277e-05, + "logits/chosen": -0.5882927775382996, + "logits/rejected": -1.4038889408111572, + "logps/chosen": -244.26333618164062, + "logps/rejected": -192.05517578125, + "loss": 0.0007, + "rewards/accuracies": 1.0, + "rewards/chosen": 5.999594688415527, + "rewards/margins": 16.232702255249023, + "rewards/rejected": -10.233107566833496, + "step": 67 + }, + { + "epoch": 0.07, + "learning_rate": 4.3862068965517245e-05, + "logits/chosen": -0.7710579633712769, + "logits/rejected": -1.5811477899551392, + "logps/chosen": -274.4286804199219, + "logps/rejected": -189.31427001953125, + "loss": 0.0003, + "rewards/accuracies": 1.0, + "rewards/chosen": 10.483807563781738, + "rewards/margins": 22.807409286499023, + "rewards/rejected": -12.323600769042969, + "step": 68 + }, + { + "epoch": 0.08, + "learning_rate": 4.381034482758621e-05, + "logits/chosen": -0.2001597136259079, + "logits/rejected": -1.5477687120437622, + "logps/chosen": -398.04296875, + "logps/rejected": -208.8218536376953, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 11.122913360595703, + "rewards/margins": 23.094343185424805, + "rewards/rejected": -11.971429824829102, + "step": 69 + }, + { + "epoch": 0.08, + "learning_rate": 4.3758620689655174e-05, + "logits/chosen": -0.5891858339309692, + "logits/rejected": -1.500455379486084, + "logps/chosen": -223.07235717773438, + "logps/rejected": -219.37405395507812, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.498288154602051, + "rewards/margins": 19.65944480895996, + "rewards/rejected": -12.16115665435791, + "step": 70 + }, + { + "epoch": 0.08, + "learning_rate": 4.370689655172414e-05, + "logits/chosen": -0.5222135186195374, + "logits/rejected": -1.459835410118103, + "logps/chosen": -264.3045349121094, + "logps/rejected": -184.4599609375, + "loss": 0.0006, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.261447429656982, + "rewards/margins": 17.37218475341797, + "rewards/rejected": -10.110737800598145, + "step": 71 + }, + { + "epoch": 0.08, + "learning_rate": 4.365517241379311e-05, + "logits/chosen": -0.4974746108055115, + "logits/rejected": -1.5458664894104004, + "logps/chosen": -272.7602233886719, + "logps/rejected": -181.00247192382812, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 10.925564765930176, + "rewards/margins": 22.81020736694336, + "rewards/rejected": -11.88464069366455, + "step": 72 + }, + { + "epoch": 0.08, + "learning_rate": 4.360344827586207e-05, + "logits/chosen": -0.3357059359550476, + "logits/rejected": -1.4441150426864624, + "logps/chosen": -414.8647766113281, + "logps/rejected": -223.109375, + "loss": 0.0004, + "rewards/accuracies": 1.0, + "rewards/chosen": 13.181656837463379, + "rewards/margins": 27.527368545532227, + "rewards/rejected": -14.345712661743164, + "step": 73 + }, + { + "epoch": 0.08, + "learning_rate": 4.355172413793104e-05, + "logits/chosen": -0.6531690359115601, + "logits/rejected": -1.4611525535583496, + "logps/chosen": -221.1251220703125, + "logps/rejected": -226.30352783203125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.807477951049805, + "rewards/margins": 21.502174377441406, + "rewards/rejected": -13.694696426391602, + "step": 74 + }, + { + "epoch": 0.08, + "learning_rate": 4.35e-05, + "logits/chosen": -0.5135932564735413, + "logits/rejected": -1.4995931386947632, + "logps/chosen": -332.56842041015625, + "logps/rejected": -270.9891662597656, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 12.301117897033691, + "rewards/margins": 26.830467224121094, + "rewards/rejected": -14.529346466064453, + "step": 75 + }, + { + "epoch": 0.08, + "learning_rate": 4.344827586206897e-05, + "logits/chosen": -0.3770020604133606, + "logits/rejected": -1.44219970703125, + "logps/chosen": -317.96240234375, + "logps/rejected": -250.3682098388672, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.8860931396484375, + "rewards/margins": 23.53211784362793, + "rewards/rejected": -15.646024703979492, + "step": 76 + }, + { + "epoch": 0.08, + "learning_rate": 4.339655172413793e-05, + "logits/chosen": -0.49628371000289917, + "logits/rejected": -1.4045476913452148, + "logps/chosen": -367.6162414550781, + "logps/rejected": -265.2467346191406, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 10.148900985717773, + "rewards/margins": 24.313669204711914, + "rewards/rejected": -14.164766311645508, + "step": 77 + }, + { + "epoch": 0.09, + "learning_rate": 4.33448275862069e-05, + "logits/chosen": -0.5287263989448547, + "logits/rejected": -1.3665097951889038, + "logps/chosen": -341.87664794921875, + "logps/rejected": -249.6795654296875, + "loss": 0.1517, + "rewards/accuracies": 0.9375, + "rewards/chosen": 8.620326042175293, + "rewards/margins": 19.850658416748047, + "rewards/rejected": -11.230331420898438, + "step": 78 + }, + { + "epoch": 0.09, + "learning_rate": 4.3293103448275865e-05, + "logits/chosen": -0.2743966281414032, + "logits/rejected": -1.4255130290985107, + "logps/chosen": -334.3340759277344, + "logps/rejected": -265.15032958984375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 10.872364044189453, + "rewards/margins": 27.00977325439453, + "rewards/rejected": -16.13741111755371, + "step": 79 + }, + { + "epoch": 0.09, + "learning_rate": 4.324137931034483e-05, + "logits/chosen": -0.5563563108444214, + "logits/rejected": -1.6934064626693726, + "logps/chosen": -408.9612731933594, + "logps/rejected": -220.00091552734375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 10.237764358520508, + "rewards/margins": 24.824390411376953, + "rewards/rejected": -14.586627960205078, + "step": 80 + }, + { + "epoch": 0.09, + "learning_rate": 4.3189655172413795e-05, + "logits/chosen": -0.617451548576355, + "logits/rejected": -1.640676736831665, + "logps/chosen": -363.1540222167969, + "logps/rejected": -226.42376708984375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 10.672024726867676, + "rewards/margins": 24.347591400146484, + "rewards/rejected": -13.675565719604492, + "step": 81 + }, + { + "epoch": 0.09, + "learning_rate": 4.313793103448276e-05, + "logits/chosen": -0.45464372634887695, + "logits/rejected": -1.4708175659179688, + "logps/chosen": -328.35675048828125, + "logps/rejected": -223.79766845703125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 6.965122699737549, + "rewards/margins": 21.22077178955078, + "rewards/rejected": -14.25564956665039, + "step": 82 + }, + { + "epoch": 0.09, + "learning_rate": 4.3086206896551724e-05, + "logits/chosen": -0.6079340577125549, + "logits/rejected": -1.5333082675933838, + "logps/chosen": -219.4099578857422, + "logps/rejected": -230.07948303222656, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 6.278050899505615, + "rewards/margins": 20.45656967163086, + "rewards/rejected": -14.178520202636719, + "step": 83 + }, + { + "epoch": 0.09, + "learning_rate": 4.303448275862069e-05, + "logits/chosen": -0.4874611794948578, + "logits/rejected": -1.5135740041732788, + "logps/chosen": -304.00274658203125, + "logps/rejected": -263.0120849609375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.557435989379883, + "rewards/margins": 24.41012954711914, + "rewards/rejected": -14.852693557739258, + "step": 84 + }, + { + "epoch": 0.09, + "learning_rate": 4.298275862068966e-05, + "logits/chosen": -0.4802761375904083, + "logits/rejected": -1.4294252395629883, + "logps/chosen": -293.41192626953125, + "logps/rejected": -283.2821350097656, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.370968818664551, + "rewards/margins": 24.32880401611328, + "rewards/rejected": -16.957834243774414, + "step": 85 + }, + { + "epoch": 0.09, + "learning_rate": 4.293103448275862e-05, + "logits/chosen": -0.3580012321472168, + "logits/rejected": -1.499201774597168, + "logps/chosen": -350.5351257324219, + "logps/rejected": -244.6523895263672, + "loss": 0.0001, + "rewards/accuracies": 1.0, + "rewards/chosen": 12.237674713134766, + "rewards/margins": 26.58084487915039, + "rewards/rejected": -14.343170166015625, + "step": 86 + }, + { + "epoch": 0.09, + "learning_rate": 4.287931034482759e-05, + "logits/chosen": -0.6917211413383484, + "logits/rejected": -1.4503819942474365, + "logps/chosen": -217.5175323486328, + "logps/rejected": -228.9481201171875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 6.635045051574707, + "rewards/margins": 20.42833709716797, + "rewards/rejected": -13.793292999267578, + "step": 87 + }, + { + "epoch": 0.1, + "learning_rate": 4.282758620689656e-05, + "logits/chosen": -0.8255922794342041, + "logits/rejected": -1.4748644828796387, + "logps/chosen": -217.336669921875, + "logps/rejected": -206.8959197998047, + "loss": 0.0005, + "rewards/accuracies": 1.0, + "rewards/chosen": 5.22888708114624, + "rewards/margins": 19.673564910888672, + "rewards/rejected": -14.444679260253906, + "step": 88 + }, + { + "epoch": 0.1, + "learning_rate": 4.2775862068965525e-05, + "logits/chosen": -0.626908004283905, + "logits/rejected": -1.6663663387298584, + "logps/chosen": -328.56231689453125, + "logps/rejected": -235.42843627929688, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 6.891221046447754, + "rewards/margins": 21.77893829345703, + "rewards/rejected": -14.887716293334961, + "step": 89 + }, + { + "epoch": 0.1, + "learning_rate": 4.2724137931034486e-05, + "logits/chosen": -0.5689429640769958, + "logits/rejected": -1.618609070777893, + "logps/chosen": -298.08770751953125, + "logps/rejected": -237.06094360351562, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.328791618347168, + "rewards/margins": 24.572046279907227, + "rewards/rejected": -15.243253707885742, + "step": 90 + }, + { + "epoch": 0.1, + "learning_rate": 4.2672413793103454e-05, + "logits/chosen": -0.5474239587783813, + "logits/rejected": -1.6142076253890991, + "logps/chosen": -295.00494384765625, + "logps/rejected": -268.9505615234375, + "loss": 0.0287, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.715471267700195, + "rewards/margins": 23.968780517578125, + "rewards/rejected": -14.25330924987793, + "step": 91 + }, + { + "epoch": 0.1, + "learning_rate": 4.2620689655172416e-05, + "logits/chosen": -0.5132906436920166, + "logits/rejected": -1.6248977184295654, + "logps/chosen": -314.5284423828125, + "logps/rejected": -210.1342315673828, + "loss": 0.0595, + "rewards/accuracies": 0.9375, + "rewards/chosen": 7.707674980163574, + "rewards/margins": 20.18167495727539, + "rewards/rejected": -12.474000930786133, + "step": 92 + }, + { + "epoch": 0.1, + "learning_rate": 4.2568965517241384e-05, + "logits/chosen": -0.45244526863098145, + "logits/rejected": -1.7539148330688477, + "logps/chosen": -394.9619140625, + "logps/rejected": -230.32550048828125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 11.32716178894043, + "rewards/margins": 26.326129913330078, + "rewards/rejected": -14.998967170715332, + "step": 93 + }, + { + "epoch": 0.1, + "learning_rate": 4.2517241379310345e-05, + "logits/chosen": -0.7538139224052429, + "logits/rejected": -1.5796599388122559, + "logps/chosen": -286.95281982421875, + "logps/rejected": -266.7412414550781, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.003694534301758, + "rewards/margins": 25.05299949645996, + "rewards/rejected": -17.04930305480957, + "step": 94 + }, + { + "epoch": 0.1, + "learning_rate": 4.246551724137931e-05, + "logits/chosen": -0.44376400113105774, + "logits/rejected": -1.5493687391281128, + "logps/chosen": -406.93951416015625, + "logps/rejected": -302.9401550292969, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.246862888336182, + "rewards/margins": 27.937149047851562, + "rewards/rejected": -20.69028663635254, + "step": 95 + }, + { + "epoch": 0.1, + "learning_rate": 4.2413793103448274e-05, + "logits/chosen": -0.7577255964279175, + "logits/rejected": -1.77558171749115, + "logps/chosen": -357.90838623046875, + "logps/rejected": -246.73458862304688, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.267894744873047, + "rewards/margins": 24.79804229736328, + "rewards/rejected": -17.530147552490234, + "step": 96 + }, + { + "epoch": 0.11, + "learning_rate": 4.236206896551724e-05, + "logits/chosen": -0.8565713763237, + "logits/rejected": -1.7760589122772217, + "logps/chosen": -338.751220703125, + "logps/rejected": -304.101806640625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.238365173339844, + "rewards/margins": 27.781978607177734, + "rewards/rejected": -18.54361343383789, + "step": 97 + }, + { + "epoch": 0.11, + "learning_rate": 4.231034482758621e-05, + "logits/chosen": -0.40401750802993774, + "logits/rejected": -1.7701493501663208, + "logps/chosen": -537.5440063476562, + "logps/rejected": -349.8548889160156, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 14.211512565612793, + "rewards/margins": 37.746315002441406, + "rewards/rejected": -23.53480339050293, + "step": 98 + }, + { + "epoch": 0.11, + "learning_rate": 4.225862068965517e-05, + "logits/chosen": -0.9799883365631104, + "logits/rejected": -1.8936114311218262, + "logps/chosen": -403.7567138671875, + "logps/rejected": -257.6401672363281, + "loss": 0.0001, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.710639476776123, + "rewards/margins": 23.923961639404297, + "rewards/rejected": -19.213321685791016, + "step": 99 + }, + { + "epoch": 0.11, + "learning_rate": 4.220689655172414e-05, + "logits/chosen": -0.94623202085495, + "logits/rejected": -1.7190004587173462, + "logps/chosen": -360.8213806152344, + "logps/rejected": -310.1994323730469, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.991710662841797, + "rewards/margins": 26.59444808959961, + "rewards/rejected": -22.60273551940918, + "step": 100 + }, + { + "epoch": 0.11, + "learning_rate": 4.215517241379311e-05, + "logits/chosen": -0.6102348566055298, + "logits/rejected": -1.8472094535827637, + "logps/chosen": -460.48699951171875, + "logps/rejected": -369.1959228515625, + "loss": 0.0002, + "rewards/accuracies": 1.0, + "rewards/chosen": 6.696294784545898, + "rewards/margins": 32.79343795776367, + "rewards/rejected": -26.097143173217773, + "step": 101 + }, + { + "epoch": 0.11, + "learning_rate": 4.210344827586207e-05, + "logits/chosen": -0.9649127721786499, + "logits/rejected": -1.8413913249969482, + "logps/chosen": -455.1119079589844, + "logps/rejected": -365.0443420410156, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.14402437210083, + "rewards/margins": 32.23406219482422, + "rewards/rejected": -25.090038299560547, + "step": 102 + }, + { + "epoch": 0.11, + "learning_rate": 4.2051724137931036e-05, + "logits/chosen": -1.176360011100769, + "logits/rejected": -1.6914632320404053, + "logps/chosen": -283.3887023925781, + "logps/rejected": -343.1808166503906, + "loss": 0.0008, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.03441965579986572, + "rewards/margins": 24.389293670654297, + "rewards/rejected": -24.423709869384766, + "step": 103 + }, + { + "epoch": 0.11, + "learning_rate": 4.2000000000000004e-05, + "logits/chosen": -0.7725791335105896, + "logits/rejected": -1.943673014640808, + "logps/chosen": -484.6883544921875, + "logps/rejected": -338.5666198730469, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.784733295440674, + "rewards/margins": 31.62430763244629, + "rewards/rejected": -23.839576721191406, + "step": 104 + }, + { + "epoch": 0.11, + "learning_rate": 4.194827586206897e-05, + "logits/chosen": -0.6000739336013794, + "logits/rejected": -1.681613802909851, + "logps/chosen": -342.3570556640625, + "logps/rejected": -256.06829833984375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.42122745513916, + "rewards/margins": 25.636981964111328, + "rewards/rejected": -18.21575355529785, + "step": 105 + }, + { + "epoch": 0.12, + "learning_rate": 4.1896551724137934e-05, + "logits/chosen": -0.5727726221084595, + "logits/rejected": -1.5278774499893188, + "logps/chosen": -340.2919921875, + "logps/rejected": -246.67628479003906, + "loss": 0.0183, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.279099464416504, + "rewards/margins": 23.8295841217041, + "rewards/rejected": -16.550485610961914, + "step": 106 + }, + { + "epoch": 0.12, + "learning_rate": 4.18448275862069e-05, + "logits/chosen": -0.588262677192688, + "logits/rejected": -1.4113011360168457, + "logps/chosen": -239.1262664794922, + "logps/rejected": -323.8817443847656, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.26235294342041, + "rewards/margins": 25.60938262939453, + "rewards/rejected": -21.34703254699707, + "step": 107 + }, + { + "epoch": 0.12, + "learning_rate": 4.179310344827587e-05, + "logits/chosen": -0.825312077999115, + "logits/rejected": -1.6536335945129395, + "logps/chosen": -245.26333618164062, + "logps/rejected": -275.9490966796875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.488056182861328, + "rewards/margins": 24.79592514038086, + "rewards/rejected": -20.30786895751953, + "step": 108 + }, + { + "epoch": 0.12, + "learning_rate": 4.174137931034483e-05, + "logits/chosen": -0.6509658694267273, + "logits/rejected": -1.7768073081970215, + "logps/chosen": -494.1702575683594, + "logps/rejected": -269.7856140136719, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 11.041641235351562, + "rewards/margins": 30.055639266967773, + "rewards/rejected": -19.013999938964844, + "step": 109 + }, + { + "epoch": 0.12, + "learning_rate": 4.16896551724138e-05, + "logits/chosen": -0.813829243183136, + "logits/rejected": -1.7704083919525146, + "logps/chosen": -370.6927795410156, + "logps/rejected": -288.31463623046875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 10.1367826461792, + "rewards/margins": 30.452945709228516, + "rewards/rejected": -20.316162109375, + "step": 110 + }, + { + "epoch": 0.12, + "learning_rate": 4.163793103448276e-05, + "logits/chosen": -0.450665146112442, + "logits/rejected": -1.4787647724151611, + "logps/chosen": -336.4142761230469, + "logps/rejected": -275.8489990234375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 6.380362510681152, + "rewards/margins": 27.303016662597656, + "rewards/rejected": -20.922651290893555, + "step": 111 + }, + { + "epoch": 0.12, + "learning_rate": 4.158620689655173e-05, + "logits/chosen": -0.7610673904418945, + "logits/rejected": -1.5079166889190674, + "logps/chosen": -290.8649597167969, + "logps/rejected": -391.47772216796875, + "loss": 0.4671, + "rewards/accuracies": 0.9375, + "rewards/chosen": 4.447336196899414, + "rewards/margins": 27.807403564453125, + "rewards/rejected": -23.36006736755371, + "step": 112 + }, + { + "epoch": 0.12, + "learning_rate": 4.153448275862069e-05, + "logits/chosen": -0.9873843193054199, + "logits/rejected": -1.6280449628829956, + "logps/chosen": -244.6575164794922, + "logps/rejected": -324.1699523925781, + "loss": 0.0036, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.1840460300445557, + "rewards/margins": 25.491558074951172, + "rewards/rejected": -23.307514190673828, + "step": 113 + }, + { + "epoch": 0.12, + "learning_rate": 4.148275862068966e-05, + "logits/chosen": -1.0418487787246704, + "logits/rejected": -1.7689330577850342, + "logps/chosen": -309.9642028808594, + "logps/rejected": -433.8427734375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.8272902965545654, + "rewards/margins": 33.52722930908203, + "rewards/rejected": -29.69993782043457, + "step": 114 + }, + { + "epoch": 0.13, + "learning_rate": 4.143103448275862e-05, + "logits/chosen": -0.9574589133262634, + "logits/rejected": -1.9435979127883911, + "logps/chosen": -485.497314453125, + "logps/rejected": -301.46038818359375, + "loss": 0.0002, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.4343650341033936, + "rewards/margins": 27.89614486694336, + "rewards/rejected": -24.46177864074707, + "step": 115 + }, + { + "epoch": 0.13, + "learning_rate": 4.1379310344827587e-05, + "logits/chosen": -1.2192739248275757, + "logits/rejected": -1.9555035829544067, + "logps/chosen": -485.7822265625, + "logps/rejected": -452.36566162109375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.4936442375183105, + "rewards/margins": 31.84035873413086, + "rewards/rejected": -35.33400344848633, + "step": 116 + }, + { + "epoch": 0.13, + "learning_rate": 4.1327586206896555e-05, + "logits/chosen": -1.3901960849761963, + "logits/rejected": -2.194340944290161, + "logps/chosen": -556.4442749023438, + "logps/rejected": -403.41094970703125, + "loss": 0.4842, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.604147434234619, + "rewards/margins": 25.96053695678711, + "rewards/rejected": -32.5646858215332, + "step": 117 + }, + { + "epoch": 0.13, + "learning_rate": 4.1275862068965516e-05, + "logits/chosen": -1.10233473777771, + "logits/rejected": -1.911782145500183, + "logps/chosen": -319.7232666015625, + "logps/rejected": -366.8639221191406, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5800843238830566, + "rewards/margins": 30.92066192626953, + "rewards/rejected": -30.3405818939209, + "step": 118 + }, + { + "epoch": 0.13, + "learning_rate": 4.1224137931034484e-05, + "logits/chosen": -0.7030006051063538, + "logits/rejected": -1.7387290000915527, + "logps/chosen": -291.53961181640625, + "logps/rejected": -350.3901062011719, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.4014167785644531, + "rewards/margins": 28.830717086791992, + "rewards/rejected": -27.429298400878906, + "step": 119 + }, + { + "epoch": 0.13, + "learning_rate": 4.117241379310345e-05, + "logits/chosen": -0.5296112895011902, + "logits/rejected": -1.5072706937789917, + "logps/chosen": -286.94879150390625, + "logps/rejected": -346.3030700683594, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.8979191780090332, + "rewards/margins": 28.363784790039062, + "rewards/rejected": -26.465864181518555, + "step": 120 + }, + { + "epoch": 0.13, + "learning_rate": 4.112068965517242e-05, + "logits/chosen": -0.42470210790634155, + "logits/rejected": -1.7659740447998047, + "logps/chosen": -477.8876953125, + "logps/rejected": -315.16717529296875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.757699966430664, + "rewards/margins": 34.151214599609375, + "rewards/rejected": -24.393516540527344, + "step": 121 + }, + { + "epoch": 0.13, + "learning_rate": 4.106896551724138e-05, + "logits/chosen": -0.37895911931991577, + "logits/rejected": -1.4872817993164062, + "logps/chosen": -360.4671936035156, + "logps/rejected": -311.3260803222656, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 10.024797439575195, + "rewards/margins": 30.34416961669922, + "rewards/rejected": -20.31937026977539, + "step": 122 + }, + { + "epoch": 0.13, + "learning_rate": 4.101724137931035e-05, + "logits/chosen": -0.4127517342567444, + "logits/rejected": -1.4179290533065796, + "logps/chosen": -396.7817077636719, + "logps/rejected": -257.0691833496094, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.115121841430664, + "rewards/margins": 25.882369995117188, + "rewards/rejected": -17.767250061035156, + "step": 123 + }, + { + "epoch": 0.14, + "learning_rate": 4.096551724137932e-05, + "logits/chosen": -0.4867606461048126, + "logits/rejected": -1.433240532875061, + "logps/chosen": -307.6831359863281, + "logps/rejected": -285.9851379394531, + "loss": 0.0006, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.156608581542969, + "rewards/margins": 24.28235626220703, + "rewards/rejected": -15.12574577331543, + "step": 124 + }, + { + "epoch": 0.14, + "learning_rate": 4.091379310344828e-05, + "logits/chosen": -0.5465874671936035, + "logits/rejected": -1.398258924484253, + "logps/chosen": -244.0363311767578, + "logps/rejected": -240.68551635742188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.43398904800415, + "rewards/margins": 23.1319522857666, + "rewards/rejected": -18.69796371459961, + "step": 125 + }, + { + "epoch": 0.14, + "learning_rate": 4.0862068965517246e-05, + "logits/chosen": -0.38893836736679077, + "logits/rejected": -1.427646279335022, + "logps/chosen": -351.76348876953125, + "logps/rejected": -289.0392150878906, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.574392795562744, + "rewards/margins": 25.838218688964844, + "rewards/rejected": -18.26382827758789, + "step": 126 + }, + { + "epoch": 0.14, + "learning_rate": 4.0810344827586214e-05, + "logits/chosen": -0.3710274398326874, + "logits/rejected": -1.5999269485473633, + "logps/chosen": -394.9821472167969, + "logps/rejected": -246.48887634277344, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.973166465759277, + "rewards/margins": 26.636192321777344, + "rewards/rejected": -17.66302490234375, + "step": 127 + }, + { + "epoch": 0.14, + "learning_rate": 4.0758620689655175e-05, + "logits/chosen": -0.673173725605011, + "logits/rejected": -1.2474948167800903, + "logps/chosen": -257.3236083984375, + "logps/rejected": -264.7505187988281, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.773941993713379, + "rewards/margins": 19.319332122802734, + "rewards/rejected": -16.545391082763672, + "step": 128 + }, + { + "epoch": 0.14, + "learning_rate": 4.0706896551724143e-05, + "logits/chosen": -0.560947060585022, + "logits/rejected": -1.374908447265625, + "logps/chosen": -301.8670959472656, + "logps/rejected": -232.6251983642578, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 10.369309425354004, + "rewards/margins": 26.155746459960938, + "rewards/rejected": -15.786436080932617, + "step": 129 + }, + { + "epoch": 0.14, + "learning_rate": 4.0655172413793105e-05, + "logits/chosen": -0.6484676003456116, + "logits/rejected": -1.3061511516571045, + "logps/chosen": -240.1058807373047, + "logps/rejected": -292.108154296875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.469289302825928, + "rewards/margins": 25.45884132385254, + "rewards/rejected": -20.989551544189453, + "step": 130 + }, + { + "epoch": 0.14, + "learning_rate": 4.060344827586207e-05, + "logits/chosen": -0.4261731505393982, + "logits/rejected": -1.5151227712631226, + "logps/chosen": -402.2912292480469, + "logps/rejected": -289.1283264160156, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 10.882993698120117, + "rewards/margins": 30.517051696777344, + "rewards/rejected": -19.63405990600586, + "step": 131 + }, + { + "epoch": 0.14, + "learning_rate": 4.0551724137931034e-05, + "logits/chosen": -0.3702995181083679, + "logits/rejected": -1.5198004245758057, + "logps/chosen": -377.6826477050781, + "logps/rejected": -287.36566162109375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 10.187080383300781, + "rewards/margins": 30.941383361816406, + "rewards/rejected": -20.754301071166992, + "step": 132 + }, + { + "epoch": 0.15, + "learning_rate": 4.05e-05, + "logits/chosen": -0.394854873418808, + "logits/rejected": -1.641805648803711, + "logps/chosen": -462.9255676269531, + "logps/rejected": -254.52276611328125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 10.504643440246582, + "rewards/margins": 27.564743041992188, + "rewards/rejected": -17.060100555419922, + "step": 133 + }, + { + "epoch": 0.15, + "learning_rate": 4.044827586206896e-05, + "logits/chosen": -0.3886590600013733, + "logits/rejected": -1.4597467184066772, + "logps/chosen": -378.5803527832031, + "logps/rejected": -263.2305908203125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 13.531683921813965, + "rewards/margins": 31.480838775634766, + "rewards/rejected": -17.949153900146484, + "step": 134 + }, + { + "epoch": 0.15, + "learning_rate": 4.039655172413793e-05, + "logits/chosen": -0.5102599859237671, + "logits/rejected": -1.250558614730835, + "logps/chosen": -295.6455078125, + "logps/rejected": -345.3118591308594, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.247247695922852, + "rewards/margins": 26.292499542236328, + "rewards/rejected": -19.04525375366211, + "step": 135 + }, + { + "epoch": 0.15, + "learning_rate": 4.03448275862069e-05, + "logits/chosen": -0.39377278089523315, + "logits/rejected": -1.5313684940338135, + "logps/chosen": -441.0325622558594, + "logps/rejected": -223.12750244140625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 12.754582405090332, + "rewards/margins": 28.21717071533203, + "rewards/rejected": -15.462587356567383, + "step": 136 + }, + { + "epoch": 0.15, + "learning_rate": 4.029310344827587e-05, + "logits/chosen": -0.45350462198257446, + "logits/rejected": -1.3594186305999756, + "logps/chosen": -313.7057189941406, + "logps/rejected": -304.0419616699219, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 6.3996429443359375, + "rewards/margins": 23.64727020263672, + "rewards/rejected": -17.24762725830078, + "step": 137 + }, + { + "epoch": 0.15, + "learning_rate": 4.024137931034483e-05, + "logits/chosen": -0.17602750658988953, + "logits/rejected": -1.2745662927627563, + "logps/chosen": -371.0513916015625, + "logps/rejected": -299.74481201171875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.569783687591553, + "rewards/margins": 26.724822998046875, + "rewards/rejected": -19.155040740966797, + "step": 138 + }, + { + "epoch": 0.15, + "learning_rate": 4.0189655172413796e-05, + "logits/chosen": -0.44520992040634155, + "logits/rejected": -1.4310128688812256, + "logps/chosen": -334.8761901855469, + "logps/rejected": -278.89349365234375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.9243433475494385, + "rewards/margins": 22.313913345336914, + "rewards/rejected": -18.389568328857422, + "step": 139 + }, + { + "epoch": 0.15, + "learning_rate": 4.0137931034482764e-05, + "logits/chosen": -0.3496861755847931, + "logits/rejected": -1.480123519897461, + "logps/chosen": -345.8548583984375, + "logps/rejected": -295.12042236328125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.938793182373047, + "rewards/margins": 28.614458084106445, + "rewards/rejected": -18.67566680908203, + "step": 140 + }, + { + "epoch": 0.15, + "learning_rate": 4.0086206896551726e-05, + "logits/chosen": -0.4202902913093567, + "logits/rejected": -1.5481184720993042, + "logps/chosen": -408.18121337890625, + "logps/rejected": -273.8868408203125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.119111061096191, + "rewards/margins": 29.09032440185547, + "rewards/rejected": -19.971214294433594, + "step": 141 + }, + { + "epoch": 0.15, + "learning_rate": 4.0034482758620694e-05, + "logits/chosen": -0.5120177268981934, + "logits/rejected": -1.4507927894592285, + "logps/chosen": -271.6726989746094, + "logps/rejected": -297.4678955078125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 5.688560962677002, + "rewards/margins": 26.492656707763672, + "rewards/rejected": -20.804094314575195, + "step": 142 + }, + { + "epoch": 0.16, + "learning_rate": 3.998275862068966e-05, + "logits/chosen": -0.41148996353149414, + "logits/rejected": -1.4985872507095337, + "logps/chosen": -343.8206481933594, + "logps/rejected": -241.0497589111328, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.314858436584473, + "rewards/margins": 26.18862533569336, + "rewards/rejected": -16.87376594543457, + "step": 143 + }, + { + "epoch": 0.16, + "learning_rate": 3.993103448275862e-05, + "logits/chosen": -0.1992441862821579, + "logits/rejected": -1.8458625078201294, + "logps/chosen": -504.0348205566406, + "logps/rejected": -318.8588562011719, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 12.68671989440918, + "rewards/margins": 34.4812126159668, + "rewards/rejected": -21.79449462890625, + "step": 144 + }, + { + "epoch": 0.16, + "learning_rate": 3.987931034482759e-05, + "logits/chosen": -0.5546131730079651, + "logits/rejected": -1.527896761894226, + "logps/chosen": -333.9448547363281, + "logps/rejected": -303.9145202636719, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 10.168367385864258, + "rewards/margins": 30.0524959564209, + "rewards/rejected": -19.88412857055664, + "step": 145 + }, + { + "epoch": 0.16, + "learning_rate": 3.982758620689656e-05, + "logits/chosen": -0.3886173367500305, + "logits/rejected": -1.621660828590393, + "logps/chosen": -369.8122253417969, + "logps/rejected": -272.4569396972656, + "loss": 0.0016, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.674747467041016, + "rewards/margins": 26.678119659423828, + "rewards/rejected": -18.003372192382812, + "step": 146 + }, + { + "epoch": 0.16, + "learning_rate": 3.977586206896552e-05, + "logits/chosen": -0.61821049451828, + "logits/rejected": -1.6792049407958984, + "logps/chosen": -345.2456359863281, + "logps/rejected": -307.6291809082031, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.595407485961914, + "rewards/margins": 29.619413375854492, + "rewards/rejected": -22.024009704589844, + "step": 147 + }, + { + "epoch": 0.16, + "learning_rate": 3.972413793103449e-05, + "logits/chosen": -0.23747998476028442, + "logits/rejected": -1.287044644355774, + "logps/chosen": -361.4759521484375, + "logps/rejected": -267.63592529296875, + "loss": 0.2105, + "rewards/accuracies": 0.9375, + "rewards/chosen": 7.393239974975586, + "rewards/margins": 23.982959747314453, + "rewards/rejected": -16.5897216796875, + "step": 148 + }, + { + "epoch": 0.16, + "learning_rate": 3.967241379310345e-05, + "logits/chosen": -0.3899969160556793, + "logits/rejected": -1.6496773958206177, + "logps/chosen": -331.9876403808594, + "logps/rejected": -278.04522705078125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.100687026977539, + "rewards/margins": 29.64002227783203, + "rewards/rejected": -21.539335250854492, + "step": 149 + }, + { + "epoch": 0.16, + "learning_rate": 3.962068965517242e-05, + "logits/chosen": -0.4621184766292572, + "logits/rejected": -1.3823509216308594, + "logps/chosen": -291.25494384765625, + "logps/rejected": -313.21734619140625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.036273002624512, + "rewards/margins": 25.113258361816406, + "rewards/rejected": -21.076988220214844, + "step": 150 + }, + { + "epoch": 0.16, + "learning_rate": 3.956896551724138e-05, + "logits/chosen": -0.3050623834133148, + "logits/rejected": -1.3223061561584473, + "logps/chosen": -388.243896484375, + "logps/rejected": -266.01983642578125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.008649826049805, + "rewards/margins": 24.918262481689453, + "rewards/rejected": -17.90961265563965, + "step": 151 + }, + { + "epoch": 0.17, + "learning_rate": 3.9517241379310346e-05, + "logits/chosen": -0.26263201236724854, + "logits/rejected": -1.262205958366394, + "logps/chosen": -300.5107421875, + "logps/rejected": -266.810302734375, + "loss": 0.0007, + "rewards/accuracies": 1.0, + "rewards/chosen": 5.9579243659973145, + "rewards/margins": 23.850536346435547, + "rewards/rejected": -17.892614364624023, + "step": 152 + }, + { + "epoch": 0.17, + "learning_rate": 3.9465517241379314e-05, + "logits/chosen": -0.34424301981925964, + "logits/rejected": -1.2062197923660278, + "logps/chosen": -273.7149353027344, + "logps/rejected": -266.54150390625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 5.983701705932617, + "rewards/margins": 26.13591957092285, + "rewards/rejected": -20.1522216796875, + "step": 153 + }, + { + "epoch": 0.17, + "learning_rate": 3.9413793103448276e-05, + "logits/chosen": -0.1809622049331665, + "logits/rejected": -1.4350645542144775, + "logps/chosen": -418.6148376464844, + "logps/rejected": -297.8706359863281, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.36530876159668, + "rewards/margins": 30.20969581604004, + "rewards/rejected": -21.84438705444336, + "step": 154 + }, + { + "epoch": 0.17, + "learning_rate": 3.9362068965517244e-05, + "logits/chosen": -0.3474291265010834, + "logits/rejected": -1.3261463642120361, + "logps/chosen": -392.76068115234375, + "logps/rejected": -282.137451171875, + "loss": 0.12, + "rewards/accuracies": 0.9375, + "rewards/chosen": 8.480328559875488, + "rewards/margins": 24.708953857421875, + "rewards/rejected": -16.228626251220703, + "step": 155 + }, + { + "epoch": 0.17, + "learning_rate": 3.931034482758621e-05, + "logits/chosen": -0.29881787300109863, + "logits/rejected": -1.59579598903656, + "logps/chosen": -491.68280029296875, + "logps/rejected": -329.0652160644531, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.562919616699219, + "rewards/margins": 31.969928741455078, + "rewards/rejected": -22.407007217407227, + "step": 156 + }, + { + "epoch": 0.17, + "learning_rate": 3.925862068965517e-05, + "logits/chosen": -0.24624793231487274, + "logits/rejected": -1.2760369777679443, + "logps/chosen": -309.42950439453125, + "logps/rejected": -337.400634765625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 5.126012802124023, + "rewards/margins": 27.284088134765625, + "rewards/rejected": -22.1580753326416, + "step": 157 + }, + { + "epoch": 0.17, + "learning_rate": 3.920689655172414e-05, + "logits/chosen": -0.24486477673053741, + "logits/rejected": -1.4145996570587158, + "logps/chosen": -435.2896423339844, + "logps/rejected": -347.84698486328125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.421405076980591, + "rewards/margins": 30.74762725830078, + "rewards/rejected": -27.32622528076172, + "step": 158 + }, + { + "epoch": 0.17, + "learning_rate": 3.915517241379311e-05, + "logits/chosen": -0.4706140458583832, + "logits/rejected": -1.4215468168258667, + "logps/chosen": -427.6326904296875, + "logps/rejected": -399.431640625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.1375885009765625, + "rewards/margins": 31.478370666503906, + "rewards/rejected": -27.340782165527344, + "step": 159 + }, + { + "epoch": 0.17, + "learning_rate": 3.910344827586207e-05, + "logits/chosen": -0.25800830125808716, + "logits/rejected": -1.604130744934082, + "logps/chosen": -430.5594482421875, + "logps/rejected": -431.40301513671875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.8110466003417969, + "rewards/margins": 36.45166015625, + "rewards/rejected": -34.64060974121094, + "step": 160 + }, + { + "epoch": 0.18, + "learning_rate": 3.905172413793104e-05, + "logits/chosen": -0.3043404221534729, + "logits/rejected": -1.5278654098510742, + "logps/chosen": -465.7344970703125, + "logps/rejected": -436.65899658203125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.1547170877456665, + "rewards/margins": 33.50754928588867, + "rewards/rejected": -34.66226577758789, + "step": 161 + }, + { + "epoch": 0.18, + "learning_rate": 3.9000000000000006e-05, + "logits/chosen": -0.22293318808078766, + "logits/rejected": -1.2031610012054443, + "logps/chosen": -428.4457092285156, + "logps/rejected": -419.1228332519531, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.0903587341308594, + "rewards/margins": 28.546253204345703, + "rewards/rejected": -31.636613845825195, + "step": 162 + }, + { + "epoch": 0.18, + "learning_rate": 3.894827586206897e-05, + "logits/chosen": -0.18892326951026917, + "logits/rejected": -1.481589913368225, + "logps/chosen": -513.7359008789062, + "logps/rejected": -456.95208740234375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.6842398643493652, + "rewards/margins": 39.1934928894043, + "rewards/rejected": -36.509254455566406, + "step": 163 + }, + { + "epoch": 0.18, + "learning_rate": 3.8896551724137935e-05, + "logits/chosen": -0.48873329162597656, + "logits/rejected": -1.4820678234100342, + "logps/chosen": -393.00079345703125, + "logps/rejected": -426.32696533203125, + "loss": 0.0194, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.972713470458984, + "rewards/margins": 28.33656120300293, + "rewards/rejected": -34.30927276611328, + "step": 164 + }, + { + "epoch": 0.18, + "learning_rate": 3.88448275862069e-05, + "logits/chosen": -0.4431387484073639, + "logits/rejected": -1.3382742404937744, + "logps/chosen": -392.2378845214844, + "logps/rejected": -503.30194091796875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.949856758117676, + "rewards/margins": 34.904537200927734, + "rewards/rejected": -40.854393005371094, + "step": 165 + }, + { + "epoch": 0.18, + "learning_rate": 3.8793103448275865e-05, + "logits/chosen": -0.2996464669704437, + "logits/rejected": -1.4847010374069214, + "logps/chosen": -483.34161376953125, + "logps/rejected": -548.205078125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.7714486122131348, + "rewards/margins": 47.965972900390625, + "rewards/rejected": -44.19452667236328, + "step": 166 + }, + { + "epoch": 0.18, + "learning_rate": 3.874137931034483e-05, + "logits/chosen": -0.3217467963695526, + "logits/rejected": -1.2237169742584229, + "logps/chosen": -396.9007568359375, + "logps/rejected": -509.6751708984375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.9992933869361877, + "rewards/margins": 39.428592681884766, + "rewards/rejected": -38.429298400878906, + "step": 167 + }, + { + "epoch": 0.18, + "learning_rate": 3.8689655172413794e-05, + "logits/chosen": -0.26858553290367126, + "logits/rejected": -1.3162765502929688, + "logps/chosen": -436.47088623046875, + "logps/rejected": -461.6593933105469, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.6044464111328125, + "rewards/margins": 38.4824104309082, + "rewards/rejected": -35.87796401977539, + "step": 168 + }, + { + "epoch": 0.18, + "learning_rate": 3.863793103448276e-05, + "logits/chosen": -0.343127965927124, + "logits/rejected": -1.390782117843628, + "logps/chosen": -392.77978515625, + "logps/rejected": -443.3890686035156, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.19043821096420288, + "rewards/margins": 34.40040969848633, + "rewards/rejected": -34.20996856689453, + "step": 169 + }, + { + "epoch": 0.19, + "learning_rate": 3.858620689655172e-05, + "logits/chosen": -0.43945077061653137, + "logits/rejected": -1.3458408117294312, + "logps/chosen": -386.0972595214844, + "logps/rejected": -481.8536071777344, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.2730538845062256, + "rewards/margins": 35.0412483215332, + "rewards/rejected": -33.76819610595703, + "step": 170 + }, + { + "epoch": 0.19, + "learning_rate": 3.853448275862069e-05, + "logits/chosen": -0.28766676783561707, + "logits/rejected": -1.314267873764038, + "logps/chosen": -423.24981689453125, + "logps/rejected": -416.8682861328125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.9727609157562256, + "rewards/margins": 34.98452377319336, + "rewards/rejected": -33.01176834106445, + "step": 171 + }, + { + "epoch": 0.19, + "learning_rate": 3.848275862068966e-05, + "logits/chosen": -0.3893183767795563, + "logits/rejected": -1.443713903427124, + "logps/chosen": -474.935546875, + "logps/rejected": -453.4057922363281, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.1096689701080322, + "rewards/margins": 38.80006790161133, + "rewards/rejected": -36.690399169921875, + "step": 172 + }, + { + "epoch": 0.19, + "learning_rate": 3.843103448275862e-05, + "logits/chosen": -0.34813588857650757, + "logits/rejected": -1.271755337715149, + "logps/chosen": -392.284423828125, + "logps/rejected": -496.442138671875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.8468620777130127, + "rewards/margins": 41.135921478271484, + "rewards/rejected": -38.289058685302734, + "step": 173 + }, + { + "epoch": 0.19, + "learning_rate": 3.837931034482759e-05, + "logits/chosen": -0.3464352786540985, + "logits/rejected": -1.223905324935913, + "logps/chosen": -372.1242370605469, + "logps/rejected": -446.0730285644531, + "loss": 0.0073, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.18368953466415405, + "rewards/margins": 33.218013763427734, + "rewards/rejected": -33.034324645996094, + "step": 174 + }, + { + "epoch": 0.19, + "learning_rate": 3.8327586206896556e-05, + "logits/chosen": -0.42792245745658875, + "logits/rejected": -1.189011812210083, + "logps/chosen": -324.2301940917969, + "logps/rejected": -503.5957946777344, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.7187633514404297, + "rewards/margins": 37.19697189331055, + "rewards/rejected": -38.91573715209961, + "step": 175 + }, + { + "epoch": 0.19, + "learning_rate": 3.827586206896552e-05, + "logits/chosen": -0.16668730974197388, + "logits/rejected": -1.5985019207000732, + "logps/chosen": -482.8082580566406, + "logps/rejected": -449.761474609375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.7022485733032227, + "rewards/margins": 42.01506805419922, + "rewards/rejected": -38.31282043457031, + "step": 176 + }, + { + "epoch": 0.19, + "learning_rate": 3.8224137931034485e-05, + "logits/chosen": -0.26949506998062134, + "logits/rejected": -1.2986763715744019, + "logps/chosen": -348.618896484375, + "logps/rejected": -442.8262023925781, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.1358022689819336, + "rewards/margins": 35.771461486816406, + "rewards/rejected": -33.63566207885742, + "step": 177 + }, + { + "epoch": 0.19, + "learning_rate": 3.8172413793103453e-05, + "logits/chosen": -0.3438015580177307, + "logits/rejected": -1.3001600503921509, + "logps/chosen": -333.9195251464844, + "logps/rejected": -457.9849853515625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.145051956176758, + "rewards/margins": 41.207923889160156, + "rewards/rejected": -39.06287384033203, + "step": 178 + }, + { + "epoch": 0.2, + "learning_rate": 3.8120689655172415e-05, + "logits/chosen": -0.10503964871168137, + "logits/rejected": -1.2384865283966064, + "logps/chosen": -451.48663330078125, + "logps/rejected": -479.8037414550781, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.525763988494873, + "rewards/margins": 38.3972282409668, + "rewards/rejected": -37.8714599609375, + "step": 179 + }, + { + "epoch": 0.2, + "learning_rate": 3.806896551724138e-05, + "logits/chosen": -0.30969691276550293, + "logits/rejected": -1.1278965473175049, + "logps/chosen": -298.7955322265625, + "logps/rejected": -499.42279052734375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.1820168495178223, + "rewards/margins": 37.28181457519531, + "rewards/rejected": -39.463829040527344, + "step": 180 + }, + { + "epoch": 0.2, + "learning_rate": 3.801724137931035e-05, + "logits/chosen": -0.3709387183189392, + "logits/rejected": -1.5942573547363281, + "logps/chosen": -422.3631286621094, + "logps/rejected": -511.52008056640625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.8516347408294678, + "rewards/margins": 41.39206314086914, + "rewards/rejected": -38.540428161621094, + "step": 181 + }, + { + "epoch": 0.2, + "learning_rate": 3.796551724137931e-05, + "logits/chosen": -0.2693447470664978, + "logits/rejected": -1.5490005016326904, + "logps/chosen": -480.881591796875, + "logps/rejected": -414.71917724609375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.7573628425598145, + "rewards/margins": 39.35446548461914, + "rewards/rejected": -36.59709930419922, + "step": 182 + }, + { + "epoch": 0.2, + "learning_rate": 3.791379310344828e-05, + "logits/chosen": -0.39544716477394104, + "logits/rejected": -1.4232298135757446, + "logps/chosen": -400.05267333984375, + "logps/rejected": -465.1136169433594, + "loss": 0.0018, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.5475125312805176, + "rewards/margins": 38.094913482666016, + "rewards/rejected": -34.547393798828125, + "step": 183 + }, + { + "epoch": 0.2, + "learning_rate": 3.786206896551725e-05, + "logits/chosen": -0.3474569320678711, + "logits/rejected": -1.4260185956954956, + "logps/chosen": -334.5423889160156, + "logps/rejected": -438.87408447265625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.5236696004867554, + "rewards/margins": 33.86663055419922, + "rewards/rejected": -35.390296936035156, + "step": 184 + }, + { + "epoch": 0.2, + "learning_rate": 3.781034482758621e-05, + "logits/chosen": -0.3836357593536377, + "logits/rejected": -1.554406762123108, + "logps/chosen": -372.57830810546875, + "logps/rejected": -439.4316101074219, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.840342402458191, + "rewards/margins": 34.46831512451172, + "rewards/rejected": -32.627967834472656, + "step": 185 + }, + { + "epoch": 0.2, + "learning_rate": 3.775862068965517e-05, + "logits/chosen": -0.18503640592098236, + "logits/rejected": -1.800915241241455, + "logps/chosen": -485.19354248046875, + "logps/rejected": -432.1221923828125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.203826427459717, + "rewards/margins": 39.722900390625, + "rewards/rejected": -35.51907730102539, + "step": 186 + }, + { + "epoch": 0.2, + "learning_rate": 3.770689655172414e-05, + "logits/chosen": -0.3300974667072296, + "logits/rejected": -1.5052634477615356, + "logps/chosen": -418.87017822265625, + "logps/rejected": -408.99554443359375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.9931721687316895, + "rewards/margins": 35.2595100402832, + "rewards/rejected": -32.266334533691406, + "step": 187 + }, + { + "epoch": 0.21, + "learning_rate": 3.7655172413793106e-05, + "logits/chosen": -0.27791130542755127, + "logits/rejected": -1.4637497663497925, + "logps/chosen": -370.642822265625, + "logps/rejected": -457.7041320800781, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.304169178009033, + "rewards/margins": 40.587310791015625, + "rewards/rejected": -36.283138275146484, + "step": 188 + }, + { + "epoch": 0.21, + "learning_rate": 3.760344827586207e-05, + "logits/chosen": -0.4701173007488251, + "logits/rejected": -1.403714895248413, + "logps/chosen": -333.7972717285156, + "logps/rejected": -436.09967041015625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.8245915174484253, + "rewards/margins": 36.37276077270508, + "rewards/rejected": -34.54816818237305, + "step": 189 + }, + { + "epoch": 0.21, + "learning_rate": 3.7551724137931035e-05, + "logits/chosen": -0.08819374442100525, + "logits/rejected": -1.395701289176941, + "logps/chosen": -401.96673583984375, + "logps/rejected": -440.0989685058594, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.565300941467285, + "rewards/margins": 33.236175537109375, + "rewards/rejected": -30.670875549316406, + "step": 190 + }, + { + "epoch": 0.21, + "learning_rate": 3.7500000000000003e-05, + "logits/chosen": -0.17811371386051178, + "logits/rejected": -1.501448154449463, + "logps/chosen": -447.5896301269531, + "logps/rejected": -375.3368835449219, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.7833425998687744, + "rewards/margins": 31.601383209228516, + "rewards/rejected": -28.818038940429688, + "step": 191 + }, + { + "epoch": 0.21, + "learning_rate": 3.7448275862068965e-05, + "logits/chosen": -0.44955122470855713, + "logits/rejected": -1.550938606262207, + "logps/chosen": -289.89959716796875, + "logps/rejected": -400.880859375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.4812740385532379, + "rewards/margins": 33.061256408691406, + "rewards/rejected": -33.54253005981445, + "step": 192 + }, + { + "epoch": 0.21, + "learning_rate": 3.739655172413793e-05, + "logits/chosen": -0.18104656040668488, + "logits/rejected": -1.4971003532409668, + "logps/chosen": -418.8290710449219, + "logps/rejected": -424.74859619140625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 6.926714897155762, + "rewards/margins": 39.01382064819336, + "rewards/rejected": -32.08710861206055, + "step": 193 + }, + { + "epoch": 0.21, + "learning_rate": 3.73448275862069e-05, + "logits/chosen": -0.16005319356918335, + "logits/rejected": -1.4771628379821777, + "logps/chosen": -437.9535827636719, + "logps/rejected": -435.7216796875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 5.183934688568115, + "rewards/margins": 40.51701354980469, + "rewards/rejected": -35.33308410644531, + "step": 194 + }, + { + "epoch": 0.21, + "learning_rate": 3.729310344827586e-05, + "logits/chosen": -0.35886967182159424, + "logits/rejected": -1.6980268955230713, + "logps/chosen": -404.4248962402344, + "logps/rejected": -412.4001159667969, + "loss": 0.0068, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.873406887054443, + "rewards/margins": 38.6983757019043, + "rewards/rejected": -30.824968338012695, + "step": 195 + }, + { + "epoch": 0.21, + "learning_rate": 3.724137931034483e-05, + "logits/chosen": -0.4241984486579895, + "logits/rejected": -1.4724136590957642, + "logps/chosen": -353.877197265625, + "logps/rejected": -414.2342529296875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.698596954345703, + "rewards/margins": 36.431583404541016, + "rewards/rejected": -33.73298645019531, + "step": 196 + }, + { + "epoch": 0.21, + "learning_rate": 3.71896551724138e-05, + "logits/chosen": -0.38190075755119324, + "logits/rejected": -1.520878553390503, + "logps/chosen": -303.9383544921875, + "logps/rejected": -443.4150390625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 5.500533103942871, + "rewards/margins": 41.76572036743164, + "rewards/rejected": -36.26518630981445, + "step": 197 + }, + { + "epoch": 0.22, + "learning_rate": 3.713793103448276e-05, + "logits/chosen": -0.5008329153060913, + "logits/rejected": -1.4159965515136719, + "logps/chosen": -298.4696044921875, + "logps/rejected": -496.4792175292969, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.6865413188934326, + "rewards/margins": 43.41154098510742, + "rewards/rejected": -39.72499465942383, + "step": 198 + }, + { + "epoch": 0.22, + "learning_rate": 3.708620689655173e-05, + "logits/chosen": -0.30573636293411255, + "logits/rejected": -1.4757778644561768, + "logps/chosen": -404.08349609375, + "logps/rejected": -536.4090576171875, + "loss": 0.0994, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.837350606918335, + "rewards/margins": 47.625877380371094, + "rewards/rejected": -44.78852844238281, + "step": 199 + }, + { + "epoch": 0.22, + "learning_rate": 3.7034482758620695e-05, + "logits/chosen": -0.08167792111635208, + "logits/rejected": -1.6543066501617432, + "logps/chosen": -478.5865173339844, + "logps/rejected": -535.1455078125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 6.218913555145264, + "rewards/margins": 51.752662658691406, + "rewards/rejected": -45.533748626708984, + "step": 200 + }, + { + "epoch": 0.22, + "learning_rate": 3.698275862068966e-05, + "logits/chosen": -0.4204535186290741, + "logits/rejected": -1.4588035345077515, + "logps/chosen": -488.2334899902344, + "logps/rejected": -772.4085083007812, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.8939381837844849, + "rewards/margins": 68.2295913696289, + "rewards/rejected": -67.33565521240234, + "step": 201 + }, + { + "epoch": 0.22, + "learning_rate": 3.6931034482758624e-05, + "logits/chosen": -0.21560296416282654, + "logits/rejected": -1.2928329706192017, + "logps/chosen": -516.2886352539062, + "logps/rejected": -823.55908203125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.315321922302246, + "rewards/margins": 68.40852355957031, + "rewards/rejected": -74.72383880615234, + "step": 202 + }, + { + "epoch": 0.22, + "learning_rate": 3.6879310344827586e-05, + "logits/chosen": -0.42431268095970154, + "logits/rejected": -0.9408793449401855, + "logps/chosen": -475.7739562988281, + "logps/rejected": -990.97802734375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -15.18862247467041, + "rewards/margins": 75.22517395019531, + "rewards/rejected": -90.41380310058594, + "step": 203 + }, + { + "epoch": 0.22, + "learning_rate": 3.6827586206896554e-05, + "logits/chosen": -0.3706437051296234, + "logits/rejected": -1.5770463943481445, + "logps/chosen": -580.4349365234375, + "logps/rejected": -982.412841796875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.3259365558624268, + "rewards/margins": 85.03890991210938, + "rewards/rejected": -87.36485290527344, + "step": 204 + }, + { + "epoch": 0.22, + "learning_rate": 3.6775862068965515e-05, + "logits/chosen": -0.46955880522727966, + "logits/rejected": -1.2554025650024414, + "logps/chosen": -406.4046325683594, + "logps/rejected": -807.9384155273438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -8.099958419799805, + "rewards/margins": 62.139991760253906, + "rewards/rejected": -70.23995208740234, + "step": 205 + }, + { + "epoch": 0.22, + "learning_rate": 3.672413793103448e-05, + "logits/chosen": -0.1373772770166397, + "logits/rejected": -1.2258226871490479, + "logps/chosen": -484.78289794921875, + "logps/rejected": -775.5526123046875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.985532283782959, + "rewards/margins": 62.29596710205078, + "rewards/rejected": -69.28150177001953, + "step": 206 + }, + { + "epoch": 0.23, + "learning_rate": 3.667241379310345e-05, + "logits/chosen": -0.3842742145061493, + "logits/rejected": -1.3139704465866089, + "logps/chosen": -575.8878784179688, + "logps/rejected": -772.5504150390625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.301702499389648, + "rewards/margins": 63.3919563293457, + "rewards/rejected": -67.69366455078125, + "step": 207 + }, + { + "epoch": 0.23, + "learning_rate": 3.662068965517241e-05, + "logits/chosen": -0.16872625052928925, + "logits/rejected": -1.103173017501831, + "logps/chosen": -407.3558349609375, + "logps/rejected": -742.6690063476562, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.2132223844528198, + "rewards/margins": 64.90454864501953, + "rewards/rejected": -66.11776733398438, + "step": 208 + }, + { + "epoch": 0.23, + "learning_rate": 3.656896551724138e-05, + "logits/chosen": -0.22817087173461914, + "logits/rejected": -1.299288272857666, + "logps/chosen": -535.9667358398438, + "logps/rejected": -746.4730224609375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.898517608642578, + "rewards/margins": 60.320106506347656, + "rewards/rejected": -66.21862030029297, + "step": 209 + }, + { + "epoch": 0.23, + "learning_rate": 3.651724137931035e-05, + "logits/chosen": -0.43456578254699707, + "logits/rejected": -1.3410723209381104, + "logps/chosen": -434.5975036621094, + "logps/rejected": -722.338623046875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.796084880828857, + "rewards/margins": 59.60700988769531, + "rewards/rejected": -64.40309143066406, + "step": 210 + }, + { + "epoch": 0.23, + "learning_rate": 3.646551724137931e-05, + "logits/chosen": -0.099545419216156, + "logits/rejected": -1.4023548364639282, + "logps/chosen": -587.4548950195312, + "logps/rejected": -724.5460205078125, + "loss": 0.0007, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.0788767337799072, + "rewards/margins": 62.40778350830078, + "rewards/rejected": -64.48665618896484, + "step": 211 + }, + { + "epoch": 0.23, + "learning_rate": 3.641379310344828e-05, + "logits/chosen": -0.39735737442970276, + "logits/rejected": -1.2577073574066162, + "logps/chosen": -360.4342956542969, + "logps/rejected": -707.16552734375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.821351051330566, + "rewards/margins": 52.31917953491211, + "rewards/rejected": -57.140525817871094, + "step": 212 + }, + { + "epoch": 0.23, + "learning_rate": 3.6362068965517245e-05, + "logits/chosen": -0.32743093371391296, + "logits/rejected": -1.23341965675354, + "logps/chosen": -430.1988830566406, + "logps/rejected": -682.2755126953125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.829233407974243, + "rewards/margins": 54.4882926940918, + "rewards/rejected": -58.317527770996094, + "step": 213 + }, + { + "epoch": 0.23, + "learning_rate": 3.6310344827586206e-05, + "logits/chosen": -0.1188780814409256, + "logits/rejected": -1.211256980895996, + "logps/chosen": -478.95294189453125, + "logps/rejected": -628.7106323242188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.967594146728516, + "rewards/margins": 46.12427520751953, + "rewards/rejected": -52.09186935424805, + "step": 214 + }, + { + "epoch": 0.23, + "learning_rate": 3.6258620689655174e-05, + "logits/chosen": -0.15380708873271942, + "logits/rejected": -1.4826284646987915, + "logps/chosen": -502.1640319824219, + "logps/rejected": -613.2930297851562, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.2820662260055542, + "rewards/margins": 54.291709899902344, + "rewards/rejected": -53.0096435546875, + "step": 215 + }, + { + "epoch": 0.24, + "learning_rate": 3.620689655172414e-05, + "logits/chosen": -0.04195930063724518, + "logits/rejected": -1.2043323516845703, + "logps/chosen": -408.9276123046875, + "logps/rejected": -659.142578125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.6782640218734741, + "rewards/margins": 56.35860824584961, + "rewards/rejected": -54.68034362792969, + "step": 216 + }, + { + "epoch": 0.24, + "learning_rate": 3.615517241379311e-05, + "logits/chosen": -0.06693141162395477, + "logits/rejected": -1.0604804754257202, + "logps/chosen": -480.0716247558594, + "logps/rejected": -639.0257568359375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.6284542083740234, + "rewards/margins": 51.55681228637695, + "rewards/rejected": -55.18526840209961, + "step": 217 + }, + { + "epoch": 0.24, + "learning_rate": 3.610344827586207e-05, + "logits/chosen": -0.1353115737438202, + "logits/rejected": -1.2414418458938599, + "logps/chosen": -441.62567138671875, + "logps/rejected": -601.8602905273438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.7760581970214844, + "rewards/margins": 48.804161071777344, + "rewards/rejected": -50.58022689819336, + "step": 218 + }, + { + "epoch": 0.24, + "learning_rate": 3.605172413793104e-05, + "logits/chosen": -0.1814473569393158, + "logits/rejected": -1.4145801067352295, + "logps/chosen": -499.2064514160156, + "logps/rejected": -635.701904296875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.639859199523926, + "rewards/margins": 49.006500244140625, + "rewards/rejected": -54.646358489990234, + "step": 219 + }, + { + "epoch": 0.24, + "learning_rate": 3.6e-05, + "logits/chosen": -0.13956058025360107, + "logits/rejected": -0.8658940196037292, + "logps/chosen": -353.8810729980469, + "logps/rejected": -680.6879272460938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.038844347000122, + "rewards/margins": 52.220481872558594, + "rewards/rejected": -54.25932312011719, + "step": 220 + }, + { + "epoch": 0.24, + "learning_rate": 3.594827586206897e-05, + "logits/chosen": -0.191183939576149, + "logits/rejected": -1.2838574647903442, + "logps/chosen": -574.6087646484375, + "logps/rejected": -622.7803955078125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.7797561883926392, + "rewards/margins": 50.8365592956543, + "rewards/rejected": -50.05680465698242, + "step": 221 + }, + { + "epoch": 0.24, + "learning_rate": 3.589655172413793e-05, + "logits/chosen": 0.004895269870758057, + "logits/rejected": -1.0355963706970215, + "logps/chosen": -378.75286865234375, + "logps/rejected": -611.6342163085938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.560927152633667, + "rewards/margins": 53.636077880859375, + "rewards/rejected": -52.07514953613281, + "step": 222 + }, + { + "epoch": 0.24, + "learning_rate": 3.58448275862069e-05, + "logits/chosen": -0.0416397899389267, + "logits/rejected": -1.0201894044876099, + "logps/chosen": -286.4012145996094, + "logps/rejected": -585.0177001953125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.24752166867256165, + "rewards/margins": 52.19663619995117, + "rewards/rejected": -51.94911193847656, + "step": 223 + }, + { + "epoch": 0.24, + "learning_rate": 3.579310344827586e-05, + "logits/chosen": 0.08275717496871948, + "logits/rejected": -1.165152668952942, + "logps/chosen": -402.03204345703125, + "logps/rejected": -646.4345092773438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17879903316497803, + "rewards/margins": 52.9560546875, + "rewards/rejected": -53.13485336303711, + "step": 224 + }, + { + "epoch": 0.25, + "learning_rate": 3.574137931034483e-05, + "logits/chosen": 0.046931296586990356, + "logits/rejected": -1.2173397541046143, + "logps/chosen": -379.2846374511719, + "logps/rejected": -596.7713623046875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.054905891418457, + "rewards/margins": 49.74352264404297, + "rewards/rejected": -52.79842758178711, + "step": 225 + }, + { + "epoch": 0.25, + "learning_rate": 3.5689655172413795e-05, + "logits/chosen": -0.16892120242118835, + "logits/rejected": -1.086595892906189, + "logps/chosen": -383.6666564941406, + "logps/rejected": -613.8699951171875, + "loss": 0.0022, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.7993450164794922, + "rewards/margins": 48.63136291503906, + "rewards/rejected": -50.43070602416992, + "step": 226 + }, + { + "epoch": 0.25, + "learning_rate": 3.5637931034482757e-05, + "logits/chosen": -0.0628051906824112, + "logits/rejected": -1.2338706254959106, + "logps/chosen": -420.1468505859375, + "logps/rejected": -569.7159423828125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.7352135181427002, + "rewards/margins": 48.35806655883789, + "rewards/rejected": -50.09328079223633, + "step": 227 + }, + { + "epoch": 0.25, + "learning_rate": 3.5586206896551725e-05, + "logits/chosen": -0.13550832867622375, + "logits/rejected": -1.3356702327728271, + "logps/chosen": -471.48358154296875, + "logps/rejected": -597.1064453125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.008322715759277, + "rewards/margins": 46.82109832763672, + "rewards/rejected": -50.82941818237305, + "step": 228 + }, + { + "epoch": 0.25, + "learning_rate": 3.553448275862069e-05, + "logits/chosen": -0.19657878577709198, + "logits/rejected": -1.144600510597229, + "logps/chosen": -416.9407653808594, + "logps/rejected": -661.7545166015625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.8045517206192017, + "rewards/margins": 57.125457763671875, + "rewards/rejected": -58.93001174926758, + "step": 229 + }, + { + "epoch": 0.25, + "learning_rate": 3.5482758620689654e-05, + "logits/chosen": -0.16479764878749847, + "logits/rejected": -1.1088446378707886, + "logps/chosen": -361.6660461425781, + "logps/rejected": -531.82275390625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.02301928400993347, + "rewards/margins": 45.43212127685547, + "rewards/rejected": -45.455135345458984, + "step": 230 + }, + { + "epoch": 0.25, + "learning_rate": 3.543103448275862e-05, + "logits/chosen": -0.06933975219726562, + "logits/rejected": -1.1754459142684937, + "logps/chosen": -392.62481689453125, + "logps/rejected": -581.8388671875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.2902095317840576, + "rewards/margins": 48.50893783569336, + "rewards/rejected": -49.79915237426758, + "step": 231 + }, + { + "epoch": 0.25, + "learning_rate": 3.537931034482759e-05, + "logits/chosen": -0.26539716124534607, + "logits/rejected": -1.2646820545196533, + "logps/chosen": -315.2926940917969, + "logps/rejected": -573.3612670898438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.7868924140930176, + "rewards/margins": 45.750999450683594, + "rewards/rejected": -44.964107513427734, + "step": 232 + }, + { + "epoch": 0.25, + "learning_rate": 3.532758620689656e-05, + "logits/chosen": 0.00513702817261219, + "logits/rejected": -1.4749549627304077, + "logps/chosen": -537.417724609375, + "logps/rejected": -592.2806396484375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.04149657487869263, + "rewards/margins": 50.454158782958984, + "rewards/rejected": -50.412662506103516, + "step": 233 + }, + { + "epoch": 0.26, + "learning_rate": 3.527586206896552e-05, + "logits/chosen": -0.05358482524752617, + "logits/rejected": -1.2646411657333374, + "logps/chosen": -435.2836608886719, + "logps/rejected": -563.5516357421875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.9884543418884277, + "rewards/margins": 47.98497009277344, + "rewards/rejected": -45.99651336669922, + "step": 234 + }, + { + "epoch": 0.26, + "learning_rate": 3.522413793103449e-05, + "logits/chosen": 0.0024913623929023743, + "logits/rejected": -1.0907244682312012, + "logps/chosen": -382.8013000488281, + "logps/rejected": -551.0772705078125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.0033998489379883, + "rewards/margins": 44.1395263671875, + "rewards/rejected": -45.14292526245117, + "step": 235 + }, + { + "epoch": 0.26, + "learning_rate": 3.5172413793103455e-05, + "logits/chosen": -0.054612670093774796, + "logits/rejected": -1.1346925497055054, + "logps/chosen": -414.9861755371094, + "logps/rejected": -596.8836059570312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.3777976036071777, + "rewards/margins": 48.898712158203125, + "rewards/rejected": -50.276512145996094, + "step": 236 + }, + { + "epoch": 0.26, + "learning_rate": 3.5120689655172416e-05, + "logits/chosen": -0.04273924231529236, + "logits/rejected": -1.4653762578964233, + "logps/chosen": -521.715576171875, + "logps/rejected": -577.45263671875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3539397120475769, + "rewards/margins": 48.13010025024414, + "rewards/rejected": -47.776161193847656, + "step": 237 + }, + { + "epoch": 0.26, + "learning_rate": 3.5068965517241384e-05, + "logits/chosen": -0.09971227496862411, + "logits/rejected": -1.412309169769287, + "logps/chosen": -541.3079833984375, + "logps/rejected": -546.3953857421875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.18726038932800293, + "rewards/margins": 47.805355072021484, + "rewards/rejected": -47.61809539794922, + "step": 238 + }, + { + "epoch": 0.26, + "learning_rate": 3.5017241379310345e-05, + "logits/chosen": 0.04640922695398331, + "logits/rejected": -1.2290853261947632, + "logps/chosen": -445.3258056640625, + "logps/rejected": -590.5760498046875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5043596029281616, + "rewards/margins": 49.53385543823242, + "rewards/rejected": -49.02949523925781, + "step": 239 + }, + { + "epoch": 0.26, + "learning_rate": 3.4965517241379313e-05, + "logits/chosen": -0.2436431348323822, + "logits/rejected": -1.1010798215866089, + "logps/chosen": -310.34173583984375, + "logps/rejected": -582.0191650390625, + "loss": 0.0723, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.261377334594727, + "rewards/margins": 41.790748596191406, + "rewards/rejected": -48.0521240234375, + "step": 240 + }, + { + "epoch": 0.26, + "learning_rate": 3.4913793103448275e-05, + "logits/chosen": -0.0976068302989006, + "logits/rejected": -1.1630501747131348, + "logps/chosen": -407.0189514160156, + "logps/rejected": -587.5228881835938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.1800687313079834, + "rewards/margins": 50.658512115478516, + "rewards/rejected": -48.47844314575195, + "step": 241 + }, + { + "epoch": 0.26, + "learning_rate": 3.486206896551724e-05, + "logits/chosen": -0.16128882765769958, + "logits/rejected": -1.3036680221557617, + "logps/chosen": -364.50506591796875, + "logps/rejected": -502.24871826171875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.9164876341819763, + "rewards/margins": 44.952972412109375, + "rewards/rejected": -44.03648376464844, + "step": 242 + }, + { + "epoch": 0.27, + "learning_rate": 3.4810344827586204e-05, + "logits/chosen": -0.020467162132263184, + "logits/rejected": -1.3299024105072021, + "logps/chosen": -440.3603820800781, + "logps/rejected": -508.4949645996094, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.407674789428711, + "rewards/margins": 41.96260070800781, + "rewards/rejected": -39.55492401123047, + "step": 243 + }, + { + "epoch": 0.27, + "learning_rate": 3.475862068965517e-05, + "logits/chosen": -0.12424595654010773, + "logits/rejected": -1.2621549367904663, + "logps/chosen": -372.3587951660156, + "logps/rejected": -545.2764282226562, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.37911176681518555, + "rewards/margins": 44.348899841308594, + "rewards/rejected": -43.96978759765625, + "step": 244 + }, + { + "epoch": 0.27, + "learning_rate": 3.470689655172414e-05, + "logits/chosen": -0.254048228263855, + "logits/rejected": -1.3378281593322754, + "logps/chosen": -400.1960754394531, + "logps/rejected": -541.253173828125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.260645866394043, + "rewards/margins": 48.459110260009766, + "rewards/rejected": -44.198463439941406, + "step": 245 + }, + { + "epoch": 0.27, + "learning_rate": 3.46551724137931e-05, + "logits/chosen": -0.09928327798843384, + "logits/rejected": -1.0079985857009888, + "logps/chosen": -318.05535888671875, + "logps/rejected": -475.6454162597656, + "loss": 0.0001, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.3407931327819824, + "rewards/margins": 39.11427307128906, + "rewards/rejected": -37.773475646972656, + "step": 246 + }, + { + "epoch": 0.27, + "learning_rate": 3.460344827586207e-05, + "logits/chosen": -0.17899078130722046, + "logits/rejected": -1.0408397912979126, + "logps/chosen": -260.2774658203125, + "logps/rejected": -587.495361328125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.1181957721710205, + "rewards/margins": 47.448177337646484, + "rewards/rejected": -48.566375732421875, + "step": 247 + }, + { + "epoch": 0.27, + "learning_rate": 3.455172413793104e-05, + "logits/chosen": -0.15649932622909546, + "logits/rejected": -1.3504737615585327, + "logps/chosen": -352.34295654296875, + "logps/rejected": -463.55999755859375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.939250946044922, + "rewards/margins": 40.67493438720703, + "rewards/rejected": -36.73568344116211, + "step": 248 + }, + { + "epoch": 0.27, + "learning_rate": 3.4500000000000005e-05, + "logits/chosen": -0.047948114573955536, + "logits/rejected": -1.2849537134170532, + "logps/chosen": -394.3268737792969, + "logps/rejected": -474.2886962890625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.573990821838379, + "rewards/margins": 43.014068603515625, + "rewards/rejected": -40.44007873535156, + "step": 249 + }, + { + "epoch": 0.27, + "learning_rate": 3.4448275862068966e-05, + "logits/chosen": -0.23074887692928314, + "logits/rejected": -1.2006328105926514, + "logps/chosen": -316.7071228027344, + "logps/rejected": -518.9232177734375, + "loss": 0.4199, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.6951587200164795, + "rewards/margins": 36.5284309387207, + "rewards/rejected": -33.833274841308594, + "step": 250 + }, + { + "epoch": 0.27, + "learning_rate": 3.4396551724137934e-05, + "logits/chosen": -0.16572320461273193, + "logits/rejected": -1.3215007781982422, + "logps/chosen": -444.8575744628906, + "logps/rejected": -486.4254455566406, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.5085513591766357, + "rewards/margins": 45.29148483276367, + "rewards/rejected": -42.78293228149414, + "step": 251 + }, + { + "epoch": 0.27, + "learning_rate": 3.43448275862069e-05, + "logits/chosen": -0.38803166151046753, + "logits/rejected": -1.2651021480560303, + "logps/chosen": -350.4290771484375, + "logps/rejected": -569.2044677734375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.44204509258270264, + "rewards/margins": 48.26247787475586, + "rewards/rejected": -47.820430755615234, + "step": 252 + }, + { + "epoch": 0.28, + "learning_rate": 3.4293103448275864e-05, + "logits/chosen": -0.19859111309051514, + "logits/rejected": -1.1871520280838013, + "logps/chosen": -354.1485900878906, + "logps/rejected": -605.5889892578125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.625746250152588, + "rewards/margins": 48.65724563598633, + "rewards/rejected": -52.282989501953125, + "step": 253 + }, + { + "epoch": 0.28, + "learning_rate": 3.424137931034483e-05, + "logits/chosen": -0.003446124028414488, + "logits/rejected": -1.4357542991638184, + "logps/chosen": -464.06402587890625, + "logps/rejected": -584.0009155273438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.0634913444519043, + "rewards/margins": 49.579891204833984, + "rewards/rejected": -51.64338302612305, + "step": 254 + }, + { + "epoch": 0.28, + "learning_rate": 3.41896551724138e-05, + "logits/chosen": 0.005802547559142113, + "logits/rejected": -1.1215245723724365, + "logps/chosen": -487.831298828125, + "logps/rejected": -645.035400390625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.5290374755859375, + "rewards/margins": 52.562137603759766, + "rewards/rejected": -56.0911750793457, + "step": 255 + }, + { + "epoch": 0.28, + "learning_rate": 3.413793103448276e-05, + "logits/chosen": -0.27775824069976807, + "logits/rejected": -1.4532489776611328, + "logps/chosen": -468.9490051269531, + "logps/rejected": -570.933837890625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.029543161392212, + "rewards/margins": 45.34672927856445, + "rewards/rejected": -48.37627029418945, + "step": 256 + }, + { + "epoch": 0.28, + "learning_rate": 3.408620689655173e-05, + "logits/chosen": -0.04865233972668648, + "logits/rejected": -1.0499883890151978, + "logps/chosen": -517.0708618164062, + "logps/rejected": -639.54541015625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.305459499359131, + "rewards/margins": 50.58137512207031, + "rewards/rejected": -55.88683319091797, + "step": 257 + }, + { + "epoch": 0.28, + "learning_rate": 3.403448275862069e-05, + "logits/chosen": -0.2616862654685974, + "logits/rejected": -1.0700899362564087, + "logps/chosen": -401.421875, + "logps/rejected": -618.0116577148438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -10.54936408996582, + "rewards/margins": 44.39057922363281, + "rewards/rejected": -54.939937591552734, + "step": 258 + }, + { + "epoch": 0.28, + "learning_rate": 3.398275862068966e-05, + "logits/chosen": 0.02191777713596821, + "logits/rejected": -1.4829727411270142, + "logps/chosen": -577.8099365234375, + "logps/rejected": -660.3970947265625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -7.7517547607421875, + "rewards/margins": 47.69034194946289, + "rewards/rejected": -55.442100524902344, + "step": 259 + }, + { + "epoch": 0.28, + "learning_rate": 3.393103448275862e-05, + "logits/chosen": -0.055667877197265625, + "logits/rejected": -1.1761151552200317, + "logps/chosen": -493.5972900390625, + "logps/rejected": -705.7115478515625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -8.951239585876465, + "rewards/margins": 50.98706817626953, + "rewards/rejected": -59.93830871582031, + "step": 260 + }, + { + "epoch": 0.28, + "learning_rate": 3.387931034482759e-05, + "logits/chosen": -0.010112253949046135, + "logits/rejected": -1.1061949729919434, + "logps/chosen": -544.2592163085938, + "logps/rejected": -648.0838012695312, + "loss": 0.0685, + "rewards/accuracies": 0.9375, + "rewards/chosen": -10.865694999694824, + "rewards/margins": 42.983341217041016, + "rewards/rejected": -53.84903335571289, + "step": 261 + }, + { + "epoch": 0.29, + "learning_rate": 3.382758620689655e-05, + "logits/chosen": -0.3130599856376648, + "logits/rejected": -1.1265610456466675, + "logps/chosen": -394.48321533203125, + "logps/rejected": -624.9985961914062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -7.234168529510498, + "rewards/margins": 44.61975860595703, + "rewards/rejected": -51.853919982910156, + "step": 262 + }, + { + "epoch": 0.29, + "learning_rate": 3.3775862068965516e-05, + "logits/chosen": -0.5001847743988037, + "logits/rejected": -1.2859660387039185, + "logps/chosen": -381.885009765625, + "logps/rejected": -276.1200256347656, + "loss": 0.0274, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.0845551490783691, + "rewards/margins": 17.89061737060547, + "rewards/rejected": -18.975173950195312, + "step": 263 + }, + { + "epoch": 0.29, + "learning_rate": 3.3724137931034484e-05, + "logits/chosen": -0.16429847478866577, + "logits/rejected": -1.3770393133163452, + "logps/chosen": -416.09918212890625, + "logps/rejected": -444.2696533203125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.0829229354858398, + "rewards/margins": 36.78654098510742, + "rewards/rejected": -37.86946487426758, + "step": 264 + }, + { + "epoch": 0.29, + "learning_rate": 3.367241379310345e-05, + "logits/chosen": -0.027936279773712158, + "logits/rejected": -1.2221875190734863, + "logps/chosen": -522.7379150390625, + "logps/rejected": -575.3955688476562, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.6051886081695557, + "rewards/margins": 44.91753005981445, + "rewards/rejected": -48.52272033691406, + "step": 265 + }, + { + "epoch": 0.29, + "learning_rate": 3.3620689655172414e-05, + "logits/chosen": -0.023054365068674088, + "logits/rejected": -1.0561506748199463, + "logps/chosen": -476.08087158203125, + "logps/rejected": -584.338134765625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.656482994556427, + "rewards/margins": 47.59941864013672, + "rewards/rejected": -48.25589370727539, + "step": 266 + }, + { + "epoch": 0.29, + "learning_rate": 3.356896551724138e-05, + "logits/chosen": -0.21733063459396362, + "logits/rejected": -0.9282070994377136, + "logps/chosen": -309.57916259765625, + "logps/rejected": -548.53564453125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -9.397457122802734, + "rewards/margins": 38.486846923828125, + "rewards/rejected": -47.884307861328125, + "step": 267 + }, + { + "epoch": 0.29, + "learning_rate": 3.351724137931035e-05, + "logits/chosen": -0.13685892522335052, + "logits/rejected": -0.881144642829895, + "logps/chosen": -402.0709228515625, + "logps/rejected": -593.3131103515625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -10.071722984313965, + "rewards/margins": 41.16486358642578, + "rewards/rejected": -51.23658752441406, + "step": 268 + }, + { + "epoch": 0.29, + "learning_rate": 3.346551724137931e-05, + "logits/chosen": 0.0004104040563106537, + "logits/rejected": -1.2138159275054932, + "logps/chosen": -586.6741333007812, + "logps/rejected": -579.4757080078125, + "loss": 0.0041, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.9185256958007812, + "rewards/margins": 41.63846206665039, + "rewards/rejected": -44.556983947753906, + "step": 269 + }, + { + "epoch": 0.29, + "learning_rate": 3.341379310344828e-05, + "logits/chosen": -0.11678376793861389, + "logits/rejected": -1.2740269899368286, + "logps/chosen": -518.05908203125, + "logps/rejected": -548.7737426757812, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -7.559596061706543, + "rewards/margins": 40.736881256103516, + "rewards/rejected": -48.296478271484375, + "step": 270 + }, + { + "epoch": 0.3, + "learning_rate": 3.336206896551725e-05, + "logits/chosen": -0.2319948375225067, + "logits/rejected": -1.038511872291565, + "logps/chosen": -457.996337890625, + "logps/rejected": -614.3683471679688, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.421997547149658, + "rewards/margins": 46.14197540283203, + "rewards/rejected": -49.56397247314453, + "step": 271 + }, + { + "epoch": 0.3, + "learning_rate": 3.331034482758621e-05, + "logits/chosen": -0.175481915473938, + "logits/rejected": -0.9189218878746033, + "logps/chosen": -413.4037170410156, + "logps/rejected": -577.8434448242188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.640353679656982, + "rewards/margins": 43.46728515625, + "rewards/rejected": -49.107635498046875, + "step": 272 + }, + { + "epoch": 0.3, + "learning_rate": 3.3258620689655176e-05, + "logits/chosen": 0.041123226284980774, + "logits/rejected": -1.239971399307251, + "logps/chosen": -625.6574096679688, + "logps/rejected": -620.5679321289062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.8174872398376465, + "rewards/margins": 47.40455627441406, + "rewards/rejected": -52.2220458984375, + "step": 273 + }, + { + "epoch": 0.3, + "learning_rate": 3.3206896551724144e-05, + "logits/chosen": 0.016035079956054688, + "logits/rejected": -1.058797836303711, + "logps/chosen": -433.7515869140625, + "logps/rejected": -610.6272583007812, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.2071919441223145, + "rewards/margins": 46.5596809387207, + "rewards/rejected": -51.766876220703125, + "step": 274 + }, + { + "epoch": 0.3, + "learning_rate": 3.3155172413793105e-05, + "logits/chosen": 0.005678463727235794, + "logits/rejected": -1.0126562118530273, + "logps/chosen": -519.1249389648438, + "logps/rejected": -667.5272216796875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.07732820510864258, + "rewards/margins": 48.706153869628906, + "rewards/rejected": -48.628822326660156, + "step": 275 + }, + { + "epoch": 0.3, + "learning_rate": 3.310344827586207e-05, + "logits/chosen": -0.06707608699798584, + "logits/rejected": -1.1867536306381226, + "logps/chosen": -434.00762939453125, + "logps/rejected": -475.2304382324219, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.4429032802581787, + "rewards/margins": 38.4698486328125, + "rewards/rejected": -41.91275405883789, + "step": 276 + }, + { + "epoch": 0.3, + "learning_rate": 3.3051724137931035e-05, + "logits/chosen": 0.016937214881181717, + "logits/rejected": -0.9644786715507507, + "logps/chosen": -438.28424072265625, + "logps/rejected": -540.373291015625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.785179615020752, + "rewards/margins": 39.099998474121094, + "rewards/rejected": -42.88517761230469, + "step": 277 + }, + { + "epoch": 0.3, + "learning_rate": 3.3e-05, + "logits/chosen": -0.13098500669002533, + "logits/rejected": -1.1797230243682861, + "logps/chosen": -481.2913818359375, + "logps/rejected": -575.146240234375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.13218802213668823, + "rewards/margins": 49.17036056518555, + "rewards/rejected": -49.03817367553711, + "step": 278 + }, + { + "epoch": 0.3, + "learning_rate": 3.2948275862068964e-05, + "logits/chosen": -0.14736391603946686, + "logits/rejected": -1.0556457042694092, + "logps/chosen": -449.7217712402344, + "logps/rejected": -555.5156860351562, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -9.082195281982422, + "rewards/margins": 34.08275604248047, + "rewards/rejected": -43.16495132446289, + "step": 279 + }, + { + "epoch": 0.31, + "learning_rate": 3.289655172413793e-05, + "logits/chosen": -0.1435016244649887, + "logits/rejected": -1.0380524396896362, + "logps/chosen": -338.7105712890625, + "logps/rejected": -543.041259765625, + "loss": 0.0016, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.8971967697143555, + "rewards/margins": 37.95754623413086, + "rewards/rejected": -43.85474395751953, + "step": 280 + }, + { + "epoch": 0.31, + "learning_rate": 3.28448275862069e-05, + "logits/chosen": -0.013942647725343704, + "logits/rejected": -1.056246042251587, + "logps/chosen": -434.2045593261719, + "logps/rejected": -555.7156372070312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.764453887939453, + "rewards/margins": 43.68494415283203, + "rewards/rejected": -46.44939422607422, + "step": 281 + }, + { + "epoch": 0.31, + "learning_rate": 3.279310344827586e-05, + "logits/chosen": -0.11425824463367462, + "logits/rejected": -0.994450569152832, + "logps/chosen": -397.4031677246094, + "logps/rejected": -641.703369140625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -7.2452192306518555, + "rewards/margins": 43.18861389160156, + "rewards/rejected": -50.433834075927734, + "step": 282 + }, + { + "epoch": 0.31, + "learning_rate": 3.274137931034483e-05, + "logits/chosen": 0.03589939698576927, + "logits/rejected": -1.0763846635818481, + "logps/chosen": -394.02056884765625, + "logps/rejected": -567.1450805664062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -7.877297401428223, + "rewards/margins": 43.63758850097656, + "rewards/rejected": -51.514888763427734, + "step": 283 + }, + { + "epoch": 0.31, + "learning_rate": 3.26896551724138e-05, + "logits/chosen": -0.042599305510520935, + "logits/rejected": -0.9894676208496094, + "logps/chosen": -390.73968505859375, + "logps/rejected": -588.50244140625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.6523871421813965, + "rewards/margins": 41.83343505859375, + "rewards/rejected": -48.48582077026367, + "step": 284 + }, + { + "epoch": 0.31, + "learning_rate": 3.263793103448276e-05, + "logits/chosen": -0.04844583570957184, + "logits/rejected": -1.3764398097991943, + "logps/chosen": -520.421875, + "logps/rejected": -584.4087524414062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.5111515522003174, + "rewards/margins": 48.571834564208984, + "rewards/rejected": -49.082984924316406, + "step": 285 + }, + { + "epoch": 0.31, + "learning_rate": 3.2586206896551726e-05, + "logits/chosen": -0.10487770289182663, + "logits/rejected": -1.2500025033950806, + "logps/chosen": -576.0562133789062, + "logps/rejected": -532.1016235351562, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -8.098315238952637, + "rewards/margins": 37.71330261230469, + "rewards/rejected": -45.811614990234375, + "step": 286 + }, + { + "epoch": 0.31, + "learning_rate": 3.2534482758620694e-05, + "logits/chosen": -0.015081293880939484, + "logits/rejected": -1.127041220664978, + "logps/chosen": -518.3914794921875, + "logps/rejected": -600.3658447265625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -9.67361068725586, + "rewards/margins": 43.359867095947266, + "rewards/rejected": -53.033477783203125, + "step": 287 + }, + { + "epoch": 0.31, + "learning_rate": 3.2482758620689655e-05, + "logits/chosen": -0.10006409883499146, + "logits/rejected": -0.9317835569381714, + "logps/chosen": -492.3525085449219, + "logps/rejected": -598.6736450195312, + "loss": 0.0192, + "rewards/accuracies": 1.0, + "rewards/chosen": -8.045526504516602, + "rewards/margins": 40.16624450683594, + "rewards/rejected": -48.21177291870117, + "step": 288 + }, + { + "epoch": 0.32, + "learning_rate": 3.2431034482758623e-05, + "logits/chosen": -0.14413213729858398, + "logits/rejected": -0.8675107955932617, + "logps/chosen": -349.6807556152344, + "logps/rejected": -615.5933227539062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -8.971858978271484, + "rewards/margins": 42.66694641113281, + "rewards/rejected": -51.63880920410156, + "step": 289 + }, + { + "epoch": 0.32, + "learning_rate": 3.237931034482759e-05, + "logits/chosen": -0.06418796628713608, + "logits/rejected": -1.261458396911621, + "logps/chosen": -456.44097900390625, + "logps/rejected": -582.775634765625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.9118471145629883, + "rewards/margins": 45.231956481933594, + "rewards/rejected": -49.143798828125, + "step": 290 + }, + { + "epoch": 0.32, + "learning_rate": 3.232758620689655e-05, + "logits/chosen": 0.015173820778727531, + "logits/rejected": -1.3602452278137207, + "logps/chosen": -556.77294921875, + "logps/rejected": -552.7382202148438, + "loss": 0.249, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.175498008728027, + "rewards/margins": 42.23072052001953, + "rewards/rejected": -47.40621566772461, + "step": 291 + }, + { + "epoch": 0.32, + "learning_rate": 3.227586206896552e-05, + "logits/chosen": 0.0050443485379219055, + "logits/rejected": -1.1559348106384277, + "logps/chosen": -483.6645812988281, + "logps/rejected": -579.65478515625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.6171573996543884, + "rewards/margins": 46.14912414550781, + "rewards/rejected": -46.76628112792969, + "step": 292 + }, + { + "epoch": 0.32, + "learning_rate": 3.222413793103449e-05, + "logits/chosen": -0.015672463923692703, + "logits/rejected": -0.8948309421539307, + "logps/chosen": -404.2826843261719, + "logps/rejected": -543.1045532226562, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.27553945779800415, + "rewards/margins": 44.960777282714844, + "rewards/rejected": -44.68523406982422, + "step": 293 + }, + { + "epoch": 0.32, + "learning_rate": 3.217241379310345e-05, + "logits/chosen": -0.0836256816983223, + "logits/rejected": -0.947052538394928, + "logps/chosen": -308.1258850097656, + "logps/rejected": -547.1864013671875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.2600884437561035, + "rewards/margins": 40.025543212890625, + "rewards/rejected": -43.28562927246094, + "step": 294 + }, + { + "epoch": 0.32, + "learning_rate": 3.212068965517242e-05, + "logits/chosen": 0.0691690593957901, + "logits/rejected": -1.4175320863723755, + "logps/chosen": -564.2918701171875, + "logps/rejected": -524.4912719726562, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 5.575163841247559, + "rewards/margins": 46.277183532714844, + "rewards/rejected": -40.70201873779297, + "step": 295 + }, + { + "epoch": 0.32, + "learning_rate": 3.206896551724138e-05, + "logits/chosen": -0.2818552851676941, + "logits/rejected": -1.078634262084961, + "logps/chosen": -350.2765808105469, + "logps/rejected": -537.8307495117188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6971882581710815, + "rewards/margins": 41.953392028808594, + "rewards/rejected": -41.256202697753906, + "step": 296 + }, + { + "epoch": 0.32, + "learning_rate": 3.201724137931035e-05, + "logits/chosen": -0.16628246009349823, + "logits/rejected": -1.2242597341537476, + "logps/chosen": -388.05078125, + "logps/rejected": -504.51910400390625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.7080140113830566, + "rewards/margins": 41.291893005371094, + "rewards/rejected": -39.58388137817383, + "step": 297 + }, + { + "epoch": 0.33, + "learning_rate": 3.196551724137931e-05, + "logits/chosen": -0.13853780925273895, + "logits/rejected": -1.0252296924591064, + "logps/chosen": -358.7446594238281, + "logps/rejected": -497.95281982421875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.08794373273849487, + "rewards/margins": 39.5137825012207, + "rewards/rejected": -39.425838470458984, + "step": 298 + }, + { + "epoch": 0.33, + "learning_rate": 3.1913793103448276e-05, + "logits/chosen": -0.10767735540866852, + "logits/rejected": -1.0644776821136475, + "logps/chosen": -374.3890075683594, + "logps/rejected": -475.7244873046875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.7640305161476135, + "rewards/margins": 39.74821472167969, + "rewards/rejected": -38.984188079833984, + "step": 299 + }, + { + "epoch": 0.33, + "learning_rate": 3.1862068965517244e-05, + "logits/chosen": -0.09947740286588669, + "logits/rejected": -1.1763038635253906, + "logps/chosen": -364.96258544921875, + "logps/rejected": -410.973876953125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.1341031789779663, + "rewards/margins": 34.35022735595703, + "rewards/rejected": -33.21612548828125, + "step": 300 + }, + { + "epoch": 0.33, + "learning_rate": 3.1810344827586206e-05, + "logits/chosen": -0.034785281866788864, + "logits/rejected": -1.2451276779174805, + "logps/chosen": -440.1756896972656, + "logps/rejected": -425.47509765625, + "loss": 0.0001, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.590677261352539, + "rewards/margins": 37.24711990356445, + "rewards/rejected": -33.65644454956055, + "step": 301 + }, + { + "epoch": 0.33, + "learning_rate": 3.1758620689655174e-05, + "logits/chosen": -0.15350031852722168, + "logits/rejected": -1.0885205268859863, + "logps/chosen": -402.491455078125, + "logps/rejected": -392.4815673828125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.4919495582580566, + "rewards/margins": 31.594680786132812, + "rewards/rejected": -28.10272979736328, + "step": 302 + }, + { + "epoch": 0.33, + "learning_rate": 3.170689655172414e-05, + "logits/chosen": -0.14404729008674622, + "logits/rejected": -1.1483252048492432, + "logps/chosen": -376.1712646484375, + "logps/rejected": -410.5417175292969, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.8741713762283325, + "rewards/margins": 34.73831558227539, + "rewards/rejected": -33.8641471862793, + "step": 303 + }, + { + "epoch": 0.33, + "learning_rate": 3.16551724137931e-05, + "logits/chosen": -0.19463568925857544, + "logits/rejected": -1.184244155883789, + "logps/chosen": -337.38482666015625, + "logps/rejected": -382.4162292480469, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.7294703722000122, + "rewards/margins": 30.66646957397461, + "rewards/rejected": -28.93699836730957, + "step": 304 + }, + { + "epoch": 0.33, + "learning_rate": 3.160344827586207e-05, + "logits/chosen": -0.1128799319267273, + "logits/rejected": -1.1655908823013306, + "logps/chosen": -394.9966735839844, + "logps/rejected": -388.04351806640625, + "loss": 0.0024, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.233245849609375, + "rewards/margins": 32.936092376708984, + "rewards/rejected": -30.702844619750977, + "step": 305 + }, + { + "epoch": 0.33, + "learning_rate": 3.155172413793104e-05, + "logits/chosen": -0.12499256432056427, + "logits/rejected": -1.0794974565505981, + "logps/chosen": -402.0507507324219, + "logps/rejected": -449.8464660644531, + "loss": 0.0965, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.6880790591239929, + "rewards/margins": 32.26117706298828, + "rewards/rejected": -31.573097229003906, + "step": 306 + }, + { + "epoch": 0.33, + "learning_rate": 3.15e-05, + "logits/chosen": -0.16316160559654236, + "logits/rejected": -1.1093841791152954, + "logps/chosen": -339.54443359375, + "logps/rejected": -498.3818664550781, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.30418586730957, + "rewards/margins": 36.54359436035156, + "rewards/rejected": -40.84778594970703, + "step": 307 + }, + { + "epoch": 0.34, + "learning_rate": 3.144827586206897e-05, + "logits/chosen": 0.031052295118570328, + "logits/rejected": -1.198643684387207, + "logps/chosen": -394.73394775390625, + "logps/rejected": -528.90966796875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.43565046787261963, + "rewards/margins": 45.74468231201172, + "rewards/rejected": -46.180335998535156, + "step": 308 + }, + { + "epoch": 0.34, + "learning_rate": 3.1396551724137936e-05, + "logits/chosen": -0.05930813029408455, + "logits/rejected": -1.2569350004196167, + "logps/chosen": -515.5428466796875, + "logps/rejected": -561.1205444335938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.4287011623382568, + "rewards/margins": 46.44762420654297, + "rewards/rejected": -45.0189208984375, + "step": 309 + }, + { + "epoch": 0.34, + "learning_rate": 3.13448275862069e-05, + "logits/chosen": 0.14158597588539124, + "logits/rejected": -1.1525517702102661, + "logps/chosen": -493.8555603027344, + "logps/rejected": -557.8233642578125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.293236255645752, + "rewards/margins": 44.08985137939453, + "rewards/rejected": -46.38308334350586, + "step": 310 + }, + { + "epoch": 0.34, + "learning_rate": 3.1293103448275865e-05, + "logits/chosen": -0.13309258222579956, + "logits/rejected": -1.1186158657073975, + "logps/chosen": -478.7046203613281, + "logps/rejected": -570.7393798828125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.5453602075576782, + "rewards/margins": 45.14695739746094, + "rewards/rejected": -45.692317962646484, + "step": 311 + }, + { + "epoch": 0.34, + "learning_rate": 3.124137931034483e-05, + "logits/chosen": 0.05075529217720032, + "logits/rejected": -1.1768913269042969, + "logps/chosen": -471.7716064453125, + "logps/rejected": -529.3003540039062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.3143967390060425, + "rewards/margins": 44.80303192138672, + "rewards/rejected": -43.48863220214844, + "step": 312 + }, + { + "epoch": 0.34, + "learning_rate": 3.1189655172413794e-05, + "logits/chosen": 0.08374657481908798, + "logits/rejected": -1.1194422245025635, + "logps/chosen": -453.62115478515625, + "logps/rejected": -556.13232421875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.535671591758728, + "rewards/margins": 45.361289978027344, + "rewards/rejected": -45.89696502685547, + "step": 313 + }, + { + "epoch": 0.34, + "learning_rate": 3.113793103448276e-05, + "logits/chosen": -0.18719206750392914, + "logits/rejected": -1.005627989768982, + "logps/chosen": -429.99652099609375, + "logps/rejected": -530.39697265625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.788636207580566, + "rewards/margins": 38.725555419921875, + "rewards/rejected": -45.51419448852539, + "step": 314 + }, + { + "epoch": 0.34, + "learning_rate": 3.1086206896551724e-05, + "logits/chosen": -0.058684833347797394, + "logits/rejected": -1.0559102296829224, + "logps/chosen": -467.2474365234375, + "logps/rejected": -530.8262939453125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.27678483724594116, + "rewards/margins": 44.17549133300781, + "rewards/rejected": -44.45227813720703, + "step": 315 + }, + { + "epoch": 0.34, + "learning_rate": 3.103448275862069e-05, + "logits/chosen": -0.19090279936790466, + "logits/rejected": -0.876602053642273, + "logps/chosen": -315.1255187988281, + "logps/rejected": -588.2965698242188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.501945495605469, + "rewards/margins": 45.68769073486328, + "rewards/rejected": -51.18963623046875, + "step": 316 + }, + { + "epoch": 0.35, + "learning_rate": 3.098275862068965e-05, + "logits/chosen": -0.10631687939167023, + "logits/rejected": -1.1011039018630981, + "logps/chosen": -434.1145935058594, + "logps/rejected": -521.3406982421875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.0800139904022217, + "rewards/margins": 44.002647399902344, + "rewards/rejected": -45.08266067504883, + "step": 317 + }, + { + "epoch": 0.35, + "learning_rate": 3.093103448275862e-05, + "logits/chosen": -0.03321276232600212, + "logits/rejected": -1.3932230472564697, + "logps/chosen": -575.6842651367188, + "logps/rejected": -527.6025390625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.441707968711853, + "rewards/margins": 43.618465423583984, + "rewards/rejected": -43.176753997802734, + "step": 318 + }, + { + "epoch": 0.35, + "learning_rate": 3.087931034482759e-05, + "logits/chosen": -0.139996737241745, + "logits/rejected": -1.3197453022003174, + "logps/chosen": -504.41864013671875, + "logps/rejected": -563.311767578125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.6999825835227966, + "rewards/margins": 47.611934661865234, + "rewards/rejected": -48.31192398071289, + "step": 319 + }, + { + "epoch": 0.35, + "learning_rate": 3.082758620689655e-05, + "logits/chosen": -0.126600980758667, + "logits/rejected": -1.053550362586975, + "logps/chosen": -388.8297119140625, + "logps/rejected": -544.5347290039062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.8857545852661133, + "rewards/margins": 43.31074523925781, + "rewards/rejected": -46.196502685546875, + "step": 320 + }, + { + "epoch": 0.35, + "learning_rate": 3.077586206896552e-05, + "logits/chosen": -0.3051377832889557, + "logits/rejected": -1.2079377174377441, + "logps/chosen": -376.6958923339844, + "logps/rejected": -553.730224609375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.7709710597991943, + "rewards/margins": 44.744571685791016, + "rewards/rejected": -46.515541076660156, + "step": 321 + }, + { + "epoch": 0.35, + "learning_rate": 3.0724137931034486e-05, + "logits/chosen": -0.18020763993263245, + "logits/rejected": -1.040939211845398, + "logps/chosen": -332.655029296875, + "logps/rejected": -551.4888916015625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.484922409057617, + "rewards/margins": 42.93204879760742, + "rewards/rejected": -47.41697311401367, + "step": 322 + }, + { + "epoch": 0.35, + "learning_rate": 3.067241379310345e-05, + "logits/chosen": -0.2714073061943054, + "logits/rejected": -1.1973769664764404, + "logps/chosen": -468.3298645019531, + "logps/rejected": -537.9442749023438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.890547752380371, + "rewards/margins": 40.39543151855469, + "rewards/rejected": -46.285980224609375, + "step": 323 + }, + { + "epoch": 0.35, + "learning_rate": 3.0620689655172415e-05, + "logits/chosen": -0.055039308965206146, + "logits/rejected": -0.961100697517395, + "logps/chosen": -479.1692810058594, + "logps/rejected": -544.91259765625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.017467498779297, + "rewards/margins": 37.9600944519043, + "rewards/rejected": -42.977561950683594, + "step": 324 + }, + { + "epoch": 0.35, + "learning_rate": 3.056896551724138e-05, + "logits/chosen": -0.09231775254011154, + "logits/rejected": -0.9333654046058655, + "logps/chosen": -383.6116943359375, + "logps/rejected": -508.18865966796875, + "loss": 0.0358, + "rewards/accuracies": 1.0, + "rewards/chosen": -7.502808094024658, + "rewards/margins": 34.478755950927734, + "rewards/rejected": -41.981563568115234, + "step": 325 + }, + { + "epoch": 0.36, + "learning_rate": 3.0517241379310344e-05, + "logits/chosen": -0.15142463147640228, + "logits/rejected": -0.8942531943321228, + "logps/chosen": -362.30755615234375, + "logps/rejected": -582.6021728515625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.6099252700805664, + "rewards/margins": 42.51545715332031, + "rewards/rejected": -46.12538528442383, + "step": 326 + }, + { + "epoch": 0.36, + "learning_rate": 3.0465517241379312e-05, + "logits/chosen": -0.0636628121137619, + "logits/rejected": -1.1498801708221436, + "logps/chosen": -452.0404052734375, + "logps/rejected": -537.0067138671875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.492896556854248, + "rewards/margins": 41.51354217529297, + "rewards/rejected": -44.00643539428711, + "step": 327 + }, + { + "epoch": 0.36, + "learning_rate": 3.0413793103448277e-05, + "logits/chosen": -0.3045711815357208, + "logits/rejected": -1.0503120422363281, + "logps/chosen": -339.1358642578125, + "logps/rejected": -538.6072998046875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.034678369760513306, + "rewards/margins": 42.621681213378906, + "rewards/rejected": -42.65635681152344, + "step": 328 + }, + { + "epoch": 0.36, + "learning_rate": 3.0362068965517245e-05, + "logits/chosen": -0.175112783908844, + "logits/rejected": -1.0160049200057983, + "logps/chosen": -369.64373779296875, + "logps/rejected": -531.3986206054688, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.621337890625, + "rewards/margins": 38.97738265991211, + "rewards/rejected": -44.598716735839844, + "step": 329 + }, + { + "epoch": 0.36, + "learning_rate": 3.0310344827586206e-05, + "logits/chosen": -0.0202659722417593, + "logits/rejected": -1.1052290201187134, + "logps/chosen": -492.6905822753906, + "logps/rejected": -529.3641357421875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.384145259857178, + "rewards/margins": 37.67775344848633, + "rewards/rejected": -42.06189727783203, + "step": 330 + }, + { + "epoch": 0.36, + "learning_rate": 3.0258620689655174e-05, + "logits/chosen": -0.16101710498332977, + "logits/rejected": -1.336228370666504, + "logps/chosen": -614.8682250976562, + "logps/rejected": -522.0177612304688, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.8445982933044434, + "rewards/margins": 38.63414764404297, + "rewards/rejected": -42.47874450683594, + "step": 331 + }, + { + "epoch": 0.36, + "learning_rate": 3.0206896551724142e-05, + "logits/chosen": -0.09683067351579666, + "logits/rejected": -0.8590956926345825, + "logps/chosen": -291.6666564941406, + "logps/rejected": -464.15509033203125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.470726728439331, + "rewards/margins": 34.38860321044922, + "rewards/rejected": -37.85932922363281, + "step": 332 + }, + { + "epoch": 0.36, + "learning_rate": 3.0155172413793104e-05, + "logits/chosen": -0.125202476978302, + "logits/rejected": -1.1684645414352417, + "logps/chosen": -504.86669921875, + "logps/rejected": -515.8795776367188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.095594882965088, + "rewards/margins": 34.49415969848633, + "rewards/rejected": -39.589752197265625, + "step": 333 + }, + { + "epoch": 0.36, + "learning_rate": 3.010344827586207e-05, + "logits/chosen": -0.10065445303916931, + "logits/rejected": -1.05722177028656, + "logps/chosen": -409.2242736816406, + "logps/rejected": -541.786376953125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.751873254776001, + "rewards/margins": 41.32733917236328, + "rewards/rejected": -44.0792121887207, + "step": 334 + }, + { + "epoch": 0.37, + "learning_rate": 3.005172413793104e-05, + "logits/chosen": -0.25555816292762756, + "logits/rejected": -0.9541343450546265, + "logps/chosen": -296.4285888671875, + "logps/rejected": -459.89288330078125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.3731026649475098, + "rewards/margins": 32.92158889770508, + "rewards/rejected": -36.29469299316406, + "step": 335 + }, + { + "epoch": 0.37, + "learning_rate": 3e-05, + "logits/chosen": -0.21287637948989868, + "logits/rejected": -1.1649916172027588, + "logps/chosen": -414.73040771484375, + "logps/rejected": -463.68597412109375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.076710224151611, + "rewards/margins": 33.77467727661133, + "rewards/rejected": -37.851383209228516, + "step": 336 + }, + { + "epoch": 0.37, + "learning_rate": 2.994827586206897e-05, + "logits/chosen": -0.07227441668510437, + "logits/rejected": -1.0752284526824951, + "logps/chosen": -487.4039001464844, + "logps/rejected": -503.8778076171875, + "loss": 0.0008, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.989980697631836, + "rewards/margins": 36.903316497802734, + "rewards/rejected": -40.89329528808594, + "step": 337 + }, + { + "epoch": 0.37, + "learning_rate": 2.9896551724137933e-05, + "logits/chosen": -0.12217733263969421, + "logits/rejected": -0.9584940075874329, + "logps/chosen": -344.06573486328125, + "logps/rejected": -557.3684692382812, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.471970558166504, + "rewards/margins": 41.59339141845703, + "rewards/rejected": -46.06536102294922, + "step": 338 + }, + { + "epoch": 0.37, + "learning_rate": 2.9844827586206898e-05, + "logits/chosen": 0.02000473439693451, + "logits/rejected": -1.2773916721343994, + "logps/chosen": -457.06158447265625, + "logps/rejected": -545.4772338867188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6830273866653442, + "rewards/margins": 48.27865982055664, + "rewards/rejected": -47.59563064575195, + "step": 339 + }, + { + "epoch": 0.37, + "learning_rate": 2.9793103448275863e-05, + "logits/chosen": 0.009161004796624184, + "logits/rejected": -1.1769722700119019, + "logps/chosen": -514.1297607421875, + "logps/rejected": -524.6531982421875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.6481988430023193, + "rewards/margins": 40.810298919677734, + "rewards/rejected": -43.45849609375, + "step": 340 + }, + { + "epoch": 0.37, + "learning_rate": 2.974137931034483e-05, + "logits/chosen": -0.07262174040079117, + "logits/rejected": -1.0641191005706787, + "logps/chosen": -353.1182861328125, + "logps/rejected": -554.7969970703125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.7675700187683105, + "rewards/margins": 41.664825439453125, + "rewards/rejected": -45.432395935058594, + "step": 341 + }, + { + "epoch": 0.37, + "learning_rate": 2.9689655172413792e-05, + "logits/chosen": -0.1689426451921463, + "logits/rejected": -1.1683162450790405, + "logps/chosen": -441.01202392578125, + "logps/rejected": -631.0665893554688, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.520883083343506, + "rewards/margins": 47.718502044677734, + "rewards/rejected": -53.2393798828125, + "step": 342 + }, + { + "epoch": 0.37, + "learning_rate": 2.963793103448276e-05, + "logits/chosen": -0.02344764769077301, + "logits/rejected": -1.1594109535217285, + "logps/chosen": -453.4601135253906, + "logps/rejected": -522.5077514648438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.1243491172790527, + "rewards/margins": 41.7907600402832, + "rewards/rejected": -44.91510772705078, + "step": 343 + }, + { + "epoch": 0.38, + "learning_rate": 2.9586206896551728e-05, + "logits/chosen": -0.01729479804635048, + "logits/rejected": -1.094846248626709, + "logps/chosen": -427.6337890625, + "logps/rejected": -603.2669067382812, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.31561219692230225, + "rewards/margins": 49.68363952636719, + "rewards/rejected": -49.36802673339844, + "step": 344 + }, + { + "epoch": 0.38, + "learning_rate": 2.9534482758620692e-05, + "logits/chosen": -0.12101568281650543, + "logits/rejected": -1.2158620357513428, + "logps/chosen": -491.6488342285156, + "logps/rejected": -572.5497436523438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.2570481300354004, + "rewards/margins": 51.22979736328125, + "rewards/rejected": -47.972747802734375, + "step": 345 + }, + { + "epoch": 0.38, + "learning_rate": 2.9482758620689657e-05, + "logits/chosen": -0.009567921981215477, + "logits/rejected": -1.029674768447876, + "logps/chosen": -377.886962890625, + "logps/rejected": -559.27294921875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.400155544281006, + "rewards/margins": 41.55246353149414, + "rewards/rejected": -46.9526252746582, + "step": 346 + }, + { + "epoch": 0.38, + "learning_rate": 2.943103448275862e-05, + "logits/chosen": -0.005190672352910042, + "logits/rejected": -1.157238483428955, + "logps/chosen": -438.5839538574219, + "logps/rejected": -551.5087890625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.0165094137191772, + "rewards/margins": 45.29164505004883, + "rewards/rejected": -44.27513885498047, + "step": 347 + }, + { + "epoch": 0.38, + "learning_rate": 2.937931034482759e-05, + "logits/chosen": -0.10370245575904846, + "logits/rejected": -1.1599996089935303, + "logps/chosen": -553.7239379882812, + "logps/rejected": -591.9037475585938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.530036449432373, + "rewards/margins": 49.52385330200195, + "rewards/rejected": -46.99381637573242, + "step": 348 + }, + { + "epoch": 0.38, + "learning_rate": 2.932758620689655e-05, + "logits/chosen": 0.012315813452005386, + "logits/rejected": -1.1227484941482544, + "logps/chosen": -420.30572509765625, + "logps/rejected": -541.201171875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.01237416267395, + "rewards/margins": 42.71653747558594, + "rewards/rejected": -44.72890853881836, + "step": 349 + }, + { + "epoch": 0.38, + "learning_rate": 2.927586206896552e-05, + "logits/chosen": -0.19439992308616638, + "logits/rejected": -1.0316534042358398, + "logps/chosen": -284.17578125, + "logps/rejected": -527.9844970703125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -7.887413024902344, + "rewards/margins": 37.930118560791016, + "rewards/rejected": -45.817535400390625, + "step": 350 + }, + { + "epoch": 0.38, + "learning_rate": 2.9224137931034487e-05, + "logits/chosen": -0.06830094754695892, + "logits/rejected": -1.3009464740753174, + "logps/chosen": -437.2919006347656, + "logps/rejected": -561.503662109375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.028534173965454, + "rewards/margins": 46.798194885253906, + "rewards/rejected": -45.76966094970703, + "step": 351 + }, + { + "epoch": 0.38, + "learning_rate": 2.9172413793103448e-05, + "logits/chosen": 0.03125571832060814, + "logits/rejected": -1.259416937828064, + "logps/chosen": -436.17852783203125, + "logps/rejected": -551.797119140625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.8364152908325195, + "rewards/margins": 48.23833465576172, + "rewards/rejected": -47.401920318603516, + "step": 352 + }, + { + "epoch": 0.39, + "learning_rate": 2.9120689655172416e-05, + "logits/chosen": 0.0014629308134317398, + "logits/rejected": -1.458472490310669, + "logps/chosen": -554.6463623046875, + "logps/rejected": -531.5646362304688, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 5.006325721740723, + "rewards/margins": 50.37799072265625, + "rewards/rejected": -45.371665954589844, + "step": 353 + }, + { + "epoch": 0.39, + "learning_rate": 2.9068965517241384e-05, + "logits/chosen": -0.16050775349140167, + "logits/rejected": -0.8299065828323364, + "logps/chosen": -335.6762390136719, + "logps/rejected": -590.0164184570312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.606829643249512, + "rewards/margins": 43.23134994506836, + "rewards/rejected": -47.83818054199219, + "step": 354 + }, + { + "epoch": 0.39, + "learning_rate": 2.9017241379310345e-05, + "logits/chosen": -0.07219342142343521, + "logits/rejected": -0.9463921189308167, + "logps/chosen": -391.2060852050781, + "logps/rejected": -568.3679809570312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.065673828125, + "rewards/margins": 41.32851028442383, + "rewards/rejected": -43.39418411254883, + "step": 355 + }, + { + "epoch": 0.39, + "learning_rate": 2.8965517241379313e-05, + "logits/chosen": -0.04772268235683441, + "logits/rejected": -1.164974570274353, + "logps/chosen": -408.1875, + "logps/rejected": -565.4370727539062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.738417387008667, + "rewards/margins": 45.41836929321289, + "rewards/rejected": -49.15678787231445, + "step": 356 + }, + { + "epoch": 0.39, + "learning_rate": 2.8913793103448278e-05, + "logits/chosen": -0.026131443679332733, + "logits/rejected": -1.257594108581543, + "logps/chosen": -436.87200927734375, + "logps/rejected": -602.8794555664062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.374298095703125, + "rewards/margins": 48.57533645629883, + "rewards/rejected": -48.94963073730469, + "step": 357 + }, + { + "epoch": 0.39, + "learning_rate": 2.8862068965517243e-05, + "logits/chosen": 0.04897276684641838, + "logits/rejected": -1.1252198219299316, + "logps/chosen": -406.7869873046875, + "logps/rejected": -565.0485229492188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.1319797039031982, + "rewards/margins": 45.98715591430664, + "rewards/rejected": -48.119140625, + "step": 358 + }, + { + "epoch": 0.39, + "learning_rate": 2.8810344827586207e-05, + "logits/chosen": -0.05338538438081741, + "logits/rejected": -1.0763384103775024, + "logps/chosen": -355.9891662597656, + "logps/rejected": -621.0955200195312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.576101779937744, + "rewards/margins": 49.743812561035156, + "rewards/rejected": -52.319915771484375, + "step": 359 + }, + { + "epoch": 0.39, + "learning_rate": 2.8758620689655175e-05, + "logits/chosen": -0.18446527421474457, + "logits/rejected": -1.0962433815002441, + "logps/chosen": -393.23675537109375, + "logps/rejected": -607.602783203125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.4161076545715332, + "rewards/margins": 50.76775360107422, + "rewards/rejected": -51.183860778808594, + "step": 360 + }, + { + "epoch": 0.39, + "learning_rate": 2.8706896551724143e-05, + "logits/chosen": -0.12225188314914703, + "logits/rejected": -1.0412880182266235, + "logps/chosen": -375.52166748046875, + "logps/rejected": -589.2161865234375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.385780930519104, + "rewards/margins": 49.27238464355469, + "rewards/rejected": -49.65816879272461, + "step": 361 + }, + { + "epoch": 0.39, + "learning_rate": 2.8655172413793104e-05, + "logits/chosen": -0.20962819457054138, + "logits/rejected": -1.0904340744018555, + "logps/chosen": -395.50238037109375, + "logps/rejected": -568.0843505859375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.58205509185791, + "rewards/margins": 40.50846862792969, + "rewards/rejected": -47.09052276611328, + "step": 362 + }, + { + "epoch": 0.4, + "learning_rate": 2.8603448275862072e-05, + "logits/chosen": 0.003986477851867676, + "logits/rejected": -1.0085983276367188, + "logps/chosen": -438.2725830078125, + "logps/rejected": -592.7444458007812, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6302861571311951, + "rewards/margins": 51.214351654052734, + "rewards/rejected": -50.58406066894531, + "step": 363 + }, + { + "epoch": 0.4, + "learning_rate": 2.8551724137931037e-05, + "logits/chosen": -0.04182690009474754, + "logits/rejected": -1.2840299606323242, + "logps/chosen": -501.47357177734375, + "logps/rejected": -599.254638671875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.2677664756774902, + "rewards/margins": 47.65995407104492, + "rewards/rejected": -44.39219665527344, + "step": 364 + }, + { + "epoch": 0.4, + "learning_rate": 2.85e-05, + "logits/chosen": 0.07004736363887787, + "logits/rejected": -1.3011246919631958, + "logps/chosen": -511.8918762207031, + "logps/rejected": -555.033447265625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.474068641662598, + "rewards/margins": 51.881038665771484, + "rewards/rejected": -47.40697479248047, + "step": 365 + }, + { + "epoch": 0.4, + "learning_rate": 2.8448275862068966e-05, + "logits/chosen": -0.03457900136709213, + "logits/rejected": -1.2160018682479858, + "logps/chosen": -474.7510070800781, + "logps/rejected": -588.1536865234375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.557379722595215, + "rewards/margins": 54.3979606628418, + "rewards/rejected": -49.840576171875, + "step": 366 + }, + { + "epoch": 0.4, + "learning_rate": 2.8396551724137934e-05, + "logits/chosen": 0.02054227888584137, + "logits/rejected": -1.1687372922897339, + "logps/chosen": -407.9102478027344, + "logps/rejected": -534.5169067382812, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.0955076217651367, + "rewards/margins": 40.78862762451172, + "rewards/rejected": -39.693119049072266, + "step": 367 + }, + { + "epoch": 0.4, + "learning_rate": 2.8344827586206895e-05, + "logits/chosen": 0.07716276496648788, + "logits/rejected": -0.8440214395523071, + "logps/chosen": -330.560546875, + "logps/rejected": -596.030517578125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.727421283721924, + "rewards/margins": 46.38547897338867, + "rewards/rejected": -51.1129035949707, + "step": 368 + }, + { + "epoch": 0.4, + "learning_rate": 2.8293103448275863e-05, + "logits/chosen": 0.014307685196399689, + "logits/rejected": -1.3238158226013184, + "logps/chosen": -495.6667175292969, + "logps/rejected": -584.3401489257812, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.3049987554550171, + "rewards/margins": 48.27531051635742, + "rewards/rejected": -48.58030700683594, + "step": 369 + }, + { + "epoch": 0.4, + "learning_rate": 2.824137931034483e-05, + "logits/chosen": 0.07619799673557281, + "logits/rejected": -1.2337863445281982, + "logps/chosen": -568.0042724609375, + "logps/rejected": -586.55908203125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.8983986377716064, + "rewards/margins": 50.09109878540039, + "rewards/rejected": -49.19269561767578, + "step": 370 + }, + { + "epoch": 0.4, + "learning_rate": 2.8189655172413793e-05, + "logits/chosen": 0.030935000628232956, + "logits/rejected": -1.1124335527420044, + "logps/chosen": -389.4969177246094, + "logps/rejected": -559.1497802734375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.30449792742729187, + "rewards/margins": 44.675933837890625, + "rewards/rejected": -44.98043441772461, + "step": 371 + }, + { + "epoch": 0.41, + "learning_rate": 2.813793103448276e-05, + "logits/chosen": -0.09724840521812439, + "logits/rejected": -1.4216426610946655, + "logps/chosen": -487.3270568847656, + "logps/rejected": -557.8433227539062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.7180730104446411, + "rewards/margins": 47.800254821777344, + "rewards/rejected": -47.082183837890625, + "step": 372 + }, + { + "epoch": 0.41, + "learning_rate": 2.808620689655173e-05, + "logits/chosen": 0.06473588198423386, + "logits/rejected": -1.2265748977661133, + "logps/chosen": -457.49407958984375, + "logps/rejected": -584.40966796875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.494109630584717, + "rewards/margins": 47.470645904541016, + "rewards/rejected": -49.964752197265625, + "step": 373 + }, + { + "epoch": 0.41, + "learning_rate": 2.803448275862069e-05, + "logits/chosen": 0.09420200437307358, + "logits/rejected": -1.1941781044006348, + "logps/chosen": -537.9132080078125, + "logps/rejected": -560.3892211914062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.515617609024048, + "rewards/margins": 48.64773178100586, + "rewards/rejected": -46.132110595703125, + "step": 374 + }, + { + "epoch": 0.41, + "learning_rate": 2.7982758620689658e-05, + "logits/chosen": -0.2046179175376892, + "logits/rejected": -1.214766025543213, + "logps/chosen": -473.283447265625, + "logps/rejected": -613.4475708007812, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 5.620456218719482, + "rewards/margins": 57.01899719238281, + "rewards/rejected": -51.39854431152344, + "step": 375 + }, + { + "epoch": 0.41, + "learning_rate": 2.7931034482758622e-05, + "logits/chosen": -0.0699472650885582, + "logits/rejected": -1.3432188034057617, + "logps/chosen": -434.32293701171875, + "logps/rejected": -562.9783325195312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.728281021118164, + "rewards/margins": 44.719356536865234, + "rewards/rejected": -49.447635650634766, + "step": 376 + }, + { + "epoch": 0.41, + "learning_rate": 2.787931034482759e-05, + "logits/chosen": -0.01713864505290985, + "logits/rejected": -0.9077965021133423, + "logps/chosen": -345.4331970214844, + "logps/rejected": -560.2376098632812, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.396304607391357, + "rewards/margins": 39.71217727661133, + "rewards/rejected": -44.108482360839844, + "step": 377 + }, + { + "epoch": 0.41, + "learning_rate": 2.782758620689655e-05, + "logits/chosen": 0.001452295109629631, + "logits/rejected": -1.2449791431427002, + "logps/chosen": -455.90850830078125, + "logps/rejected": -564.8043212890625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.484553575515747, + "rewards/margins": 45.819183349609375, + "rewards/rejected": -48.303741455078125, + "step": 378 + }, + { + "epoch": 0.41, + "learning_rate": 2.777586206896552e-05, + "logits/chosen": 0.04156869277358055, + "logits/rejected": -1.0869159698486328, + "logps/chosen": -431.6122741699219, + "logps/rejected": -553.697021484375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3313959836959839, + "rewards/margins": 44.75462341308594, + "rewards/rejected": -44.42322540283203, + "step": 379 + }, + { + "epoch": 0.41, + "learning_rate": 2.7724137931034488e-05, + "logits/chosen": 0.0004380419850349426, + "logits/rejected": -1.0738462209701538, + "logps/chosen": -392.25018310546875, + "logps/rejected": -583.8248901367188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.7267297506332397, + "rewards/margins": 48.91001892089844, + "rewards/rejected": -50.63674545288086, + "step": 380 + }, + { + "epoch": 0.42, + "learning_rate": 2.767241379310345e-05, + "logits/chosen": -0.05203678458929062, + "logits/rejected": -0.9276233315467834, + "logps/chosen": -341.73779296875, + "logps/rejected": -574.5068359375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.9113528728485107, + "rewards/margins": 48.13732147216797, + "rewards/rejected": -51.048675537109375, + "step": 381 + }, + { + "epoch": 0.42, + "learning_rate": 2.7620689655172417e-05, + "logits/chosen": 0.12498117983341217, + "logits/rejected": -1.2847096920013428, + "logps/chosen": -471.0482177734375, + "logps/rejected": -578.107177734375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.4137356281280518, + "rewards/margins": 51.37191390991211, + "rewards/rejected": -48.95817565917969, + "step": 382 + }, + { + "epoch": 0.42, + "learning_rate": 2.756896551724138e-05, + "logits/chosen": -0.15520095825195312, + "logits/rejected": -1.0619099140167236, + "logps/chosen": -404.9841613769531, + "logps/rejected": -578.9855346679688, + "loss": 0.0194, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.767134666442871, + "rewards/margins": 47.32525634765625, + "rewards/rejected": -51.09239196777344, + "step": 383 + }, + { + "epoch": 0.42, + "learning_rate": 2.7517241379310346e-05, + "logits/chosen": 0.04274128004908562, + "logits/rejected": -1.1932741403579712, + "logps/chosen": -418.20404052734375, + "logps/rejected": -512.047607421875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.06832236051559448, + "rewards/margins": 44.48112487792969, + "rewards/rejected": -44.412803649902344, + "step": 384 + }, + { + "epoch": 0.42, + "learning_rate": 2.746551724137931e-05, + "logits/chosen": 0.01695055142045021, + "logits/rejected": -1.1045211553573608, + "logps/chosen": -367.1639404296875, + "logps/rejected": -537.3077392578125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1754894256591797, + "rewards/margins": 43.70709991455078, + "rewards/rejected": -43.882591247558594, + "step": 385 + }, + { + "epoch": 0.42, + "learning_rate": 2.741379310344828e-05, + "logits/chosen": 0.12017740309238434, + "logits/rejected": -1.0955898761749268, + "logps/chosen": -463.5010986328125, + "logps/rejected": -543.324462890625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.6687986850738525, + "rewards/margins": 46.83396911621094, + "rewards/rejected": -44.16516876220703, + "step": 386 + }, + { + "epoch": 0.42, + "learning_rate": 2.736206896551724e-05, + "logits/chosen": -0.07626432925462723, + "logits/rejected": -1.0542856454849243, + "logps/chosen": -368.75933837890625, + "logps/rejected": -546.5941772460938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.629738450050354, + "rewards/margins": 47.353370666503906, + "rewards/rejected": -46.7236328125, + "step": 387 + }, + { + "epoch": 0.42, + "learning_rate": 2.7310344827586208e-05, + "logits/chosen": -0.039779115468263626, + "logits/rejected": -1.3472245931625366, + "logps/chosen": -466.123779296875, + "logps/rejected": -488.5738525390625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2881481647491455, + "rewards/margins": 40.72649383544922, + "rewards/rejected": -40.4383430480957, + "step": 388 + }, + { + "epoch": 0.42, + "learning_rate": 2.7258620689655176e-05, + "logits/chosen": -0.033790040761232376, + "logits/rejected": -1.141743779182434, + "logps/chosen": -403.8807067871094, + "logps/rejected": -495.78570556640625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.32837265729904175, + "rewards/margins": 38.956748962402344, + "rewards/rejected": -39.285118103027344, + "step": 389 + }, + { + "epoch": 0.43, + "learning_rate": 2.7206896551724137e-05, + "logits/chosen": -0.015193854458630085, + "logits/rejected": -1.3545093536376953, + "logps/chosen": -489.96087646484375, + "logps/rejected": -515.5687866210938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 6.122744560241699, + "rewards/margins": 52.13509750366211, + "rewards/rejected": -46.012351989746094, + "step": 390 + }, + { + "epoch": 0.43, + "learning_rate": 2.7155172413793105e-05, + "logits/chosen": 0.02489817887544632, + "logits/rejected": -0.991384744644165, + "logps/chosen": -390.23370361328125, + "logps/rejected": -568.0363159179688, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.53804612159729, + "rewards/margins": 42.114593505859375, + "rewards/rejected": -42.652645111083984, + "step": 391 + }, + { + "epoch": 0.43, + "learning_rate": 2.7103448275862073e-05, + "logits/chosen": -0.15990671515464783, + "logits/rejected": -0.9720957279205322, + "logps/chosen": -299.734619140625, + "logps/rejected": -480.4046630859375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.4684109687805176, + "rewards/margins": 39.979103088378906, + "rewards/rejected": -40.447513580322266, + "step": 392 + }, + { + "epoch": 0.43, + "learning_rate": 2.7051724137931038e-05, + "logits/chosen": -0.03375757485628128, + "logits/rejected": -1.021787166595459, + "logps/chosen": -322.0400390625, + "logps/rejected": -487.57537841796875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.355576753616333, + "rewards/margins": 44.75249481201172, + "rewards/rejected": -42.39692306518555, + "step": 393 + }, + { + "epoch": 0.43, + "learning_rate": 2.7000000000000002e-05, + "logits/chosen": -0.020571894943714142, + "logits/rejected": -1.2173579931259155, + "logps/chosen": -417.3011169433594, + "logps/rejected": -449.94512939453125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.091635227203369, + "rewards/margins": 37.5725212097168, + "rewards/rejected": -34.48088455200195, + "step": 394 + }, + { + "epoch": 0.43, + "learning_rate": 2.6948275862068967e-05, + "logits/chosen": 0.026321958750486374, + "logits/rejected": -0.9667563438415527, + "logps/chosen": -341.38720703125, + "logps/rejected": -466.8248291015625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.7460142374038696, + "rewards/margins": 40.34065628051758, + "rewards/rejected": -38.594642639160156, + "step": 395 + }, + { + "epoch": 0.43, + "learning_rate": 2.6896551724137935e-05, + "logits/chosen": -0.08959555625915527, + "logits/rejected": -1.1910420656204224, + "logps/chosen": -381.2998962402344, + "logps/rejected": -472.2957763671875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.0525035858154297, + "rewards/margins": 42.368186950683594, + "rewards/rejected": -40.31568145751953, + "step": 396 + }, + { + "epoch": 0.43, + "learning_rate": 2.6844827586206896e-05, + "logits/chosen": -0.17987710237503052, + "logits/rejected": -1.02032470703125, + "logps/chosen": -298.4346923828125, + "logps/rejected": -471.966552734375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.6901694536209106, + "rewards/margins": 35.03578567504883, + "rewards/rejected": -36.725955963134766, + "step": 397 + }, + { + "epoch": 0.43, + "learning_rate": 2.6793103448275864e-05, + "logits/chosen": -0.20967192947864532, + "logits/rejected": -0.9511845707893372, + "logps/chosen": -299.4791564941406, + "logps/rejected": -455.33782958984375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.385829448699951, + "rewards/margins": 37.45805358886719, + "rewards/rejected": -35.07223129272461, + "step": 398 + }, + { + "epoch": 0.44, + "learning_rate": 2.6741379310344832e-05, + "logits/chosen": -0.01965043693780899, + "logits/rejected": -1.0445890426635742, + "logps/chosen": -275.1196594238281, + "logps/rejected": -483.07647705078125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.6957775354385376, + "rewards/margins": 38.007179260253906, + "rewards/rejected": -39.70295715332031, + "step": 399 + }, + { + "epoch": 0.44, + "learning_rate": 2.6689655172413793e-05, + "logits/chosen": -0.098607636988163, + "logits/rejected": -1.0000914335250854, + "logps/chosen": -346.13018798828125, + "logps/rejected": -495.4217224121094, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.0000643730163574, + "rewards/margins": 42.866764068603516, + "rewards/rejected": -39.866703033447266, + "step": 400 + }, + { + "epoch": 0.44, + "learning_rate": 2.663793103448276e-05, + "logits/chosen": 0.1724672019481659, + "logits/rejected": -1.2024016380310059, + "logps/chosen": -505.9283752441406, + "logps/rejected": -510.1684875488281, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.5972766876220703, + "rewards/margins": 42.12846755981445, + "rewards/rejected": -40.531192779541016, + "step": 401 + }, + { + "epoch": 0.44, + "learning_rate": 2.6586206896551726e-05, + "logits/chosen": -0.1967574805021286, + "logits/rejected": -1.045470952987671, + "logps/chosen": -335.9580993652344, + "logps/rejected": -520.8634643554688, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14454174041748047, + "rewards/margins": 41.4933967590332, + "rewards/rejected": -41.637939453125, + "step": 402 + }, + { + "epoch": 0.44, + "learning_rate": 2.653448275862069e-05, + "logits/chosen": 0.11029797792434692, + "logits/rejected": -1.4310297966003418, + "logps/chosen": -571.0155029296875, + "logps/rejected": -496.65087890625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 9.46963119506836, + "rewards/margins": 48.58985900878906, + "rewards/rejected": -39.12022399902344, + "step": 403 + }, + { + "epoch": 0.44, + "learning_rate": 2.6482758620689655e-05, + "logits/chosen": 0.030145570635795593, + "logits/rejected": -1.0499293804168701, + "logps/chosen": -399.5472717285156, + "logps/rejected": -472.23272705078125, + "loss": 0.0001, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.7181732654571533, + "rewards/margins": 40.335968017578125, + "rewards/rejected": -37.617794036865234, + "step": 404 + }, + { + "epoch": 0.44, + "learning_rate": 2.6431034482758623e-05, + "logits/chosen": -0.17437051236629486, + "logits/rejected": -0.8545927405357361, + "logps/chosen": -317.4663391113281, + "logps/rejected": -524.6553955078125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.37684276700019836, + "rewards/margins": 39.592159271240234, + "rewards/rejected": -39.96900177001953, + "step": 405 + }, + { + "epoch": 0.44, + "learning_rate": 2.6379310344827584e-05, + "logits/chosen": -0.2994159460067749, + "logits/rejected": -1.063322901725769, + "logps/chosen": -286.9337158203125, + "logps/rejected": -480.55389404296875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.5861432552337646, + "rewards/margins": 36.732574462890625, + "rewards/rejected": -37.318721771240234, + "step": 406 + }, + { + "epoch": 0.44, + "learning_rate": 2.6327586206896552e-05, + "logits/chosen": -0.1211957260966301, + "logits/rejected": -0.9387801885604858, + "logps/chosen": -288.6614990234375, + "logps/rejected": -440.61798095703125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6156797409057617, + "rewards/margins": 36.306339263916016, + "rewards/rejected": -35.6906623840332, + "step": 407 + }, + { + "epoch": 0.45, + "learning_rate": 2.627586206896552e-05, + "logits/chosen": -0.1856449842453003, + "logits/rejected": -0.8845812082290649, + "logps/chosen": -263.8268127441406, + "logps/rejected": -498.94940185546875, + "loss": 0.0006, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.013650417327881, + "rewards/margins": 37.46383285522461, + "rewards/rejected": -41.47748565673828, + "step": 408 + }, + { + "epoch": 0.45, + "learning_rate": 2.622413793103449e-05, + "logits/chosen": 0.009420637041330338, + "logits/rejected": -1.0488317012786865, + "logps/chosen": -388.43743896484375, + "logps/rejected": -487.9730224609375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5363948345184326, + "rewards/margins": 41.2370719909668, + "rewards/rejected": -40.700679779052734, + "step": 409 + }, + { + "epoch": 0.45, + "learning_rate": 2.617241379310345e-05, + "logits/chosen": -0.1033945381641388, + "logits/rejected": -1.3271011114120483, + "logps/chosen": -510.8887939453125, + "logps/rejected": -464.1139221191406, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.8576178550720215, + "rewards/margins": 45.34534454345703, + "rewards/rejected": -37.487728118896484, + "step": 410 + }, + { + "epoch": 0.45, + "learning_rate": 2.6120689655172418e-05, + "logits/chosen": 0.012298082932829857, + "logits/rejected": -1.1230072975158691, + "logps/chosen": -376.9397277832031, + "logps/rejected": -465.98681640625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2561565339565277, + "rewards/margins": 37.64347839355469, + "rewards/rejected": -37.387325286865234, + "step": 411 + }, + { + "epoch": 0.45, + "learning_rate": 2.6068965517241382e-05, + "logits/chosen": -0.0367039330303669, + "logits/rejected": -1.370044469833374, + "logps/chosen": -433.99237060546875, + "logps/rejected": -526.426513671875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.9706013202667236, + "rewards/margins": 47.25657272338867, + "rewards/rejected": -45.285972595214844, + "step": 412 + }, + { + "epoch": 0.45, + "learning_rate": 2.6017241379310344e-05, + "logits/chosen": 0.05295756831765175, + "logits/rejected": -1.0558881759643555, + "logps/chosen": -333.7112731933594, + "logps/rejected": -486.7742614746094, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.21756157279014587, + "rewards/margins": 43.01340103149414, + "rewards/rejected": -42.79584503173828, + "step": 413 + }, + { + "epoch": 0.45, + "learning_rate": 2.596551724137931e-05, + "logits/chosen": -0.16404397785663605, + "logits/rejected": -1.3011796474456787, + "logps/chosen": -487.19012451171875, + "logps/rejected": -492.16259765625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 5.502512454986572, + "rewards/margins": 45.011375427246094, + "rewards/rejected": -39.50886535644531, + "step": 414 + }, + { + "epoch": 0.45, + "learning_rate": 2.591379310344828e-05, + "logits/chosen": -0.03245847672224045, + "logits/rejected": -1.338456153869629, + "logps/chosen": -393.41278076171875, + "logps/rejected": -507.5992431640625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.42661714553833, + "rewards/margins": 46.058433532714844, + "rewards/rejected": -43.631813049316406, + "step": 415 + }, + { + "epoch": 0.45, + "learning_rate": 2.586206896551724e-05, + "logits/chosen": -0.17630226910114288, + "logits/rejected": -1.2694957256317139, + "logps/chosen": -406.9403076171875, + "logps/rejected": -529.5288696289062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.6201605796813965, + "rewards/margins": 45.6422004699707, + "rewards/rejected": -43.022037506103516, + "step": 416 + }, + { + "epoch": 0.45, + "learning_rate": 2.581034482758621e-05, + "logits/chosen": 0.04229099303483963, + "logits/rejected": -1.3653508424758911, + "logps/chosen": -456.75592041015625, + "logps/rejected": -489.34307861328125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 5.883845806121826, + "rewards/margins": 45.551551818847656, + "rewards/rejected": -39.66770553588867, + "step": 417 + }, + { + "epoch": 0.46, + "learning_rate": 2.5758620689655177e-05, + "logits/chosen": -0.07038390636444092, + "logits/rejected": -1.2386696338653564, + "logps/chosen": -437.032958984375, + "logps/rejected": -516.738037109375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5063050985336304, + "rewards/margins": 42.439186096191406, + "rewards/rejected": -41.93287658691406, + "step": 418 + }, + { + "epoch": 0.46, + "learning_rate": 2.5706896551724138e-05, + "logits/chosen": 0.04438025876879692, + "logits/rejected": -1.2710415124893188, + "logps/chosen": -419.35028076171875, + "logps/rejected": -500.75164794921875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6055812239646912, + "rewards/margins": 43.148094177246094, + "rewards/rejected": -42.542510986328125, + "step": 419 + }, + { + "epoch": 0.46, + "learning_rate": 2.5655172413793106e-05, + "logits/chosen": -0.14012601971626282, + "logits/rejected": -0.9432665109634399, + "logps/chosen": -322.7103271484375, + "logps/rejected": -559.9931030273438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.232179045677185, + "rewards/margins": 46.507049560546875, + "rewards/rejected": -47.739227294921875, + "step": 420 + }, + { + "epoch": 0.46, + "learning_rate": 2.560344827586207e-05, + "logits/chosen": -0.13305935263633728, + "logits/rejected": -1.078338384628296, + "logps/chosen": -298.7225646972656, + "logps/rejected": -552.3433837890625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11677652597427368, + "rewards/margins": 48.314998626708984, + "rewards/rejected": -48.43178176879883, + "step": 421 + }, + { + "epoch": 0.46, + "learning_rate": 2.5551724137931035e-05, + "logits/chosen": -0.09384839236736298, + "logits/rejected": -1.350205898284912, + "logps/chosen": -406.8752746582031, + "logps/rejected": -576.1410522460938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.6324269771575928, + "rewards/margins": 51.2014274597168, + "rewards/rejected": -47.569000244140625, + "step": 422 + }, + { + "epoch": 0.46, + "learning_rate": 2.55e-05, + "logits/chosen": -0.14802265167236328, + "logits/rejected": -1.068876028060913, + "logps/chosen": -297.4754333496094, + "logps/rejected": -581.18994140625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11329731345176697, + "rewards/margins": 49.50238037109375, + "rewards/rejected": -49.61567687988281, + "step": 423 + }, + { + "epoch": 0.46, + "learning_rate": 2.5448275862068968e-05, + "logits/chosen": 0.1324731409549713, + "logits/rejected": -1.66242253780365, + "logps/chosen": -557.9464721679688, + "logps/rejected": -542.5173950195312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 10.66864013671875, + "rewards/margins": 57.00450897216797, + "rewards/rejected": -46.33586883544922, + "step": 424 + }, + { + "epoch": 0.46, + "learning_rate": 2.5396551724137936e-05, + "logits/chosen": -0.14472703635692596, + "logits/rejected": -1.0066394805908203, + "logps/chosen": -287.56622314453125, + "logps/rejected": -527.8837280273438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.7066973447799683, + "rewards/margins": 41.968414306640625, + "rewards/rejected": -42.67510986328125, + "step": 425 + }, + { + "epoch": 0.46, + "learning_rate": 2.5344827586206897e-05, + "logits/chosen": -0.1694573163986206, + "logits/rejected": -1.0617188215255737, + "logps/chosen": -359.6497497558594, + "logps/rejected": -551.6131591796875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.4302568435668945, + "rewards/margins": 48.28759002685547, + "rewards/rejected": -46.857330322265625, + "step": 426 + }, + { + "epoch": 0.47, + "learning_rate": 2.5293103448275865e-05, + "logits/chosen": -0.12637755274772644, + "logits/rejected": -0.9839163422584534, + "logps/chosen": -312.71148681640625, + "logps/rejected": -523.2889404296875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07784199714660645, + "rewards/margins": 42.848533630371094, + "rewards/rejected": -42.92637634277344, + "step": 427 + }, + { + "epoch": 0.47, + "learning_rate": 2.5241379310344833e-05, + "logits/chosen": -0.15449154376983643, + "logits/rejected": -1.0122627019882202, + "logps/chosen": -364.6027526855469, + "logps/rejected": -519.9754638671875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.042532920837402, + "rewards/margins": 44.28017807006836, + "rewards/rejected": -40.237648010253906, + "step": 428 + }, + { + "epoch": 0.47, + "learning_rate": 2.5189655172413794e-05, + "logits/chosen": -0.015961170196533203, + "logits/rejected": -0.9020889401435852, + "logps/chosen": -385.604736328125, + "logps/rejected": -582.2443237304688, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09042125940322876, + "rewards/margins": 45.73094177246094, + "rewards/rejected": -45.82136535644531, + "step": 429 + }, + { + "epoch": 0.47, + "learning_rate": 2.513793103448276e-05, + "logits/chosen": -0.08048711717128754, + "logits/rejected": -1.1989467144012451, + "logps/chosen": -406.6160888671875, + "logps/rejected": -526.4555053710938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.1125061511993408, + "rewards/margins": 44.74122619628906, + "rewards/rejected": -45.85373306274414, + "step": 430 + }, + { + "epoch": 0.47, + "learning_rate": 2.5086206896551727e-05, + "logits/chosen": -0.07307261973619461, + "logits/rejected": -1.1985551118850708, + "logps/chosen": -395.5963439941406, + "logps/rejected": -579.1031494140625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.7303085327148438, + "rewards/margins": 49.54568862915039, + "rewards/rejected": -48.81537628173828, + "step": 431 + }, + { + "epoch": 0.47, + "learning_rate": 2.5034482758620688e-05, + "logits/chosen": 0.11793918162584305, + "logits/rejected": -1.4616129398345947, + "logps/chosen": -433.7359619140625, + "logps/rejected": -542.4663696289062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.1155309677124023, + "rewards/margins": 48.859405517578125, + "rewards/rejected": -45.743873596191406, + "step": 432 + }, + { + "epoch": 0.47, + "learning_rate": 2.4982758620689656e-05, + "logits/chosen": -0.06499344110488892, + "logits/rejected": -1.393472671508789, + "logps/chosen": -494.056640625, + "logps/rejected": -539.1574096679688, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.363572120666504, + "rewards/margins": 50.28820037841797, + "rewards/rejected": -45.92462921142578, + "step": 433 + }, + { + "epoch": 0.47, + "learning_rate": 2.4931034482758624e-05, + "logits/chosen": -0.18045151233673096, + "logits/rejected": -0.9066936373710632, + "logps/chosen": -269.164794921875, + "logps/rejected": -516.2396850585938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.3069080114364624, + "rewards/margins": 39.70387649536133, + "rewards/rejected": -41.010780334472656, + "step": 434 + }, + { + "epoch": 0.47, + "learning_rate": 2.4879310344827585e-05, + "logits/chosen": -0.16510559618473053, + "logits/rejected": -0.9158352017402649, + "logps/chosen": -342.11920166015625, + "logps/rejected": -594.5755615234375, + "loss": 0.0037, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.06957662105560303, + "rewards/margins": 44.563594818115234, + "rewards/rejected": -44.494014739990234, + "step": 435 + }, + { + "epoch": 0.48, + "learning_rate": 2.4827586206896553e-05, + "logits/chosen": -0.08779498934745789, + "logits/rejected": -0.9681074023246765, + "logps/chosen": -233.8495330810547, + "logps/rejected": -577.7623291015625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.8844499588012695, + "rewards/margins": 47.862998962402344, + "rewards/rejected": -48.7474479675293, + "step": 436 + }, + { + "epoch": 0.48, + "learning_rate": 2.477586206896552e-05, + "logits/chosen": 0.04494841396808624, + "logits/rejected": -1.3290835618972778, + "logps/chosen": -458.4309387207031, + "logps/rejected": -547.5213012695312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.0667998790740967, + "rewards/margins": 49.19175338745117, + "rewards/rejected": -46.12495422363281, + "step": 437 + }, + { + "epoch": 0.48, + "learning_rate": 2.4724137931034483e-05, + "logits/chosen": -0.07239006459712982, + "logits/rejected": -1.1752184629440308, + "logps/chosen": -362.7398681640625, + "logps/rejected": -527.936279296875, + "loss": 0.0051, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.697190523147583, + "rewards/margins": 42.5650634765625, + "rewards/rejected": -43.26225662231445, + "step": 438 + }, + { + "epoch": 0.48, + "learning_rate": 2.467241379310345e-05, + "logits/chosen": -0.11375527083873749, + "logits/rejected": -1.442746639251709, + "logps/chosen": -463.52410888671875, + "logps/rejected": -559.883056640625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.263465881347656, + "rewards/margins": 53.06332778930664, + "rewards/rejected": -48.79986572265625, + "step": 439 + }, + { + "epoch": 0.48, + "learning_rate": 2.4620689655172415e-05, + "logits/chosen": 0.02915153093636036, + "logits/rejected": -1.106976866722107, + "logps/chosen": -505.97552490234375, + "logps/rejected": -572.701171875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4833635687828064, + "rewards/margins": 50.768310546875, + "rewards/rejected": -50.28494644165039, + "step": 440 + }, + { + "epoch": 0.48, + "learning_rate": 2.4568965517241383e-05, + "logits/chosen": -0.012120015919208527, + "logits/rejected": -1.398685336112976, + "logps/chosen": -491.70361328125, + "logps/rejected": -593.99951171875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.6562215089797974, + "rewards/margins": 49.44664001464844, + "rewards/rejected": -51.10285949707031, + "step": 441 + }, + { + "epoch": 0.48, + "learning_rate": 2.4517241379310344e-05, + "logits/chosen": -0.02736329287290573, + "logits/rejected": -1.475520372390747, + "logps/chosen": -519.62255859375, + "logps/rejected": -649.6115112304688, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.501253604888916, + "rewards/margins": 54.74308395385742, + "rewards/rejected": -55.24433517456055, + "step": 442 + }, + { + "epoch": 0.48, + "learning_rate": 2.4465517241379312e-05, + "logits/chosen": -0.14074942469596863, + "logits/rejected": -1.3152858018875122, + "logps/chosen": -390.683349609375, + "logps/rejected": -604.4539794921875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.0559771060943604, + "rewards/margins": 50.112056732177734, + "rewards/rejected": -53.16802978515625, + "step": 443 + }, + { + "epoch": 0.48, + "learning_rate": 2.441379310344828e-05, + "logits/chosen": -0.11957734823226929, + "logits/rejected": -1.1977393627166748, + "logps/chosen": -309.5727844238281, + "logps/rejected": -558.4000244140625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.4718704223632812, + "rewards/margins": 47.193809509277344, + "rewards/rejected": -49.66568374633789, + "step": 444 + }, + { + "epoch": 0.49, + "learning_rate": 2.436206896551724e-05, + "logits/chosen": -0.027118626981973648, + "logits/rejected": -1.4130396842956543, + "logps/chosen": -512.9483642578125, + "logps/rejected": -661.2791748046875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.5184032917022705, + "rewards/margins": 58.53022766113281, + "rewards/rejected": -56.01182556152344, + "step": 445 + }, + { + "epoch": 0.49, + "learning_rate": 2.431034482758621e-05, + "logits/chosen": -0.10026545077562332, + "logits/rejected": -1.0776184797286987, + "logps/chosen": -390.74957275390625, + "logps/rejected": -613.8366088867188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.3963590860366821, + "rewards/margins": 51.305171966552734, + "rewards/rejected": -52.701534271240234, + "step": 446 + }, + { + "epoch": 0.49, + "learning_rate": 2.4258620689655174e-05, + "logits/chosen": -0.06135738641023636, + "logits/rejected": -1.1190871000289917, + "logps/chosen": -402.94940185546875, + "logps/rejected": -593.6253662109375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.729300022125244, + "rewards/margins": 46.670902252197266, + "rewards/rejected": -49.40019989013672, + "step": 447 + }, + { + "epoch": 0.49, + "learning_rate": 2.420689655172414e-05, + "logits/chosen": -0.08222100138664246, + "logits/rejected": -0.9519947171211243, + "logps/chosen": -333.4215087890625, + "logps/rejected": -581.1034545898438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.2264509201049805, + "rewards/margins": 45.486000061035156, + "rewards/rejected": -49.71245574951172, + "step": 448 + }, + { + "epoch": 0.49, + "learning_rate": 2.4155172413793103e-05, + "logits/chosen": -0.09325332939624786, + "logits/rejected": -1.1721282005310059, + "logps/chosen": -467.85150146484375, + "logps/rejected": -599.2661743164062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5591703057289124, + "rewards/margins": 47.47113037109375, + "rewards/rejected": -46.911964416503906, + "step": 449 + }, + { + "epoch": 0.49, + "learning_rate": 2.410344827586207e-05, + "logits/chosen": -0.1012139841914177, + "logits/rejected": -1.1209399700164795, + "logps/chosen": -377.45355224609375, + "logps/rejected": -592.563232421875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.7665712833404541, + "rewards/margins": 47.60783386230469, + "rewards/rejected": -48.37440872192383, + "step": 450 + }, + { + "epoch": 0.49, + "learning_rate": 2.4051724137931033e-05, + "logits/chosen": -0.25119173526763916, + "logits/rejected": -1.3089392185211182, + "logps/chosen": -395.1696472167969, + "logps/rejected": -575.6404418945312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.25265008211135864, + "rewards/margins": 50.09402847290039, + "rewards/rejected": -49.841373443603516, + "step": 451 + }, + { + "epoch": 0.49, + "learning_rate": 2.4e-05, + "logits/chosen": -0.06219256669282913, + "logits/rejected": -1.17987859249115, + "logps/chosen": -423.6275634765625, + "logps/rejected": -604.2877807617188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.2522249221801758, + "rewards/margins": 55.68733596801758, + "rewards/rejected": -54.43511199951172, + "step": 452 + }, + { + "epoch": 0.49, + "learning_rate": 2.394827586206897e-05, + "logits/chosen": -0.15801295638084412, + "logits/rejected": -1.2335947751998901, + "logps/chosen": -380.48291015625, + "logps/rejected": -573.8751831054688, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.369384765625, + "rewards/margins": 46.3884391784668, + "rewards/rejected": -48.7578239440918, + "step": 453 + }, + { + "epoch": 0.5, + "learning_rate": 2.389655172413793e-05, + "logits/chosen": -0.10853154957294464, + "logits/rejected": -1.2493653297424316, + "logps/chosen": -535.5428466796875, + "logps/rejected": -618.5908813476562, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.8845133781433105, + "rewards/margins": 54.12300491333008, + "rewards/rejected": -50.238494873046875, + "step": 454 + }, + { + "epoch": 0.5, + "learning_rate": 2.3844827586206898e-05, + "logits/chosen": -0.22083908319473267, + "logits/rejected": -1.021941900253296, + "logps/chosen": -373.7161865234375, + "logps/rejected": -669.4287719726562, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.2017593383789062, + "rewards/margins": 50.782039642333984, + "rewards/rejected": -52.98379898071289, + "step": 455 + }, + { + "epoch": 0.5, + "learning_rate": 2.3793103448275866e-05, + "logits/chosen": -0.004196269437670708, + "logits/rejected": -1.2873380184173584, + "logps/chosen": -507.622802734375, + "logps/rejected": -605.938232421875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.7006635665893555, + "rewards/margins": 50.75455093383789, + "rewards/rejected": -53.4552116394043, + "step": 456 + }, + { + "epoch": 0.5, + "learning_rate": 2.374137931034483e-05, + "logits/chosen": -0.06312916427850723, + "logits/rejected": -1.2262274026870728, + "logps/chosen": -396.0963134765625, + "logps/rejected": -557.1597290039062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.6157703399658203, + "rewards/margins": 45.1955680847168, + "rewards/rejected": -46.81133270263672, + "step": 457 + }, + { + "epoch": 0.5, + "learning_rate": 2.3689655172413795e-05, + "logits/chosen": 0.06958001852035522, + "logits/rejected": -1.2452892065048218, + "logps/chosen": -524.7869873046875, + "logps/rejected": -566.8543701171875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.04834580421447754, + "rewards/margins": 48.23228073120117, + "rewards/rejected": -48.28062438964844, + "step": 458 + }, + { + "epoch": 0.5, + "learning_rate": 2.363793103448276e-05, + "logits/chosen": -0.16556937992572784, + "logits/rejected": -1.327263355255127, + "logps/chosen": -464.6548767089844, + "logps/rejected": -662.5446166992188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.6031289100646973, + "rewards/margins": 57.662391662597656, + "rewards/rejected": -54.059261322021484, + "step": 459 + }, + { + "epoch": 0.5, + "learning_rate": 2.3586206896551728e-05, + "logits/chosen": 0.03939518332481384, + "logits/rejected": -1.1463898420333862, + "logps/chosen": -471.4759216308594, + "logps/rejected": -572.1267700195312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.5924675464630127, + "rewards/margins": 47.605220794677734, + "rewards/rejected": -50.197689056396484, + "step": 460 + }, + { + "epoch": 0.5, + "learning_rate": 2.353448275862069e-05, + "logits/chosen": -0.148860901594162, + "logits/rejected": -1.0923480987548828, + "logps/chosen": -390.76983642578125, + "logps/rejected": -567.6170043945312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.437652111053467, + "rewards/margins": 46.786190032958984, + "rewards/rejected": -49.223838806152344, + "step": 461 + }, + { + "epoch": 0.5, + "learning_rate": 2.3482758620689657e-05, + "logits/chosen": -0.10093279182910919, + "logits/rejected": -1.3032270669937134, + "logps/chosen": -547.587890625, + "logps/rejected": -627.3497314453125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.0484240055084229, + "rewards/margins": 48.94506072998047, + "rewards/rejected": -47.89663314819336, + "step": 462 + }, + { + "epoch": 0.51, + "learning_rate": 2.3431034482758625e-05, + "logits/chosen": -0.2709757685661316, + "logits/rejected": -1.1485973596572876, + "logps/chosen": -395.17950439453125, + "logps/rejected": -636.9188232421875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.9400150775909424, + "rewards/margins": 54.22304153442383, + "rewards/rejected": -53.28302764892578, + "step": 463 + }, + { + "epoch": 0.51, + "learning_rate": 2.3379310344827586e-05, + "logits/chosen": -0.06289319694042206, + "logits/rejected": -1.114414930343628, + "logps/chosen": -409.9918212890625, + "logps/rejected": -590.608642578125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.87382972240448, + "rewards/margins": 48.40852737426758, + "rewards/rejected": -50.28235626220703, + "step": 464 + }, + { + "epoch": 0.51, + "learning_rate": 2.3327586206896554e-05, + "logits/chosen": -0.14943581819534302, + "logits/rejected": -1.1048765182495117, + "logps/chosen": -391.01165771484375, + "logps/rejected": -591.4696655273438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.6352293491363525, + "rewards/margins": 51.1685905456543, + "rewards/rejected": -49.533363342285156, + "step": 465 + }, + { + "epoch": 0.51, + "learning_rate": 2.327586206896552e-05, + "logits/chosen": -0.27370375394821167, + "logits/rejected": -0.9103720188140869, + "logps/chosen": -281.7870788574219, + "logps/rejected": -599.569091796875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.6924283504486084, + "rewards/margins": 46.583003997802734, + "rewards/rejected": -50.275428771972656, + "step": 466 + }, + { + "epoch": 0.51, + "learning_rate": 2.3224137931034483e-05, + "logits/chosen": -0.07111064344644547, + "logits/rejected": -1.0503026247024536, + "logps/chosen": -385.4219970703125, + "logps/rejected": -592.8201293945312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.1101531982421875, + "rewards/margins": 44.58013916015625, + "rewards/rejected": -49.69029235839844, + "step": 467 + }, + { + "epoch": 0.51, + "learning_rate": 2.3172413793103448e-05, + "logits/chosen": -0.09694894403219223, + "logits/rejected": -1.4143892526626587, + "logps/chosen": -435.46929931640625, + "logps/rejected": -538.757568359375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.979873538017273, + "rewards/margins": 43.746910095214844, + "rewards/rejected": -45.726783752441406, + "step": 468 + }, + { + "epoch": 0.51, + "learning_rate": 2.3120689655172416e-05, + "logits/chosen": -0.18315471708774567, + "logits/rejected": -0.99836266040802, + "logps/chosen": -350.4498291015625, + "logps/rejected": -529.862548828125, + "loss": 0.0006, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.8093771934509277, + "rewards/margins": 41.57612991333008, + "rewards/rejected": -44.38550567626953, + "step": 469 + }, + { + "epoch": 0.51, + "learning_rate": 2.3068965517241377e-05, + "logits/chosen": -0.18518385291099548, + "logits/rejected": -1.1606504917144775, + "logps/chosen": -393.2962646484375, + "logps/rejected": -538.1412353515625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.910660743713379, + "rewards/margins": 42.86040115356445, + "rewards/rejected": -48.771060943603516, + "step": 470 + }, + { + "epoch": 0.51, + "learning_rate": 2.3017241379310345e-05, + "logits/chosen": 0.00019899383187294006, + "logits/rejected": -1.1684370040893555, + "logps/chosen": -479.20159912109375, + "logps/rejected": -641.7985229492188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2206273078918457, + "rewards/margins": 54.14768600463867, + "rewards/rejected": -53.92705535888672, + "step": 471 + }, + { + "epoch": 0.51, + "learning_rate": 2.2965517241379313e-05, + "logits/chosen": -0.03899521380662918, + "logits/rejected": -1.3559366464614868, + "logps/chosen": -433.9493103027344, + "logps/rejected": -585.2498779296875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.505762100219727, + "rewards/margins": 46.427223205566406, + "rewards/rejected": -51.932987213134766, + "step": 472 + }, + { + "epoch": 0.52, + "learning_rate": 2.291379310344828e-05, + "logits/chosen": -0.1787998378276825, + "logits/rejected": -1.2400846481323242, + "logps/chosen": -417.8245849609375, + "logps/rejected": -625.9854736328125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.0074243545532227, + "rewards/margins": 57.41107940673828, + "rewards/rejected": -55.403656005859375, + "step": 473 + }, + { + "epoch": 0.52, + "learning_rate": 2.2862068965517242e-05, + "logits/chosen": -0.1422000527381897, + "logits/rejected": -1.2094967365264893, + "logps/chosen": -420.824462890625, + "logps/rejected": -646.4536743164062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.564375638961792, + "rewards/margins": 52.2685546875, + "rewards/rejected": -54.83293151855469, + "step": 474 + }, + { + "epoch": 0.52, + "learning_rate": 2.281034482758621e-05, + "logits/chosen": -0.26421085000038147, + "logits/rejected": -1.0333547592163086, + "logps/chosen": -301.2657165527344, + "logps/rejected": -649.3005981445312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.931241989135742, + "rewards/margins": 50.742645263671875, + "rewards/rejected": -55.673885345458984, + "step": 475 + }, + { + "epoch": 0.52, + "learning_rate": 2.2758620689655175e-05, + "logits/chosen": -0.3443467915058136, + "logits/rejected": -1.1682060956954956, + "logps/chosen": -374.7920837402344, + "logps/rejected": -641.9935913085938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.4264507293701172, + "rewards/margins": 48.89411544799805, + "rewards/rejected": -50.3205680847168, + "step": 476 + }, + { + "epoch": 0.52, + "learning_rate": 2.270689655172414e-05, + "logits/chosen": -0.14052361249923706, + "logits/rejected": -1.365431547164917, + "logps/chosen": -428.28253173828125, + "logps/rejected": -611.1834716796875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.9522128105163574, + "rewards/margins": 49.015159606933594, + "rewards/rejected": -52.967376708984375, + "step": 477 + }, + { + "epoch": 0.52, + "learning_rate": 2.2655172413793104e-05, + "logits/chosen": -0.07368788123130798, + "logits/rejected": -1.3622853755950928, + "logps/chosen": -448.222412109375, + "logps/rejected": -609.247802734375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.316495895385742, + "rewards/margins": 51.841697692871094, + "rewards/rejected": -55.15819549560547, + "step": 478 + }, + { + "epoch": 0.52, + "learning_rate": 2.2603448275862072e-05, + "logits/chosen": -0.15994326770305634, + "logits/rejected": -1.2854161262512207, + "logps/chosen": -365.8324890136719, + "logps/rejected": -639.3682861328125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.163121223449707, + "rewards/margins": 51.78949737548828, + "rewards/rejected": -55.95261764526367, + "step": 479 + }, + { + "epoch": 0.52, + "learning_rate": 2.2551724137931033e-05, + "logits/chosen": -0.15817472338676453, + "logits/rejected": -1.3019388914108276, + "logps/chosen": -445.13543701171875, + "logps/rejected": -656.9911499023438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.5493353605270386, + "rewards/margins": 54.51856231689453, + "rewards/rejected": -55.067893981933594, + "step": 480 + }, + { + "epoch": 0.52, + "learning_rate": 2.25e-05, + "logits/chosen": -0.39189618825912476, + "logits/rejected": -1.1742621660232544, + "logps/chosen": -364.6230773925781, + "logps/rejected": -649.7832641601562, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -7.308798789978027, + "rewards/margins": 49.49714660644531, + "rewards/rejected": -56.80595016479492, + "step": 481 + }, + { + "epoch": 0.53, + "learning_rate": 2.2448275862068966e-05, + "logits/chosen": -0.17453332245349884, + "logits/rejected": -1.2001525163650513, + "logps/chosen": -444.8879699707031, + "logps/rejected": -657.28515625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.9432910680770874, + "rewards/margins": 53.479248046875, + "rewards/rejected": -55.42253875732422, + "step": 482 + }, + { + "epoch": 0.53, + "learning_rate": 2.2396551724137934e-05, + "logits/chosen": -0.11637411266565323, + "logits/rejected": -1.3278591632843018, + "logps/chosen": -434.76885986328125, + "logps/rejected": -645.82275390625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.3694510459899902, + "rewards/margins": 53.313201904296875, + "rewards/rejected": -56.682647705078125, + "step": 483 + }, + { + "epoch": 0.53, + "learning_rate": 2.23448275862069e-05, + "logits/chosen": -0.07361380755901337, + "logits/rejected": -1.4948939085006714, + "logps/chosen": -519.01416015625, + "logps/rejected": -701.379150390625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.0733790397644043, + "rewards/margins": 57.03836441040039, + "rewards/rejected": -59.11174392700195, + "step": 484 + }, + { + "epoch": 0.53, + "learning_rate": 2.2293103448275863e-05, + "logits/chosen": -0.12567904591560364, + "logits/rejected": -1.4412992000579834, + "logps/chosen": -454.69085693359375, + "logps/rejected": -657.5882568359375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14197838306427002, + "rewards/margins": 58.170379638671875, + "rewards/rejected": -58.312355041503906, + "step": 485 + }, + { + "epoch": 0.53, + "learning_rate": 2.2241379310344828e-05, + "logits/chosen": -0.10440649092197418, + "logits/rejected": -1.5682653188705444, + "logps/chosen": -645.4185180664062, + "logps/rejected": -665.8847045898438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 5.017826557159424, + "rewards/margins": 60.47355270385742, + "rewards/rejected": -55.45573043823242, + "step": 486 + }, + { + "epoch": 0.53, + "learning_rate": 2.2189655172413792e-05, + "logits/chosen": -0.005357783287763596, + "logits/rejected": -1.3899575471878052, + "logps/chosen": -551.1363525390625, + "logps/rejected": -617.5321044921875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.363417148590088, + "rewards/margins": 49.92127990722656, + "rewards/rejected": -54.284698486328125, + "step": 487 + }, + { + "epoch": 0.53, + "learning_rate": 2.213793103448276e-05, + "logits/chosen": -0.20832586288452148, + "logits/rejected": -1.3670223951339722, + "logps/chosen": -424.8078308105469, + "logps/rejected": -620.9976806640625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.274914264678955, + "rewards/margins": 52.706390380859375, + "rewards/rejected": -54.981300354003906, + "step": 488 + }, + { + "epoch": 0.53, + "learning_rate": 2.2086206896551725e-05, + "logits/chosen": -0.28159618377685547, + "logits/rejected": -1.283168077468872, + "logps/chosen": -391.3514099121094, + "logps/rejected": -634.2545776367188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.998907089233398, + "rewards/margins": 50.08463668823242, + "rewards/rejected": -56.08354187011719, + "step": 489 + }, + { + "epoch": 0.53, + "learning_rate": 2.203448275862069e-05, + "logits/chosen": -0.022036129608750343, + "logits/rejected": -1.4799420833587646, + "logps/chosen": -567.5009155273438, + "logps/rejected": -633.41259765625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.2931156158447266, + "rewards/margins": 54.43924331665039, + "rewards/rejected": -53.1461296081543, + "step": 490 + }, + { + "epoch": 0.54, + "learning_rate": 2.1982758620689658e-05, + "logits/chosen": -0.189289852976799, + "logits/rejected": -1.0637376308441162, + "logps/chosen": -420.50128173828125, + "logps/rejected": -661.4995727539062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.507000207901001, + "rewards/margins": 54.35142135620117, + "rewards/rejected": -57.85841751098633, + "step": 491 + }, + { + "epoch": 0.54, + "learning_rate": 2.1931034482758622e-05, + "logits/chosen": -0.03940558806061745, + "logits/rejected": -1.313617467880249, + "logps/chosen": -475.4183349609375, + "logps/rejected": -630.4754638671875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5654001235961914, + "rewards/margins": 54.049644470214844, + "rewards/rejected": -53.4842414855957, + "step": 492 + }, + { + "epoch": 0.54, + "learning_rate": 2.1879310344827587e-05, + "logits/chosen": -0.07513591647148132, + "logits/rejected": -1.212783694267273, + "logps/chosen": -403.09796142578125, + "logps/rejected": -616.3046875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.01631498336792, + "rewards/margins": 51.85172653198242, + "rewards/rejected": -54.868045806884766, + "step": 493 + }, + { + "epoch": 0.54, + "learning_rate": 2.1827586206896555e-05, + "logits/chosen": -0.2881481647491455, + "logits/rejected": -1.2288963794708252, + "logps/chosen": -396.6158752441406, + "logps/rejected": -694.475341796875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.2079168558120728, + "rewards/margins": 54.35879135131836, + "rewards/rejected": -55.566707611083984, + "step": 494 + }, + { + "epoch": 0.54, + "learning_rate": 2.177586206896552e-05, + "logits/chosen": 0.02672622725367546, + "logits/rejected": -1.3478881120681763, + "logps/chosen": -530.7757568359375, + "logps/rejected": -684.7239990234375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.0076181888580322, + "rewards/margins": 56.306922912597656, + "rewards/rejected": -58.31454086303711, + "step": 495 + }, + { + "epoch": 0.54, + "learning_rate": 2.1724137931034484e-05, + "logits/chosen": -0.1358601450920105, + "logits/rejected": -1.2526161670684814, + "logps/chosen": -440.38702392578125, + "logps/rejected": -610.0796508789062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.512816429138184, + "rewards/margins": 48.28220748901367, + "rewards/rejected": -52.79502487182617, + "step": 496 + }, + { + "epoch": 0.54, + "learning_rate": 2.167241379310345e-05, + "logits/chosen": -0.06988359987735748, + "logits/rejected": -1.1959302425384521, + "logps/chosen": -449.9613342285156, + "logps/rejected": -671.1732788085938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.2779130935668945, + "rewards/margins": 55.88106155395508, + "rewards/rejected": -60.15897750854492, + "step": 497 + }, + { + "epoch": 0.54, + "learning_rate": 2.1620689655172413e-05, + "logits/chosen": -0.28054380416870117, + "logits/rejected": -1.2163547277450562, + "logps/chosen": -388.01300048828125, + "logps/rejected": -652.7429809570312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.8053398132324219, + "rewards/margins": 56.216575622558594, + "rewards/rejected": -57.02192306518555, + "step": 498 + }, + { + "epoch": 0.54, + "learning_rate": 2.156896551724138e-05, + "logits/chosen": -0.06607590615749359, + "logits/rejected": -1.2197513580322266, + "logps/chosen": -454.84771728515625, + "logps/rejected": -654.5570678710938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.4068199396133423, + "rewards/margins": 56.8363151550293, + "rewards/rejected": -58.24313735961914, + "step": 499 + }, + { + "epoch": 0.55, + "learning_rate": 2.1517241379310346e-05, + "logits/chosen": -0.28458642959594727, + "logits/rejected": -1.0344970226287842, + "logps/chosen": -370.1960144042969, + "logps/rejected": -655.9469604492188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.959820747375488, + "rewards/margins": 51.25876998901367, + "rewards/rejected": -56.218589782714844, + "step": 500 + }, + { + "epoch": 0.55, + "learning_rate": 2.146551724137931e-05, + "logits/chosen": -0.20156291127204895, + "logits/rejected": -1.1681430339813232, + "logps/chosen": -385.0158386230469, + "logps/rejected": -635.7328491210938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.1334333419799805, + "rewards/margins": 50.82726287841797, + "rewards/rejected": -55.960693359375, + "step": 501 + }, + { + "epoch": 0.55, + "learning_rate": 2.141379310344828e-05, + "logits/chosen": -0.0020601898431777954, + "logits/rejected": -1.1352349519729614, + "logps/chosen": -448.7326965332031, + "logps/rejected": -693.2049560546875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.15153491497039795, + "rewards/margins": 58.00741958618164, + "rewards/rejected": -58.15895462036133, + "step": 502 + }, + { + "epoch": 0.55, + "learning_rate": 2.1362068965517243e-05, + "logits/chosen": 0.022678162902593613, + "logits/rejected": -1.2108732461929321, + "logps/chosen": -430.61895751953125, + "logps/rejected": -625.4843139648438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.665067672729492, + "rewards/margins": 50.22642517089844, + "rewards/rejected": -54.89149475097656, + "step": 503 + }, + { + "epoch": 0.55, + "learning_rate": 2.1310344827586208e-05, + "logits/chosen": -0.13953523337841034, + "logits/rejected": -1.1829777956008911, + "logps/chosen": -472.4615173339844, + "logps/rejected": -752.0328369140625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.490379810333252, + "rewards/margins": 54.96733474731445, + "rewards/rejected": -56.45771408081055, + "step": 504 + }, + { + "epoch": 0.55, + "learning_rate": 2.1258620689655172e-05, + "logits/chosen": -0.12544049322605133, + "logits/rejected": -1.331096887588501, + "logps/chosen": -456.3561706542969, + "logps/rejected": -655.4522705078125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.4183454513549805, + "rewards/margins": 53.30242156982422, + "rewards/rejected": -56.72076416015625, + "step": 505 + }, + { + "epoch": 0.55, + "learning_rate": 2.1206896551724137e-05, + "logits/chosen": -0.15563945472240448, + "logits/rejected": -1.2740764617919922, + "logps/chosen": -437.84893798828125, + "logps/rejected": -658.6337280273438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4970543384552002, + "rewards/margins": 58.15724563598633, + "rewards/rejected": -57.660194396972656, + "step": 506 + }, + { + "epoch": 0.55, + "learning_rate": 2.1155172413793105e-05, + "logits/chosen": -0.08426739275455475, + "logits/rejected": -1.1883736848831177, + "logps/chosen": -474.1285705566406, + "logps/rejected": -654.4371337890625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.8744089603424072, + "rewards/margins": 52.474884033203125, + "rewards/rejected": -53.34928894042969, + "step": 507 + }, + { + "epoch": 0.55, + "learning_rate": 2.110344827586207e-05, + "logits/chosen": -0.014494461007416248, + "logits/rejected": -1.1064589023590088, + "logps/chosen": -423.87103271484375, + "logps/rejected": -697.6011962890625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.6249542236328125, + "rewards/margins": 58.5837287902832, + "rewards/rejected": -61.20867919921875, + "step": 508 + }, + { + "epoch": 0.56, + "learning_rate": 2.1051724137931034e-05, + "logits/chosen": -0.12398696690797806, + "logits/rejected": -1.3565784692764282, + "logps/chosen": -491.5214538574219, + "logps/rejected": -613.30712890625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.7314184308052063, + "rewards/margins": 53.077938079833984, + "rewards/rejected": -53.80936050415039, + "step": 509 + }, + { + "epoch": 0.56, + "learning_rate": 2.1000000000000002e-05, + "logits/chosen": -0.051289428025484085, + "logits/rejected": -1.492192268371582, + "logps/chosen": -518.4822387695312, + "logps/rejected": -630.14013671875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1812780499458313, + "rewards/margins": 54.831336975097656, + "rewards/rejected": -54.650054931640625, + "step": 510 + }, + { + "epoch": 0.56, + "learning_rate": 2.0948275862068967e-05, + "logits/chosen": -0.2157306969165802, + "logits/rejected": -1.2566158771514893, + "logps/chosen": -459.4637145996094, + "logps/rejected": -669.658935546875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.579399585723877, + "rewards/margins": 53.49375915527344, + "rewards/rejected": -55.073158264160156, + "step": 511 + }, + { + "epoch": 0.56, + "learning_rate": 2.0896551724137935e-05, + "logits/chosen": -0.11594392359256744, + "logits/rejected": -1.24819815158844, + "logps/chosen": -514.4534912109375, + "logps/rejected": -639.0775146484375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.502558171749115, + "rewards/margins": 54.507659912109375, + "rewards/rejected": -55.010215759277344, + "step": 512 + }, + { + "epoch": 0.56, + "learning_rate": 2.08448275862069e-05, + "logits/chosen": -0.17627860605716705, + "logits/rejected": -1.393364429473877, + "logps/chosen": -440.92340087890625, + "logps/rejected": -675.639404296875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.33227670192718506, + "rewards/margins": 60.44564437866211, + "rewards/rejected": -60.777923583984375, + "step": 513 + }, + { + "epoch": 0.56, + "learning_rate": 2.0793103448275864e-05, + "logits/chosen": -0.14404740929603577, + "logits/rejected": -1.421351671218872, + "logps/chosen": -504.71197509765625, + "logps/rejected": -648.1561889648438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6551969051361084, + "rewards/margins": 57.213863372802734, + "rewards/rejected": -56.55866622924805, + "step": 514 + }, + { + "epoch": 0.56, + "learning_rate": 2.074137931034483e-05, + "logits/chosen": 0.004668328911066055, + "logits/rejected": -1.4230290651321411, + "logps/chosen": -436.03009033203125, + "logps/rejected": -611.609375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.6679420471191406, + "rewards/margins": 49.031368255615234, + "rewards/rejected": -51.699310302734375, + "step": 515 + }, + { + "epoch": 0.56, + "learning_rate": 2.0689655172413793e-05, + "logits/chosen": -0.029186539351940155, + "logits/rejected": -1.4701117277145386, + "logps/chosen": -523.4320068359375, + "logps/rejected": -659.4295654296875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.930753707885742, + "rewards/margins": 55.25088882446289, + "rewards/rejected": -59.181644439697266, + "step": 516 + }, + { + "epoch": 0.56, + "learning_rate": 2.0637931034482758e-05, + "logits/chosen": -0.2120245397090912, + "logits/rejected": -0.9436817765235901, + "logps/chosen": -356.55645751953125, + "logps/rejected": -751.9764404296875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.7937495708465576, + "rewards/margins": 54.103370666503906, + "rewards/rejected": -57.897125244140625, + "step": 517 + }, + { + "epoch": 0.57, + "learning_rate": 2.0586206896551726e-05, + "logits/chosen": -0.155109241604805, + "logits/rejected": -1.2690709829330444, + "logps/chosen": -445.1922912597656, + "logps/rejected": -650.4595947265625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.628649711608887, + "rewards/margins": 51.917388916015625, + "rewards/rejected": -56.54603576660156, + "step": 518 + }, + { + "epoch": 0.57, + "learning_rate": 2.053448275862069e-05, + "logits/chosen": -0.04659181088209152, + "logits/rejected": -1.4601080417633057, + "logps/chosen": -477.2423095703125, + "logps/rejected": -616.805908203125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1397993117570877, + "rewards/margins": 53.21185302734375, + "rewards/rejected": -53.07205581665039, + "step": 519 + }, + { + "epoch": 0.57, + "learning_rate": 2.048275862068966e-05, + "logits/chosen": -0.1084991991519928, + "logits/rejected": -1.009717583656311, + "logps/chosen": -416.266845703125, + "logps/rejected": -649.83251953125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -7.471712112426758, + "rewards/margins": 49.83021926879883, + "rewards/rejected": -57.30193328857422, + "step": 520 + }, + { + "epoch": 0.57, + "learning_rate": 2.0431034482758623e-05, + "logits/chosen": -0.11132999509572983, + "logits/rejected": -1.2182048559188843, + "logps/chosen": -427.03985595703125, + "logps/rejected": -652.3914184570312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.08266544342041, + "rewards/margins": 52.819427490234375, + "rewards/rejected": -57.90208435058594, + "step": 521 + }, + { + "epoch": 0.57, + "learning_rate": 2.0379310344827588e-05, + "logits/chosen": -0.07166096568107605, + "logits/rejected": -0.8901804089546204, + "logps/chosen": -398.4652404785156, + "logps/rejected": -603.7015991210938, + "loss": 0.0472, + "rewards/accuracies": 0.9375, + "rewards/chosen": -7.047485828399658, + "rewards/margins": 45.51860809326172, + "rewards/rejected": -52.566097259521484, + "step": 522 + }, + { + "epoch": 0.57, + "learning_rate": 2.0327586206896552e-05, + "logits/chosen": -0.10595792531967163, + "logits/rejected": -1.0550121068954468, + "logps/chosen": -347.60546875, + "logps/rejected": -728.9971313476562, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.260272026062012, + "rewards/margins": 56.09633255004883, + "rewards/rejected": -62.35660171508789, + "step": 523 + }, + { + "epoch": 0.57, + "learning_rate": 2.0275862068965517e-05, + "logits/chosen": -0.27664729952812195, + "logits/rejected": -1.2738081216812134, + "logps/chosen": -419.85357666015625, + "logps/rejected": -647.8483276367188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.494794845581055, + "rewards/margins": 52.42628860473633, + "rewards/rejected": -56.92108154296875, + "step": 524 + }, + { + "epoch": 0.57, + "learning_rate": 2.022413793103448e-05, + "logits/chosen": -0.07392889261245728, + "logits/rejected": -1.3131624460220337, + "logps/chosen": -538.642333984375, + "logps/rejected": -608.0081787109375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.9629731178283691, + "rewards/margins": 52.443206787109375, + "rewards/rejected": -53.40617752075195, + "step": 525 + }, + { + "epoch": 0.57, + "learning_rate": 2.017241379310345e-05, + "logits/chosen": -0.21235141158103943, + "logits/rejected": -1.2620832920074463, + "logps/chosen": -470.26904296875, + "logps/rejected": -648.2606811523438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.8237268924713135, + "rewards/margins": 51.55657196044922, + "rewards/rejected": -54.38030242919922, + "step": 526 + }, + { + "epoch": 0.57, + "learning_rate": 2.0120689655172414e-05, + "logits/chosen": -0.07883001118898392, + "logits/rejected": -1.4777758121490479, + "logps/chosen": -513.1040649414062, + "logps/rejected": -663.640380859375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.5773372650146484, + "rewards/margins": 58.29991149902344, + "rewards/rejected": -60.87725067138672, + "step": 527 + }, + { + "epoch": 0.58, + "learning_rate": 2.0068965517241382e-05, + "logits/chosen": -0.31512337923049927, + "logits/rejected": -1.1388442516326904, + "logps/chosen": -370.7431640625, + "logps/rejected": -662.3712768554688, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -9.118474960327148, + "rewards/margins": 48.47383117675781, + "rewards/rejected": -57.59230422973633, + "step": 528 + }, + { + "epoch": 0.58, + "learning_rate": 2.0017241379310347e-05, + "logits/chosen": -0.2608606815338135, + "logits/rejected": -1.227593183517456, + "logps/chosen": -443.54779052734375, + "logps/rejected": -685.2706298828125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.979219436645508, + "rewards/margins": 54.63554000854492, + "rewards/rejected": -58.6147575378418, + "step": 529 + }, + { + "epoch": 0.58, + "learning_rate": 1.996551724137931e-05, + "logits/chosen": -0.21307611465454102, + "logits/rejected": -1.1337976455688477, + "logps/chosen": -437.9703063964844, + "logps/rejected": -671.6073608398438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.212578296661377, + "rewards/margins": 53.49246597290039, + "rewards/rejected": -57.705047607421875, + "step": 530 + }, + { + "epoch": 0.58, + "learning_rate": 1.991379310344828e-05, + "logits/chosen": -0.16399875283241272, + "logits/rejected": -1.2640507221221924, + "logps/chosen": -428.9464111328125, + "logps/rejected": -718.56884765625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.8005006313323975, + "rewards/margins": 59.8723030090332, + "rewards/rejected": -62.67280578613281, + "step": 531 + }, + { + "epoch": 0.58, + "learning_rate": 1.9862068965517244e-05, + "logits/chosen": -0.03811350464820862, + "logits/rejected": -1.584404468536377, + "logps/chosen": -647.8243408203125, + "logps/rejected": -653.6069946289062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.426783084869385, + "rewards/margins": 55.79122543334961, + "rewards/rejected": -60.2180061340332, + "step": 532 + }, + { + "epoch": 0.58, + "learning_rate": 1.981034482758621e-05, + "logits/chosen": -0.16661666333675385, + "logits/rejected": -1.3489412069320679, + "logps/chosen": -576.0233154296875, + "logps/rejected": -677.5670776367188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.02166748046875, + "rewards/margins": 55.28102111816406, + "rewards/rejected": -60.30269241333008, + "step": 533 + }, + { + "epoch": 0.58, + "learning_rate": 1.9758620689655173e-05, + "logits/chosen": -0.42992162704467773, + "logits/rejected": -1.0344213247299194, + "logps/chosen": -331.75604248046875, + "logps/rejected": -705.9281005859375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -10.415375709533691, + "rewards/margins": 49.89491271972656, + "rewards/rejected": -60.31028747558594, + "step": 534 + }, + { + "epoch": 0.58, + "learning_rate": 1.9706896551724138e-05, + "logits/chosen": -0.15657415986061096, + "logits/rejected": -1.419963002204895, + "logps/chosen": -616.8851928710938, + "logps/rejected": -672.5630493164062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -7.5933356285095215, + "rewards/margins": 52.59075927734375, + "rewards/rejected": -60.1840934753418, + "step": 535 + }, + { + "epoch": 0.58, + "learning_rate": 1.9655172413793106e-05, + "logits/chosen": -0.38475874066352844, + "logits/rejected": -1.1948922872543335, + "logps/chosen": -431.2165832519531, + "logps/rejected": -674.835205078125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -7.450318813323975, + "rewards/margins": 51.454200744628906, + "rewards/rejected": -58.90452575683594, + "step": 536 + }, + { + "epoch": 0.59, + "learning_rate": 1.960344827586207e-05, + "logits/chosen": -0.6328589916229248, + "logits/rejected": -1.2824782133102417, + "logps/chosen": -460.2396545410156, + "logps/rejected": -756.9679565429688, + "loss": 0.0105, + "rewards/accuracies": 1.0, + "rewards/chosen": -11.223982810974121, + "rewards/margins": 51.80082321166992, + "rewards/rejected": -63.024803161621094, + "step": 537 + }, + { + "epoch": 0.59, + "learning_rate": 1.9551724137931035e-05, + "logits/chosen": -0.28524112701416016, + "logits/rejected": -1.3562718629837036, + "logps/chosen": -548.3995971679688, + "logps/rejected": -717.23388671875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.4443503618240356, + "rewards/margins": 59.99860382080078, + "rewards/rejected": -61.44295883178711, + "step": 538 + }, + { + "epoch": 0.59, + "learning_rate": 1.9500000000000003e-05, + "logits/chosen": -0.2540721595287323, + "logits/rejected": -1.2168906927108765, + "logps/chosen": -408.9739074707031, + "logps/rejected": -692.2616577148438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -9.923998832702637, + "rewards/margins": 49.68036651611328, + "rewards/rejected": -59.6043701171875, + "step": 539 + }, + { + "epoch": 0.59, + "learning_rate": 1.9448275862068968e-05, + "logits/chosen": -0.42604437470436096, + "logits/rejected": -1.289304256439209, + "logps/chosen": -458.89276123046875, + "logps/rejected": -748.4998779296875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -11.387511253356934, + "rewards/margins": 51.951114654541016, + "rewards/rejected": -63.338623046875, + "step": 540 + }, + { + "epoch": 0.59, + "learning_rate": 1.9396551724137932e-05, + "logits/chosen": -0.30175092816352844, + "logits/rejected": -1.4019683599472046, + "logps/chosen": -481.57489013671875, + "logps/rejected": -683.3731079101562, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.1864436864852905, + "rewards/margins": 59.80683898925781, + "rewards/rejected": -60.993282318115234, + "step": 541 + }, + { + "epoch": 0.59, + "learning_rate": 1.9344827586206897e-05, + "logits/chosen": -0.22525370121002197, + "logits/rejected": -1.2713929414749146, + "logps/chosen": -441.145751953125, + "logps/rejected": -655.5973510742188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.433875322341919, + "rewards/margins": 54.15318298339844, + "rewards/rejected": -57.58705139160156, + "step": 542 + }, + { + "epoch": 0.59, + "learning_rate": 1.929310344827586e-05, + "logits/chosen": -0.36525341868400574, + "logits/rejected": -1.1360570192337036, + "logps/chosen": -379.4644470214844, + "logps/rejected": -715.0608520507812, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.589846611022949, + "rewards/margins": 54.25578689575195, + "rewards/rejected": -59.84563064575195, + "step": 543 + }, + { + "epoch": 0.59, + "learning_rate": 1.924137931034483e-05, + "logits/chosen": -0.06538578867912292, + "logits/rejected": -1.3779454231262207, + "logps/chosen": -580.675537109375, + "logps/rejected": -661.8748168945312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -8.214271545410156, + "rewards/margins": 48.59384536743164, + "rewards/rejected": -56.8081169128418, + "step": 544 + }, + { + "epoch": 0.59, + "learning_rate": 1.9189655172413794e-05, + "logits/chosen": -0.1274988353252411, + "logits/rejected": -1.3928804397583008, + "logps/chosen": -509.0260925292969, + "logps/rejected": -606.0980224609375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.795976161956787, + "rewards/margins": 46.74262237548828, + "rewards/rejected": -50.538597106933594, + "step": 545 + }, + { + "epoch": 0.6, + "learning_rate": 1.913793103448276e-05, + "logits/chosen": -0.08550414443016052, + "logits/rejected": -1.2893277406692505, + "logps/chosen": -506.2032165527344, + "logps/rejected": -692.5704956054688, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.392428874969482, + "rewards/margins": 55.658851623535156, + "rewards/rejected": -60.0512809753418, + "step": 546 + }, + { + "epoch": 0.6, + "learning_rate": 1.9086206896551727e-05, + "logits/chosen": -0.24850007891654968, + "logits/rejected": -1.2666964530944824, + "logps/chosen": -445.8536071777344, + "logps/rejected": -691.1626586914062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -7.471304416656494, + "rewards/margins": 50.54898452758789, + "rewards/rejected": -58.020286560058594, + "step": 547 + }, + { + "epoch": 0.6, + "learning_rate": 1.903448275862069e-05, + "logits/chosen": -0.09865757077932358, + "logits/rejected": -1.2371084690093994, + "logps/chosen": -543.6181640625, + "logps/rejected": -687.4448852539062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.3608412742614746, + "rewards/margins": 54.74359893798828, + "rewards/rejected": -57.104434967041016, + "step": 548 + }, + { + "epoch": 0.6, + "learning_rate": 1.8982758620689656e-05, + "logits/chosen": -0.10665954649448395, + "logits/rejected": -1.450705885887146, + "logps/chosen": -486.8913879394531, + "logps/rejected": -677.2809448242188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.5832552909851074, + "rewards/margins": 57.06523132324219, + "rewards/rejected": -58.64849090576172, + "step": 549 + }, + { + "epoch": 0.6, + "learning_rate": 1.8931034482758624e-05, + "logits/chosen": -0.1286657452583313, + "logits/rejected": -1.407023549079895, + "logps/chosen": -486.1256103515625, + "logps/rejected": -709.9680786132812, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.5269105434417725, + "rewards/margins": 59.747928619384766, + "rewards/rejected": -62.274845123291016, + "step": 550 + }, + { + "epoch": 0.6, + "learning_rate": 1.8879310344827585e-05, + "logits/chosen": -0.0675632432103157, + "logits/rejected": -1.272240400314331, + "logps/chosen": -468.42352294921875, + "logps/rejected": -674.0982055664062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.776965141296387, + "rewards/margins": 52.262508392333984, + "rewards/rejected": -59.03947830200195, + "step": 551 + }, + { + "epoch": 0.6, + "learning_rate": 1.8827586206896553e-05, + "logits/chosen": -0.34433895349502563, + "logits/rejected": -1.113890290260315, + "logps/chosen": -298.00396728515625, + "logps/rejected": -606.363525390625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -9.223278045654297, + "rewards/margins": 44.2802619934082, + "rewards/rejected": -53.5035400390625, + "step": 552 + }, + { + "epoch": 0.6, + "learning_rate": 1.8775862068965518e-05, + "logits/chosen": -0.21515408158302307, + "logits/rejected": -1.428251028060913, + "logps/chosen": -425.70794677734375, + "logps/rejected": -670.1383666992188, + "loss": 0.0167, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.1336541175842285, + "rewards/margins": 50.13953399658203, + "rewards/rejected": -56.273189544677734, + "step": 553 + }, + { + "epoch": 0.6, + "learning_rate": 1.8724137931034482e-05, + "logits/chosen": -0.21798893809318542, + "logits/rejected": -1.3687691688537598, + "logps/chosen": -487.7384033203125, + "logps/rejected": -666.2149658203125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.636868476867676, + "rewards/margins": 54.46982192993164, + "rewards/rejected": -57.106689453125, + "step": 554 + }, + { + "epoch": 0.61, + "learning_rate": 1.867241379310345e-05, + "logits/chosen": -0.2618967592716217, + "logits/rejected": -1.3180971145629883, + "logps/chosen": -451.2808837890625, + "logps/rejected": -653.58837890625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -7.717561721801758, + "rewards/margins": 50.419822692871094, + "rewards/rejected": -58.13738250732422, + "step": 555 + }, + { + "epoch": 0.61, + "learning_rate": 1.8620689655172415e-05, + "logits/chosen": -0.30724576115608215, + "logits/rejected": -1.066850185394287, + "logps/chosen": -341.5390930175781, + "logps/rejected": -702.739501953125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.248960971832275, + "rewards/margins": 53.42186737060547, + "rewards/rejected": -59.67082595825195, + "step": 556 + }, + { + "epoch": 0.61, + "learning_rate": 1.856896551724138e-05, + "logits/chosen": -0.2094910740852356, + "logits/rejected": -1.2574561834335327, + "logps/chosen": -449.03106689453125, + "logps/rejected": -726.781982421875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -8.954678535461426, + "rewards/margins": 52.55630874633789, + "rewards/rejected": -61.510990142822266, + "step": 557 + }, + { + "epoch": 0.61, + "learning_rate": 1.8517241379310348e-05, + "logits/chosen": -0.0532219298183918, + "logits/rejected": -1.2439594268798828, + "logps/chosen": -488.92144775390625, + "logps/rejected": -674.4827270507812, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -8.375113487243652, + "rewards/margins": 51.293792724609375, + "rewards/rejected": -59.668907165527344, + "step": 558 + }, + { + "epoch": 0.61, + "learning_rate": 1.8465517241379312e-05, + "logits/chosen": 0.03209938108921051, + "logits/rejected": -1.5995020866394043, + "logps/chosen": -532.6072998046875, + "logps/rejected": -616.2276611328125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.855282783508301, + "rewards/margins": 50.27061462402344, + "rewards/rejected": -54.125892639160156, + "step": 559 + }, + { + "epoch": 0.61, + "learning_rate": 1.8413793103448277e-05, + "logits/chosen": -0.12263540923595428, + "logits/rejected": -1.1568797826766968, + "logps/chosen": -516.74365234375, + "logps/rejected": -714.9854736328125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.593143463134766, + "rewards/margins": 52.229610443115234, + "rewards/rejected": -58.82275390625, + "step": 560 + }, + { + "epoch": 0.61, + "learning_rate": 1.836206896551724e-05, + "logits/chosen": -0.13906851410865784, + "logits/rejected": -1.1512776613235474, + "logps/chosen": -455.60467529296875, + "logps/rejected": -710.459716796875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.503664493560791, + "rewards/margins": 55.149513244628906, + "rewards/rejected": -60.65318298339844, + "step": 561 + }, + { + "epoch": 0.61, + "learning_rate": 1.8310344827586206e-05, + "logits/chosen": -0.01586013287305832, + "logits/rejected": -1.5763020515441895, + "logps/chosen": -649.6324462890625, + "logps/rejected": -634.0162963867188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.6673918962478638, + "rewards/margins": 52.934112548828125, + "rewards/rejected": -54.60150146484375, + "step": 562 + }, + { + "epoch": 0.61, + "learning_rate": 1.8258620689655174e-05, + "logits/chosen": -0.09369073808193207, + "logits/rejected": -1.5399290323257446, + "logps/chosen": -532.3624877929688, + "logps/rejected": -650.81689453125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.8866522312164307, + "rewards/margins": 54.70314025878906, + "rewards/rejected": -56.58979415893555, + "step": 563 + }, + { + "epoch": 0.62, + "learning_rate": 1.820689655172414e-05, + "logits/chosen": -0.1317727118730545, + "logits/rejected": -1.3313921689987183, + "logps/chosen": -494.91180419921875, + "logps/rejected": -686.72607421875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.157313108444214, + "rewards/margins": 55.9873046875, + "rewards/rejected": -58.14461898803711, + "step": 564 + }, + { + "epoch": 0.62, + "learning_rate": 1.8155172413793103e-05, + "logits/chosen": -0.12760236859321594, + "logits/rejected": -1.4653446674346924, + "logps/chosen": -491.8769836425781, + "logps/rejected": -678.2470092773438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.088787078857422, + "rewards/margins": 56.210838317871094, + "rewards/rejected": -60.299625396728516, + "step": 565 + }, + { + "epoch": 0.62, + "learning_rate": 1.810344827586207e-05, + "logits/chosen": -0.2516867220401764, + "logits/rejected": -1.2494770288467407, + "logps/chosen": -428.3562316894531, + "logps/rejected": -669.3509521484375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.195309638977051, + "rewards/margins": 52.69432067871094, + "rewards/rejected": -58.889625549316406, + "step": 566 + }, + { + "epoch": 0.62, + "learning_rate": 1.8051724137931036e-05, + "logits/chosen": -0.30673977732658386, + "logits/rejected": -1.483799695968628, + "logps/chosen": -428.2577209472656, + "logps/rejected": -589.807373046875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -7.883467674255371, + "rewards/margins": 43.719886779785156, + "rewards/rejected": -51.603355407714844, + "step": 567 + }, + { + "epoch": 0.62, + "learning_rate": 1.8e-05, + "logits/chosen": -0.19813589751720428, + "logits/rejected": -1.5698930025100708, + "logps/chosen": -486.477783203125, + "logps/rejected": -661.2116088867188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.44511079788208, + "rewards/margins": 55.299400329589844, + "rewards/rejected": -58.7445068359375, + "step": 568 + }, + { + "epoch": 0.62, + "learning_rate": 1.7948275862068965e-05, + "logits/chosen": -0.22892844676971436, + "logits/rejected": -1.4997096061706543, + "logps/chosen": -470.40216064453125, + "logps/rejected": -651.0798950195312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -9.123993873596191, + "rewards/margins": 48.75234603881836, + "rewards/rejected": -57.876338958740234, + "step": 569 + }, + { + "epoch": 0.62, + "learning_rate": 1.789655172413793e-05, + "logits/chosen": -0.10823440551757812, + "logits/rejected": -1.3152951002120972, + "logps/chosen": -547.5245971679688, + "logps/rejected": -685.3779907226562, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.454561233520508, + "rewards/margins": 52.54243850708008, + "rewards/rejected": -56.99700164794922, + "step": 570 + }, + { + "epoch": 0.62, + "learning_rate": 1.7844827586206898e-05, + "logits/chosen": -0.24758121371269226, + "logits/rejected": -1.208099603652954, + "logps/chosen": -401.03070068359375, + "logps/rejected": -678.6541137695312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -7.605196952819824, + "rewards/margins": 49.06074142456055, + "rewards/rejected": -56.66593551635742, + "step": 571 + }, + { + "epoch": 0.62, + "learning_rate": 1.7793103448275862e-05, + "logits/chosen": -0.2494858354330063, + "logits/rejected": -1.3483632802963257, + "logps/chosen": -478.4145812988281, + "logps/rejected": -692.2071533203125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.4422650337219238, + "rewards/margins": 55.957786560058594, + "rewards/rejected": -57.400054931640625, + "step": 572 + }, + { + "epoch": 0.63, + "learning_rate": 1.7741379310344827e-05, + "logits/chosen": -0.18455466628074646, + "logits/rejected": -1.5773600339889526, + "logps/chosen": -532.016845703125, + "logps/rejected": -631.9456787109375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.839169502258301, + "rewards/margins": 52.62319564819336, + "rewards/rejected": -55.462364196777344, + "step": 573 + }, + { + "epoch": 0.63, + "learning_rate": 1.7689655172413795e-05, + "logits/chosen": -0.08394797146320343, + "logits/rejected": -1.2678709030151367, + "logps/chosen": -426.6719665527344, + "logps/rejected": -693.8532104492188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -8.079269409179688, + "rewards/margins": 54.31288528442383, + "rewards/rejected": -62.39215087890625, + "step": 574 + }, + { + "epoch": 0.63, + "learning_rate": 1.763793103448276e-05, + "logits/chosen": -0.13648506999015808, + "logits/rejected": -1.4690591096878052, + "logps/chosen": -586.6487426757812, + "logps/rejected": -710.5147094726562, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.839778184890747, + "rewards/margins": 59.91093444824219, + "rewards/rejected": -61.750709533691406, + "step": 575 + }, + { + "epoch": 0.63, + "learning_rate": 1.7586206896551728e-05, + "logits/chosen": -0.21827536821365356, + "logits/rejected": -1.2792953252792358, + "logps/chosen": -414.493408203125, + "logps/rejected": -647.998779296875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.311338901519775, + "rewards/margins": 51.5853157043457, + "rewards/rejected": -56.89664840698242, + "step": 576 + }, + { + "epoch": 0.63, + "learning_rate": 1.7534482758620692e-05, + "logits/chosen": -0.0653618648648262, + "logits/rejected": -1.6563713550567627, + "logps/chosen": -604.5464477539062, + "logps/rejected": -647.69873046875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.1551756858825684, + "rewards/margins": 54.157005310058594, + "rewards/rejected": -55.31217956542969, + "step": 577 + }, + { + "epoch": 0.63, + "learning_rate": 1.7482758620689657e-05, + "logits/chosen": -0.10746625810861588, + "logits/rejected": -1.0887423753738403, + "logps/chosen": -423.400634765625, + "logps/rejected": -635.4210205078125, + "loss": 0.6823, + "rewards/accuracies": 0.9375, + "rewards/chosen": -7.090230941772461, + "rewards/margins": 45.47228240966797, + "rewards/rejected": -52.56251525878906, + "step": 578 + }, + { + "epoch": 0.63, + "learning_rate": 1.743103448275862e-05, + "logits/chosen": -0.2545757591724396, + "logits/rejected": -1.3590258359909058, + "logps/chosen": -404.650146484375, + "logps/rejected": -669.2538452148438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.74318265914917, + "rewards/margins": 56.251548767089844, + "rewards/rejected": -58.99473190307617, + "step": 579 + }, + { + "epoch": 0.63, + "learning_rate": 1.7379310344827586e-05, + "logits/chosen": -0.050807081162929535, + "logits/rejected": -1.2381399869918823, + "logps/chosen": -412.51385498046875, + "logps/rejected": -616.2285766601562, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.05122709274292, + "rewards/margins": 50.5135498046875, + "rewards/rejected": -53.564781188964844, + "step": 580 + }, + { + "epoch": 0.63, + "learning_rate": 1.732758620689655e-05, + "logits/chosen": -0.14367303252220154, + "logits/rejected": -1.3293488025665283, + "logps/chosen": -436.77374267578125, + "logps/rejected": -644.4839477539062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.5984394550323486, + "rewards/margins": 51.456111907958984, + "rewards/rejected": -55.0545539855957, + "step": 581 + }, + { + "epoch": 0.63, + "learning_rate": 1.727586206896552e-05, + "logits/chosen": -0.08829887211322784, + "logits/rejected": -1.4413114786148071, + "logps/chosen": -538.6859130859375, + "logps/rejected": -633.6636962890625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.367769718170166, + "rewards/margins": 54.034610748291016, + "rewards/rejected": -56.40237808227539, + "step": 582 + }, + { + "epoch": 0.64, + "learning_rate": 1.7224137931034483e-05, + "logits/chosen": -0.35657984018325806, + "logits/rejected": -1.4725849628448486, + "logps/chosen": -414.2039794921875, + "logps/rejected": -550.0541381835938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.6985023021698, + "rewards/margins": 46.69954299926758, + "rewards/rejected": -49.39804458618164, + "step": 583 + }, + { + "epoch": 0.64, + "learning_rate": 1.717241379310345e-05, + "logits/chosen": -0.08540710806846619, + "logits/rejected": -1.550531268119812, + "logps/chosen": -530.1245727539062, + "logps/rejected": -635.4656372070312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.296605110168457, + "rewards/margins": 48.70091247558594, + "rewards/rejected": -54.99751663208008, + "step": 584 + }, + { + "epoch": 0.64, + "learning_rate": 1.7120689655172416e-05, + "logits/chosen": -0.11361716687679291, + "logits/rejected": -1.359609842300415, + "logps/chosen": -429.12786865234375, + "logps/rejected": -649.3413696289062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.8643945455551147, + "rewards/margins": 53.536781311035156, + "rewards/rejected": -54.40117645263672, + "step": 585 + }, + { + "epoch": 0.64, + "learning_rate": 1.706896551724138e-05, + "logits/chosen": -0.24846045672893524, + "logits/rejected": -1.479831576347351, + "logps/chosen": -515.0034790039062, + "logps/rejected": -680.9791870117188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.7362096309661865, + "rewards/margins": 52.485355377197266, + "rewards/rejected": -51.749141693115234, + "step": 586 + }, + { + "epoch": 0.64, + "learning_rate": 1.7017241379310345e-05, + "logits/chosen": -0.3287639617919922, + "logits/rejected": -1.1477540731430054, + "logps/chosen": -367.2208251953125, + "logps/rejected": -636.5546264648438, + "loss": 0.0023, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.831266403198242, + "rewards/margins": 45.8513069152832, + "rewards/rejected": -50.68257522583008, + "step": 587 + }, + { + "epoch": 0.64, + "learning_rate": 1.696551724137931e-05, + "logits/chosen": -0.2563868463039398, + "logits/rejected": -1.21188485622406, + "logps/chosen": -340.62530517578125, + "logps/rejected": -597.3062744140625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.9416985511779785, + "rewards/margins": 44.73276901245117, + "rewards/rejected": -49.67446517944336, + "step": 588 + }, + { + "epoch": 0.64, + "learning_rate": 1.6913793103448274e-05, + "logits/chosen": -0.06057710573077202, + "logits/rejected": -1.5127232074737549, + "logps/chosen": -515.742919921875, + "logps/rejected": -595.0543212890625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.528551697731018, + "rewards/margins": 50.152740478515625, + "rewards/rejected": -51.68128967285156, + "step": 589 + }, + { + "epoch": 0.64, + "learning_rate": 1.6862068965517242e-05, + "logits/chosen": -0.2441636025905609, + "logits/rejected": -1.4141565561294556, + "logps/chosen": -401.9132995605469, + "logps/rejected": -682.8711547851562, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.38480621576309204, + "rewards/margins": 52.949195861816406, + "rewards/rejected": -52.56439208984375, + "step": 590 + }, + { + "epoch": 0.64, + "learning_rate": 1.6810344827586207e-05, + "logits/chosen": -0.19429653882980347, + "logits/rejected": -1.3483132123947144, + "logps/chosen": -394.2885437011719, + "logps/rejected": -612.275634765625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.2922842502593994, + "rewards/margins": 52.64678955078125, + "rewards/rejected": -52.9390754699707, + "step": 591 + }, + { + "epoch": 0.65, + "learning_rate": 1.6758620689655175e-05, + "logits/chosen": -0.1201917976140976, + "logits/rejected": -1.3542145490646362, + "logps/chosen": -425.7287292480469, + "logps/rejected": -636.2640991210938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.1215896606445312, + "rewards/margins": 54.03361892700195, + "rewards/rejected": -50.912025451660156, + "step": 592 + }, + { + "epoch": 0.65, + "learning_rate": 1.670689655172414e-05, + "logits/chosen": -0.2743023931980133, + "logits/rejected": -1.4589341878890991, + "logps/chosen": -355.8106689453125, + "logps/rejected": -547.2426147460938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.09041150659322739, + "rewards/margins": 46.82219314575195, + "rewards/rejected": -46.731781005859375, + "step": 593 + }, + { + "epoch": 0.65, + "learning_rate": 1.6655172413793104e-05, + "logits/chosen": -0.2365323305130005, + "logits/rejected": -1.5025275945663452, + "logps/chosen": -385.47705078125, + "logps/rejected": -538.5193481445312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.30189573764801025, + "rewards/margins": 46.392642974853516, + "rewards/rejected": -46.69453811645508, + "step": 594 + }, + { + "epoch": 0.65, + "learning_rate": 1.6603448275862072e-05, + "logits/chosen": -0.1523529291152954, + "logits/rejected": -1.3015763759613037, + "logps/chosen": -412.3084716796875, + "logps/rejected": -591.6860961914062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.2983438968658447, + "rewards/margins": 45.98234558105469, + "rewards/rejected": -46.28068542480469, + "step": 595 + }, + { + "epoch": 0.65, + "learning_rate": 1.6551724137931037e-05, + "logits/chosen": -0.0005629090592265129, + "logits/rejected": -1.2052149772644043, + "logps/chosen": -383.7427673339844, + "logps/rejected": -578.141357421875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.46424204111099243, + "rewards/margins": 48.80424499511719, + "rewards/rejected": -48.34000778198242, + "step": 596 + }, + { + "epoch": 0.65, + "learning_rate": 1.65e-05, + "logits/chosen": -0.14516018331050873, + "logits/rejected": -1.4024372100830078, + "logps/chosen": -506.8284912109375, + "logps/rejected": -537.9654541015625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.7340108156204224, + "rewards/margins": 46.76782989501953, + "rewards/rejected": -46.033817291259766, + "step": 597 + }, + { + "epoch": 0.65, + "learning_rate": 1.6448275862068966e-05, + "logits/chosen": -0.13159377872943878, + "logits/rejected": -1.1646668910980225, + "logps/chosen": -351.59735107421875, + "logps/rejected": -531.7355346679688, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.1915807723999023, + "rewards/margins": 41.97750473022461, + "rewards/rejected": -44.169090270996094, + "step": 598 + }, + { + "epoch": 0.65, + "learning_rate": 1.639655172413793e-05, + "logits/chosen": -0.12550602853298187, + "logits/rejected": -1.2537415027618408, + "logps/chosen": -360.52362060546875, + "logps/rejected": -534.72265625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.4122576713562012, + "rewards/margins": 43.383888244628906, + "rewards/rejected": -41.97162628173828, + "step": 599 + }, + { + "epoch": 0.65, + "learning_rate": 1.63448275862069e-05, + "logits/chosen": -0.35067713260650635, + "logits/rejected": -1.285046100616455, + "logps/chosen": -342.4931640625, + "logps/rejected": -598.63330078125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.16610080003738403, + "rewards/margins": 50.52566909790039, + "rewards/rejected": -50.35956573486328, + "step": 600 + }, + { + "epoch": 0.66, + "learning_rate": 1.6293103448275863e-05, + "logits/chosen": -0.05313819646835327, + "logits/rejected": -1.6483476161956787, + "logps/chosen": -498.56707763671875, + "logps/rejected": -560.79541015625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.678995609283447, + "rewards/margins": 56.548095703125, + "rewards/rejected": -48.86909866333008, + "step": 601 + }, + { + "epoch": 0.66, + "learning_rate": 1.6241379310344828e-05, + "logits/chosen": -0.00626419298350811, + "logits/rejected": -1.4823583364486694, + "logps/chosen": -574.1875, + "logps/rejected": -544.60302734375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.469341278076172, + "rewards/margins": 46.32347106933594, + "rewards/rejected": -43.8541259765625, + "step": 602 + }, + { + "epoch": 0.66, + "learning_rate": 1.6189655172413796e-05, + "logits/chosen": -0.2479826658964157, + "logits/rejected": -1.2744426727294922, + "logps/chosen": -427.8579406738281, + "logps/rejected": -573.8703002929688, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.790971040725708, + "rewards/margins": 44.90638732910156, + "rewards/rejected": -43.11541748046875, + "step": 603 + }, + { + "epoch": 0.66, + "learning_rate": 1.613793103448276e-05, + "logits/chosen": -0.22728334367275238, + "logits/rejected": -1.0308204889297485, + "logps/chosen": -311.8193359375, + "logps/rejected": -536.7919311523438, + "loss": 0.0001, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.5045533180236816, + "rewards/margins": 35.646881103515625, + "rewards/rejected": -39.151432037353516, + "step": 604 + }, + { + "epoch": 0.66, + "learning_rate": 1.6086206896551725e-05, + "logits/chosen": -0.5185431838035583, + "logits/rejected": -1.3834552764892578, + "logps/chosen": -269.7243957519531, + "logps/rejected": -575.7864379882812, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.5515775680541992, + "rewards/margins": 45.755775451660156, + "rewards/rejected": -46.30735397338867, + "step": 605 + }, + { + "epoch": 0.66, + "learning_rate": 1.603448275862069e-05, + "logits/chosen": -0.2218950241804123, + "logits/rejected": -1.3695735931396484, + "logps/chosen": -407.8547058105469, + "logps/rejected": -541.52099609375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.885260581970215, + "rewards/margins": 42.87828826904297, + "rewards/rejected": -45.763553619384766, + "step": 606 + }, + { + "epoch": 0.66, + "learning_rate": 1.5982758620689654e-05, + "logits/chosen": -0.0793599933385849, + "logits/rejected": -1.3909116983413696, + "logps/chosen": -479.3802490234375, + "logps/rejected": -564.6279296875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.8253229856491089, + "rewards/margins": 46.38885498046875, + "rewards/rejected": -45.563533782958984, + "step": 607 + }, + { + "epoch": 0.66, + "learning_rate": 1.5931034482758622e-05, + "logits/chosen": -0.2918356955051422, + "logits/rejected": -1.4191017150878906, + "logps/chosen": -426.25250244140625, + "logps/rejected": -561.0745849609375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.03518211841583252, + "rewards/margins": 48.18317413330078, + "rewards/rejected": -48.218353271484375, + "step": 608 + }, + { + "epoch": 0.66, + "learning_rate": 1.5879310344827587e-05, + "logits/chosen": -0.02587204799056053, + "logits/rejected": -1.6397631168365479, + "logps/chosen": -512.5316772460938, + "logps/rejected": -578.6962890625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 6.294554233551025, + "rewards/margins": 54.153038024902344, + "rewards/rejected": -47.85848617553711, + "step": 609 + }, + { + "epoch": 0.67, + "learning_rate": 1.582758620689655e-05, + "logits/chosen": -0.1111781969666481, + "logits/rejected": -1.660740852355957, + "logps/chosen": -539.9187622070312, + "logps/rejected": -529.7341918945312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 5.2654643058776855, + "rewards/margins": 49.64447021484375, + "rewards/rejected": -44.37900161743164, + "step": 610 + }, + { + "epoch": 0.67, + "learning_rate": 1.577586206896552e-05, + "logits/chosen": -0.34613269567489624, + "logits/rejected": -1.256672739982605, + "logps/chosen": -286.6711730957031, + "logps/rejected": -601.25927734375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.5742961168289185, + "rewards/margins": 49.86055374145508, + "rewards/rejected": -48.286258697509766, + "step": 611 + }, + { + "epoch": 0.67, + "learning_rate": 1.5724137931034484e-05, + "logits/chosen": 0.08210857212543488, + "logits/rejected": -1.394779920578003, + "logps/chosen": -454.6331481933594, + "logps/rejected": -548.888671875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.741332530975342, + "rewards/margins": 50.020511627197266, + "rewards/rejected": -47.279178619384766, + "step": 612 + }, + { + "epoch": 0.67, + "learning_rate": 1.567241379310345e-05, + "logits/chosen": -0.14018212258815765, + "logits/rejected": -1.1777931451797485, + "logps/chosen": -330.23046875, + "logps/rejected": -555.2692260742188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.9217461347579956, + "rewards/margins": 44.80604934692383, + "rewards/rejected": -46.7277946472168, + "step": 613 + }, + { + "epoch": 0.67, + "learning_rate": 1.5620689655172417e-05, + "logits/chosen": -0.21515248715877533, + "logits/rejected": -1.533896565437317, + "logps/chosen": -451.9747314453125, + "logps/rejected": -555.5947265625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.842181444168091, + "rewards/margins": 48.474666595458984, + "rewards/rejected": -45.63249206542969, + "step": 614 + }, + { + "epoch": 0.67, + "learning_rate": 1.556896551724138e-05, + "logits/chosen": -0.1645764410495758, + "logits/rejected": -1.1189565658569336, + "logps/chosen": -305.49365234375, + "logps/rejected": -509.23590087890625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.9992731809616089, + "rewards/margins": 41.10477066040039, + "rewards/rejected": -43.104042053222656, + "step": 615 + }, + { + "epoch": 0.67, + "learning_rate": 1.5517241379310346e-05, + "logits/chosen": -0.04968830943107605, + "logits/rejected": -1.4535218477249146, + "logps/chosen": -488.84185791015625, + "logps/rejected": -516.9566040039062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.233278751373291, + "rewards/margins": 44.68144226074219, + "rewards/rejected": -40.44816970825195, + "step": 616 + }, + { + "epoch": 0.67, + "learning_rate": 1.546551724137931e-05, + "logits/chosen": -0.1743217259645462, + "logits/rejected": -1.4794538021087646, + "logps/chosen": -550.0068969726562, + "logps/rejected": -529.5560913085938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.273037910461426, + "rewards/margins": 44.289337158203125, + "rewards/rejected": -42.01629638671875, + "step": 617 + }, + { + "epoch": 0.67, + "learning_rate": 1.5413793103448275e-05, + "logits/chosen": -0.16211533546447754, + "logits/rejected": -1.1423048973083496, + "logps/chosen": -349.4170227050781, + "logps/rejected": -523.6246948242188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.9724212288856506, + "rewards/margins": 48.36107635498047, + "rewards/rejected": -47.38865280151367, + "step": 618 + }, + { + "epoch": 0.68, + "learning_rate": 1.5362068965517243e-05, + "logits/chosen": -0.15734046697616577, + "logits/rejected": -1.2859727144241333, + "logps/chosen": -335.3069763183594, + "logps/rejected": -533.359130859375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.9588116407394409, + "rewards/margins": 42.21016311645508, + "rewards/rejected": -43.16897964477539, + "step": 619 + }, + { + "epoch": 0.68, + "learning_rate": 1.5310344827586208e-05, + "logits/chosen": -0.1451762467622757, + "logits/rejected": -1.2424870729446411, + "logps/chosen": -388.3422546386719, + "logps/rejected": -519.1555786132812, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18462783098220825, + "rewards/margins": 45.364524841308594, + "rewards/rejected": -45.54915237426758, + "step": 620 + }, + { + "epoch": 0.68, + "learning_rate": 1.5258620689655172e-05, + "logits/chosen": -0.033223796635866165, + "logits/rejected": -1.55014967918396, + "logps/chosen": -621.535400390625, + "logps/rejected": -508.3518371582031, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.288414478302002, + "rewards/margins": 45.626564025878906, + "rewards/rejected": -41.33815002441406, + "step": 621 + }, + { + "epoch": 0.68, + "learning_rate": 1.5206896551724139e-05, + "logits/chosen": -0.28730061650276184, + "logits/rejected": -1.3408746719360352, + "logps/chosen": -420.8293762207031, + "logps/rejected": -494.9945068359375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.09170261025428772, + "rewards/margins": 42.52814865112305, + "rewards/rejected": -42.43645095825195, + "step": 622 + }, + { + "epoch": 0.68, + "learning_rate": 1.5155172413793103e-05, + "logits/chosen": -0.08658464252948761, + "logits/rejected": -1.4125157594680786, + "logps/chosen": -446.3509521484375, + "logps/rejected": -545.7614135742188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2800441086292267, + "rewards/margins": 45.323646545410156, + "rewards/rejected": -45.043601989746094, + "step": 623 + }, + { + "epoch": 0.68, + "learning_rate": 1.5103448275862071e-05, + "logits/chosen": -0.2937692403793335, + "logits/rejected": -1.3026137351989746, + "logps/chosen": -320.6386413574219, + "logps/rejected": -528.2933959960938, + "loss": 0.0001, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.7214214205741882, + "rewards/margins": 44.59417724609375, + "rewards/rejected": -43.87274932861328, + "step": 624 + }, + { + "epoch": 0.68, + "learning_rate": 1.5051724137931036e-05, + "logits/chosen": -0.15354903042316437, + "logits/rejected": -1.4917157888412476, + "logps/chosen": -394.09259033203125, + "logps/rejected": -537.6986694335938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.2884288728237152, + "rewards/margins": 45.02061080932617, + "rewards/rejected": -45.30903625488281, + "step": 625 + }, + { + "epoch": 0.68, + "learning_rate": 1.5e-05, + "logits/chosen": -0.0401911586523056, + "logits/rejected": -1.4450793266296387, + "logps/chosen": -405.8299255371094, + "logps/rejected": -544.3627319335938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.1770844459533691, + "rewards/margins": 47.842506408691406, + "rewards/rejected": -46.66542053222656, + "step": 626 + }, + { + "epoch": 0.68, + "learning_rate": 1.4948275862068967e-05, + "logits/chosen": -0.08096438646316528, + "logits/rejected": -1.5375018119812012, + "logps/chosen": -469.0930480957031, + "logps/rejected": -517.4960327148438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.0593535900115967, + "rewards/margins": 44.87061309814453, + "rewards/rejected": -41.811256408691406, + "step": 627 + }, + { + "epoch": 0.69, + "learning_rate": 1.4896551724137931e-05, + "logits/chosen": -0.10965342819690704, + "logits/rejected": -1.202269434928894, + "logps/chosen": -384.6773681640625, + "logps/rejected": -531.308349609375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.8238025903701782, + "rewards/margins": 43.14710235595703, + "rewards/rejected": -42.32329177856445, + "step": 628 + }, + { + "epoch": 0.69, + "learning_rate": 1.4844827586206896e-05, + "logits/chosen": -0.06810637563467026, + "logits/rejected": -1.3470923900604248, + "logps/chosen": -437.27947998046875, + "logps/rejected": -509.29443359375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.51701545715332, + "rewards/margins": 46.15340042114258, + "rewards/rejected": -41.636383056640625, + "step": 629 + }, + { + "epoch": 0.69, + "learning_rate": 1.4793103448275864e-05, + "logits/chosen": -0.2248406708240509, + "logits/rejected": -1.3975605964660645, + "logps/chosen": -341.30010986328125, + "logps/rejected": -519.501953125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.0808305740356445, + "rewards/margins": 40.74700164794922, + "rewards/rejected": -42.82783126831055, + "step": 630 + }, + { + "epoch": 0.69, + "learning_rate": 1.4741379310344829e-05, + "logits/chosen": -0.1267438530921936, + "logits/rejected": -1.2797744274139404, + "logps/chosen": -402.70172119140625, + "logps/rejected": -568.7130737304688, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3171030580997467, + "rewards/margins": 46.492828369140625, + "rewards/rejected": -46.17572021484375, + "step": 631 + }, + { + "epoch": 0.69, + "learning_rate": 1.4689655172413795e-05, + "logits/chosen": -0.27014362812042236, + "logits/rejected": -1.4330971240997314, + "logps/chosen": -315.55706787109375, + "logps/rejected": -537.5396118164062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.189436912536621, + "rewards/margins": 47.06514358520508, + "rewards/rejected": -43.875709533691406, + "step": 632 + }, + { + "epoch": 0.69, + "learning_rate": 1.463793103448276e-05, + "logits/chosen": -0.3762741684913635, + "logits/rejected": -1.147342324256897, + "logps/chosen": -306.3317565917969, + "logps/rejected": -525.963623046875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08659607172012329, + "rewards/margins": 43.48544692993164, + "rewards/rejected": -43.572044372558594, + "step": 633 + }, + { + "epoch": 0.69, + "learning_rate": 1.4586206896551724e-05, + "logits/chosen": -0.24545836448669434, + "logits/rejected": -1.2215213775634766, + "logps/chosen": -313.8126220703125, + "logps/rejected": -523.89111328125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.9955374598503113, + "rewards/margins": 42.0698127746582, + "rewards/rejected": -43.06534957885742, + "step": 634 + }, + { + "epoch": 0.69, + "learning_rate": 1.4534482758620692e-05, + "logits/chosen": -0.3359462022781372, + "logits/rejected": -1.3295055627822876, + "logps/chosen": -422.6155090332031, + "logps/rejected": -530.6246337890625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5714225769042969, + "rewards/margins": 41.921669006347656, + "rewards/rejected": -41.350242614746094, + "step": 635 + }, + { + "epoch": 0.69, + "learning_rate": 1.4482758620689657e-05, + "logits/chosen": -0.13072825968265533, + "logits/rejected": -1.5437922477722168, + "logps/chosen": -520.2494506835938, + "logps/rejected": -502.7685546875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.3034181594848633, + "rewards/margins": 44.092994689941406, + "rewards/rejected": -40.789573669433594, + "step": 636 + }, + { + "epoch": 0.69, + "learning_rate": 1.4431034482758621e-05, + "logits/chosen": -0.04517197981476784, + "logits/rejected": -1.6300750970840454, + "logps/chosen": -465.6365966796875, + "logps/rejected": -516.446533203125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.421183109283447, + "rewards/margins": 47.232208251953125, + "rewards/rejected": -42.81102752685547, + "step": 637 + }, + { + "epoch": 0.7, + "learning_rate": 1.4379310344827588e-05, + "logits/chosen": -0.015193352475762367, + "logits/rejected": -1.5817826986312866, + "logps/chosen": -502.16845703125, + "logps/rejected": -550.2406005859375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.9052822589874268, + "rewards/margins": 49.294677734375, + "rewards/rejected": -46.38939666748047, + "step": 638 + }, + { + "epoch": 0.7, + "learning_rate": 1.4327586206896552e-05, + "logits/chosen": -0.31254029273986816, + "logits/rejected": -1.3829731941223145, + "logps/chosen": -282.6502380371094, + "logps/rejected": -471.7291564941406, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.455610752105713, + "rewards/margins": 39.598594665527344, + "rewards/rejected": -41.05420684814453, + "step": 639 + }, + { + "epoch": 0.7, + "learning_rate": 1.4275862068965518e-05, + "logits/chosen": -0.022501356899738312, + "logits/rejected": -1.5619714260101318, + "logps/chosen": -438.0868835449219, + "logps/rejected": -462.1986389160156, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.933338165283203, + "rewards/margins": 44.02751541137695, + "rewards/rejected": -40.09416961669922, + "step": 640 + }, + { + "epoch": 0.7, + "learning_rate": 1.4224137931034483e-05, + "logits/chosen": -0.25200387835502625, + "logits/rejected": -1.3560858964920044, + "logps/chosen": -398.2036437988281, + "logps/rejected": -535.743408203125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.0920121669769287, + "rewards/margins": 46.25288391113281, + "rewards/rejected": -45.16087341308594, + "step": 641 + }, + { + "epoch": 0.7, + "learning_rate": 1.4172413793103448e-05, + "logits/chosen": -0.08297184109687805, + "logits/rejected": -1.4250469207763672, + "logps/chosen": -503.2852478027344, + "logps/rejected": -576.9325561523438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.053221702575684, + "rewards/margins": 50.68095779418945, + "rewards/rejected": -46.62773513793945, + "step": 642 + }, + { + "epoch": 0.7, + "learning_rate": 1.4120689655172416e-05, + "logits/chosen": 0.031028956174850464, + "logits/rejected": -1.5391318798065186, + "logps/chosen": -506.672119140625, + "logps/rejected": -528.0322875976562, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 5.037855625152588, + "rewards/margins": 50.48542022705078, + "rewards/rejected": -45.44756317138672, + "step": 643 + }, + { + "epoch": 0.7, + "learning_rate": 1.406896551724138e-05, + "logits/chosen": -0.303537517786026, + "logits/rejected": -1.3789228200912476, + "logps/chosen": -358.866455078125, + "logps/rejected": -512.6773071289062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.4551244378089905, + "rewards/margins": 42.338409423828125, + "rewards/rejected": -42.79353332519531, + "step": 644 + }, + { + "epoch": 0.7, + "learning_rate": 1.4017241379310345e-05, + "logits/chosen": -0.2305237501859665, + "logits/rejected": -1.196969747543335, + "logps/chosen": -333.74456787109375, + "logps/rejected": -562.4696655273438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.2080453634262085, + "rewards/margins": 43.72755813598633, + "rewards/rejected": -44.935604095458984, + "step": 645 + }, + { + "epoch": 0.7, + "learning_rate": 1.3965517241379311e-05, + "logits/chosen": -0.28576573729515076, + "logits/rejected": -1.430737853050232, + "logps/chosen": -381.3027648925781, + "logps/rejected": -523.885986328125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.6614712476730347, + "rewards/margins": 40.87557601928711, + "rewards/rejected": -41.53704833984375, + "step": 646 + }, + { + "epoch": 0.71, + "learning_rate": 1.3913793103448276e-05, + "logits/chosen": -0.04188717156648636, + "logits/rejected": -1.3778163194656372, + "logps/chosen": -506.2252502441406, + "logps/rejected": -551.3240966796875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.9459335803985596, + "rewards/margins": 48.985408782958984, + "rewards/rejected": -46.03947448730469, + "step": 647 + }, + { + "epoch": 0.71, + "learning_rate": 1.3862068965517244e-05, + "logits/chosen": -0.27598288655281067, + "logits/rejected": -1.0301058292388916, + "logps/chosen": -285.4984436035156, + "logps/rejected": -535.9169311523438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.29662734270095825, + "rewards/margins": 42.88330841064453, + "rewards/rejected": -43.179931640625, + "step": 648 + }, + { + "epoch": 0.71, + "learning_rate": 1.3810344827586208e-05, + "logits/chosen": -0.1300501823425293, + "logits/rejected": -1.5005356073379517, + "logps/chosen": -425.9429626464844, + "logps/rejected": -479.8569641113281, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.6732308864593506, + "rewards/margins": 41.68931198120117, + "rewards/rejected": -39.016082763671875, + "step": 649 + }, + { + "epoch": 0.71, + "learning_rate": 1.3758620689655173e-05, + "logits/chosen": -0.04178273305296898, + "logits/rejected": -1.523526906967163, + "logps/chosen": -423.0726318359375, + "logps/rejected": -539.8958129882812, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.544985771179199, + "rewards/margins": 47.55220031738281, + "rewards/rejected": -45.0072135925293, + "step": 650 + }, + { + "epoch": 0.71, + "learning_rate": 1.370689655172414e-05, + "logits/chosen": -0.1655365526676178, + "logits/rejected": -1.5684573650360107, + "logps/chosen": -430.7422790527344, + "logps/rejected": -489.2760009765625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.3517260551452637, + "rewards/margins": 44.93771743774414, + "rewards/rejected": -41.585994720458984, + "step": 651 + }, + { + "epoch": 0.71, + "learning_rate": 1.3655172413793104e-05, + "logits/chosen": -0.2269890308380127, + "logits/rejected": -1.4303154945373535, + "logps/chosen": -358.328369140625, + "logps/rejected": -570.949951171875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.13788434863090515, + "rewards/margins": 47.54113006591797, + "rewards/rejected": -47.40325164794922, + "step": 652 + }, + { + "epoch": 0.71, + "learning_rate": 1.3603448275862069e-05, + "logits/chosen": -0.39984795451164246, + "logits/rejected": -1.350335955619812, + "logps/chosen": -280.1457824707031, + "logps/rejected": -507.47808837890625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.025997459888458252, + "rewards/margins": 41.7269401550293, + "rewards/rejected": -41.75293731689453, + "step": 653 + }, + { + "epoch": 0.71, + "learning_rate": 1.3551724137931037e-05, + "logits/chosen": -0.3223649561405182, + "logits/rejected": -1.3828343152999878, + "logps/chosen": -334.5537414550781, + "logps/rejected": -506.3958435058594, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.21468442678451538, + "rewards/margins": 44.35294723510742, + "rewards/rejected": -44.13825988769531, + "step": 654 + }, + { + "epoch": 0.71, + "learning_rate": 1.3500000000000001e-05, + "logits/chosen": -0.3293633759021759, + "logits/rejected": -1.333666443824768, + "logps/chosen": -323.5306701660156, + "logps/rejected": -511.315185546875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.06329803913831711, + "rewards/margins": 42.38307571411133, + "rewards/rejected": -42.446372985839844, + "step": 655 + }, + { + "epoch": 0.72, + "learning_rate": 1.3448275862068967e-05, + "logits/chosen": -0.11880247294902802, + "logits/rejected": -1.29387629032135, + "logps/chosen": -400.0235900878906, + "logps/rejected": -535.8289794921875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5334283709526062, + "rewards/margins": 44.15065383911133, + "rewards/rejected": -43.61722946166992, + "step": 656 + }, + { + "epoch": 0.72, + "learning_rate": 1.3396551724137932e-05, + "logits/chosen": -0.15652622282505035, + "logits/rejected": -1.434822916984558, + "logps/chosen": -408.1186218261719, + "logps/rejected": -553.8369140625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.40162843465805054, + "rewards/margins": 47.67634582519531, + "rewards/rejected": -47.27471160888672, + "step": 657 + }, + { + "epoch": 0.72, + "learning_rate": 1.3344827586206897e-05, + "logits/chosen": -0.0915599837899208, + "logits/rejected": -1.5216375589370728, + "logps/chosen": -477.429443359375, + "logps/rejected": -531.134521484375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.0920250415802, + "rewards/margins": 45.700225830078125, + "rewards/rejected": -43.60820007324219, + "step": 658 + }, + { + "epoch": 0.72, + "learning_rate": 1.3293103448275863e-05, + "logits/chosen": -0.1342640519142151, + "logits/rejected": -1.3003381490707397, + "logps/chosen": -340.65655517578125, + "logps/rejected": -537.4246215820312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.2339905500411987, + "rewards/margins": 45.6250114440918, + "rewards/rejected": -46.8590087890625, + "step": 659 + }, + { + "epoch": 0.72, + "learning_rate": 1.3241379310344828e-05, + "logits/chosen": -0.2628743648529053, + "logits/rejected": -1.5235884189605713, + "logps/chosen": -385.9111633300781, + "logps/rejected": -488.7979431152344, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.9308697581291199, + "rewards/margins": 40.1331901550293, + "rewards/rejected": -41.06406021118164, + "step": 660 + }, + { + "epoch": 0.72, + "learning_rate": 1.3189655172413792e-05, + "logits/chosen": -0.35871365666389465, + "logits/rejected": -1.2679202556610107, + "logps/chosen": -331.3196716308594, + "logps/rejected": -539.1953125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.141901969909668, + "rewards/margins": 39.91340255737305, + "rewards/rejected": -41.05530548095703, + "step": 661 + }, + { + "epoch": 0.72, + "learning_rate": 1.313793103448276e-05, + "logits/chosen": -0.10699907690286636, + "logits/rejected": -1.1208184957504272, + "logps/chosen": -307.62115478515625, + "logps/rejected": -582.8983154296875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.7876830101013184, + "rewards/margins": 46.260215759277344, + "rewards/rejected": -47.04790115356445, + "step": 662 + }, + { + "epoch": 0.72, + "learning_rate": 1.3086206896551725e-05, + "logits/chosen": -0.12584511935710907, + "logits/rejected": -1.3415498733520508, + "logps/chosen": -445.26910400390625, + "logps/rejected": -551.9942016601562, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.583847999572754, + "rewards/margins": 46.69656753540039, + "rewards/rejected": -43.11271667480469, + "step": 663 + }, + { + "epoch": 0.72, + "learning_rate": 1.3034482758620691e-05, + "logits/chosen": -0.039403751492500305, + "logits/rejected": -1.5524853467941284, + "logps/chosen": -431.6235656738281, + "logps/rejected": -548.6947021484375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.0976274013519287, + "rewards/margins": 46.144187927246094, + "rewards/rejected": -48.2418098449707, + "step": 664 + }, + { + "epoch": 0.73, + "learning_rate": 1.2982758620689656e-05, + "logits/chosen": -0.07293981313705444, + "logits/rejected": -1.555285930633545, + "logps/chosen": -477.1157531738281, + "logps/rejected": -531.8021240234375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.3638932704925537, + "rewards/margins": 47.931007385253906, + "rewards/rejected": -44.567115783691406, + "step": 665 + }, + { + "epoch": 0.73, + "learning_rate": 1.293103448275862e-05, + "logits/chosen": -0.11820126324892044, + "logits/rejected": -1.6684420108795166, + "logps/chosen": -517.3621215820312, + "logps/rejected": -509.4004211425781, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.188424587249756, + "rewards/margins": 45.912174224853516, + "rewards/rejected": -42.723751068115234, + "step": 666 + }, + { + "epoch": 0.73, + "learning_rate": 1.2879310344827588e-05, + "logits/chosen": -0.05734536051750183, + "logits/rejected": -1.5235246419906616, + "logps/chosen": -592.1263427734375, + "logps/rejected": -585.6410522460938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 8.330058097839355, + "rewards/margins": 54.89848327636719, + "rewards/rejected": -46.56842041015625, + "step": 667 + }, + { + "epoch": 0.73, + "learning_rate": 1.2827586206896553e-05, + "logits/chosen": -0.23646149039268494, + "logits/rejected": -1.4570469856262207, + "logps/chosen": -388.5771484375, + "logps/rejected": -562.6555786132812, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.438934803009033, + "rewards/margins": 48.871097564697266, + "rewards/rejected": -46.432159423828125, + "step": 668 + }, + { + "epoch": 0.73, + "learning_rate": 1.2775862068965518e-05, + "logits/chosen": -0.277557909488678, + "logits/rejected": -1.520776391029358, + "logps/chosen": -431.8119812011719, + "logps/rejected": -535.2166748046875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.241364002227783, + "rewards/margins": 47.845359802246094, + "rewards/rejected": -45.60399627685547, + "step": 669 + }, + { + "epoch": 0.73, + "learning_rate": 1.2724137931034484e-05, + "logits/chosen": -0.37299636006355286, + "logits/rejected": -1.433702826499939, + "logps/chosen": -367.0413513183594, + "logps/rejected": -529.3248901367188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.2390081882476807, + "rewards/margins": 44.13026428222656, + "rewards/rejected": -41.89125442504883, + "step": 670 + }, + { + "epoch": 0.73, + "learning_rate": 1.2672413793103449e-05, + "logits/chosen": -0.135738343000412, + "logits/rejected": -1.4803038835525513, + "logps/chosen": -486.78131103515625, + "logps/rejected": -586.236572265625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.0751945972442627, + "rewards/margins": 44.44581604003906, + "rewards/rejected": -45.52101516723633, + "step": 671 + }, + { + "epoch": 0.73, + "learning_rate": 1.2620689655172417e-05, + "logits/chosen": -0.37156784534454346, + "logits/rejected": -1.2309331893920898, + "logps/chosen": -310.4812927246094, + "logps/rejected": -520.0112915039062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.6325098276138306, + "rewards/margins": 42.75358963012695, + "rewards/rejected": -41.12108612060547, + "step": 672 + }, + { + "epoch": 0.73, + "learning_rate": 1.256896551724138e-05, + "logits/chosen": -0.20378383994102478, + "logits/rejected": -1.3526654243469238, + "logps/chosen": -379.8397521972656, + "logps/rejected": -542.7678833007812, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.5514822006225586, + "rewards/margins": 43.52738571166992, + "rewards/rejected": -44.07886505126953, + "step": 673 + }, + { + "epoch": 0.74, + "learning_rate": 1.2517241379310344e-05, + "logits/chosen": -0.07714121043682098, + "logits/rejected": -1.528071403503418, + "logps/chosen": -472.7803955078125, + "logps/rejected": -530.1329345703125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 5.302211284637451, + "rewards/margins": 50.1279411315918, + "rewards/rejected": -44.82572555541992, + "step": 674 + }, + { + "epoch": 0.74, + "learning_rate": 1.2465517241379312e-05, + "logits/chosen": -0.03753519058227539, + "logits/rejected": -1.259669303894043, + "logps/chosen": -373.8651123046875, + "logps/rejected": -540.652099609375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.7355949878692627, + "rewards/margins": 44.318416595458984, + "rewards/rejected": -43.58282470703125, + "step": 675 + }, + { + "epoch": 0.74, + "learning_rate": 1.2413793103448277e-05, + "logits/chosen": -0.4580439329147339, + "logits/rejected": -1.498665690422058, + "logps/chosen": -304.34051513671875, + "logps/rejected": -517.0997314453125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.3485918045043945, + "rewards/margins": 46.20613098144531, + "rewards/rejected": -42.8575439453125, + "step": 676 + }, + { + "epoch": 0.74, + "learning_rate": 1.2362068965517241e-05, + "logits/chosen": -0.08015082776546478, + "logits/rejected": -1.5275059938430786, + "logps/chosen": -477.7775573730469, + "logps/rejected": -539.4383544921875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.337691307067871, + "rewards/margins": 48.43022155761719, + "rewards/rejected": -45.092533111572266, + "step": 677 + }, + { + "epoch": 0.74, + "learning_rate": 1.2310344827586208e-05, + "logits/chosen": -0.07246221601963043, + "logits/rejected": -1.1152292490005493, + "logps/chosen": -283.9908752441406, + "logps/rejected": -551.6624755859375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.0209674835205078, + "rewards/margins": 47.72657012939453, + "rewards/rejected": -46.70560073852539, + "step": 678 + }, + { + "epoch": 0.74, + "learning_rate": 1.2258620689655172e-05, + "logits/chosen": -0.17554022371768951, + "logits/rejected": -1.381176471710205, + "logps/chosen": -391.78155517578125, + "logps/rejected": -587.4243774414062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.602041721343994, + "rewards/margins": 51.59418869018555, + "rewards/rejected": -48.99215316772461, + "step": 679 + }, + { + "epoch": 0.74, + "learning_rate": 1.220689655172414e-05, + "logits/chosen": -0.17825302481651306, + "logits/rejected": -1.2341221570968628, + "logps/chosen": -376.4874267578125, + "logps/rejected": -495.99993896484375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.1336874961853027, + "rewards/margins": 40.05615997314453, + "rewards/rejected": -38.92247009277344, + "step": 680 + }, + { + "epoch": 0.74, + "learning_rate": 1.2155172413793105e-05, + "logits/chosen": -0.26450762152671814, + "logits/rejected": -1.11971914768219, + "logps/chosen": -328.3924255371094, + "logps/rejected": -612.40673828125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.4581549167633057, + "rewards/margins": 45.34051513671875, + "rewards/rejected": -46.798667907714844, + "step": 681 + }, + { + "epoch": 0.74, + "learning_rate": 1.210344827586207e-05, + "logits/chosen": -0.21210908889770508, + "logits/rejected": -1.5598255395889282, + "logps/chosen": -469.20172119140625, + "logps/rejected": -538.340087890625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 6.307070732116699, + "rewards/margins": 50.93186950683594, + "rewards/rejected": -44.62479782104492, + "step": 682 + }, + { + "epoch": 0.75, + "learning_rate": 1.2051724137931036e-05, + "logits/chosen": -0.1790524572134018, + "logits/rejected": -1.5198571681976318, + "logps/chosen": -408.3398132324219, + "logps/rejected": -512.6113891601562, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.877450704574585, + "rewards/margins": 47.00334167480469, + "rewards/rejected": -44.125892639160156, + "step": 683 + }, + { + "epoch": 0.75, + "learning_rate": 1.2e-05, + "logits/chosen": -0.08723405003547668, + "logits/rejected": -1.3432382345199585, + "logps/chosen": -401.7087707519531, + "logps/rejected": -542.4974975585938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.334695339202881, + "rewards/margins": 46.966678619384766, + "rewards/rejected": -44.631988525390625, + "step": 684 + }, + { + "epoch": 0.75, + "learning_rate": 1.1948275862068965e-05, + "logits/chosen": -0.14494843780994415, + "logits/rejected": -1.2598520517349243, + "logps/chosen": -376.4900817871094, + "logps/rejected": -536.0132446289062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4198228418827057, + "rewards/margins": 41.54653549194336, + "rewards/rejected": -41.126708984375, + "step": 685 + }, + { + "epoch": 0.75, + "learning_rate": 1.1896551724137933e-05, + "logits/chosen": 0.07660377770662308, + "logits/rejected": -1.5494967699050903, + "logps/chosen": -583.0078735351562, + "logps/rejected": -480.3558349609375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 6.250246047973633, + "rewards/margins": 44.277164459228516, + "rewards/rejected": -38.02691650390625, + "step": 686 + }, + { + "epoch": 0.75, + "learning_rate": 1.1844827586206898e-05, + "logits/chosen": -0.38664710521698, + "logits/rejected": -1.5083786249160767, + "logps/chosen": -289.0353698730469, + "logps/rejected": -502.0042419433594, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.9512491226196289, + "rewards/margins": 42.77705383300781, + "rewards/rejected": -43.72829818725586, + "step": 687 + }, + { + "epoch": 0.75, + "learning_rate": 1.1793103448275864e-05, + "logits/chosen": -0.042457547038793564, + "logits/rejected": -1.386946201324463, + "logps/chosen": -455.3371276855469, + "logps/rejected": -545.7581176757812, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.2653028964996338, + "rewards/margins": 46.720619201660156, + "rewards/rejected": -45.45531463623047, + "step": 688 + }, + { + "epoch": 0.75, + "learning_rate": 1.1741379310344828e-05, + "logits/chosen": -0.4539385139942169, + "logits/rejected": -1.2734832763671875, + "logps/chosen": -255.11451721191406, + "logps/rejected": -506.87811279296875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.33655714988708496, + "rewards/margins": 41.31004333496094, + "rewards/rejected": -40.973487854003906, + "step": 689 + }, + { + "epoch": 0.75, + "learning_rate": 1.1689655172413793e-05, + "logits/chosen": -0.2328207790851593, + "logits/rejected": -1.257619023323059, + "logps/chosen": -375.7318115234375, + "logps/rejected": -545.8134765625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.9642629623413086, + "rewards/margins": 43.943382263183594, + "rewards/rejected": -44.90764617919922, + "step": 690 + }, + { + "epoch": 0.75, + "learning_rate": 1.163793103448276e-05, + "logits/chosen": 0.0281175896525383, + "logits/rejected": -1.4880707263946533, + "logps/chosen": -515.5869140625, + "logps/rejected": -544.9164428710938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.243173122406006, + "rewards/margins": 45.39672088623047, + "rewards/rejected": -47.639892578125, + "step": 691 + }, + { + "epoch": 0.75, + "learning_rate": 1.1586206896551724e-05, + "logits/chosen": -0.2516327500343323, + "logits/rejected": -1.316880702972412, + "logps/chosen": -299.51953125, + "logps/rejected": -528.8114624023438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.6539808511734009, + "rewards/margins": 43.63589859008789, + "rewards/rejected": -44.28988265991211, + "step": 692 + }, + { + "epoch": 0.76, + "learning_rate": 1.1534482758620689e-05, + "logits/chosen": -0.00018626078963279724, + "logits/rejected": -1.4975652694702148, + "logps/chosen": -448.61126708984375, + "logps/rejected": -538.84375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.434826374053955, + "rewards/margins": 48.413856506347656, + "rewards/rejected": -45.979034423828125, + "step": 693 + }, + { + "epoch": 0.76, + "learning_rate": 1.1482758620689657e-05, + "logits/chosen": -0.16736753284931183, + "logits/rejected": -1.333862543106079, + "logps/chosen": -464.8886413574219, + "logps/rejected": -554.8616943359375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.5128366947174072, + "rewards/margins": 47.98447036743164, + "rewards/rejected": -46.47163772583008, + "step": 694 + }, + { + "epoch": 0.76, + "learning_rate": 1.1431034482758621e-05, + "logits/chosen": -0.33041834831237793, + "logits/rejected": -1.2662274837493896, + "logps/chosen": -372.60516357421875, + "logps/rejected": -512.045654296875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.0543805360794067, + "rewards/margins": 42.66836166381836, + "rewards/rejected": -41.61397933959961, + "step": 695 + }, + { + "epoch": 0.76, + "learning_rate": 1.1379310344827587e-05, + "logits/chosen": -0.24941149353981018, + "logits/rejected": -1.3205037117004395, + "logps/chosen": -318.0282897949219, + "logps/rejected": -536.583984375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.4145708084106445, + "rewards/margins": 48.180206298828125, + "rewards/rejected": -45.76564025878906, + "step": 696 + }, + { + "epoch": 0.76, + "learning_rate": 1.1327586206896552e-05, + "logits/chosen": -0.1426786631345749, + "logits/rejected": -1.5278555154800415, + "logps/chosen": -491.4092712402344, + "logps/rejected": -490.41351318359375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 6.8129377365112305, + "rewards/margins": 47.30105209350586, + "rewards/rejected": -40.48811721801758, + "step": 697 + }, + { + "epoch": 0.76, + "learning_rate": 1.1275862068965517e-05, + "logits/chosen": -0.2483963519334793, + "logits/rejected": -1.4913365840911865, + "logps/chosen": -435.83770751953125, + "logps/rejected": -544.1882934570312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.600954294204712, + "rewards/margins": 47.01934814453125, + "rewards/rejected": -44.418392181396484, + "step": 698 + }, + { + "epoch": 0.76, + "learning_rate": 1.1224137931034483e-05, + "logits/chosen": -0.24883654713630676, + "logits/rejected": -1.4840902090072632, + "logps/chosen": -385.3328857421875, + "logps/rejected": -508.5963439941406, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.485741376876831, + "rewards/margins": 45.306007385253906, + "rewards/rejected": -42.82026672363281, + "step": 699 + }, + { + "epoch": 0.76, + "learning_rate": 1.117241379310345e-05, + "logits/chosen": -0.26029643416404724, + "logits/rejected": -1.319836139678955, + "logps/chosen": -328.52813720703125, + "logps/rejected": -542.1229858398438, + "loss": 0.0045, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.319135069847107, + "rewards/margins": 40.14945983886719, + "rewards/rejected": -41.468589782714844, + "step": 700 + }, + { + "epoch": 0.76, + "learning_rate": 1.1120689655172414e-05, + "logits/chosen": -0.3635248839855194, + "logits/rejected": -1.3621838092803955, + "logps/chosen": -375.1762390136719, + "logps/rejected": -552.1834716796875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.356740951538086, + "rewards/margins": 45.516422271728516, + "rewards/rejected": -43.1596794128418, + "step": 701 + }, + { + "epoch": 0.77, + "learning_rate": 1.106896551724138e-05, + "logits/chosen": -0.2026928812265396, + "logits/rejected": -1.6469862461090088, + "logps/chosen": -468.78851318359375, + "logps/rejected": -544.870849609375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.385049819946289, + "rewards/margins": 51.63226318359375, + "rewards/rejected": -47.247215270996094, + "step": 702 + }, + { + "epoch": 0.77, + "learning_rate": 1.1017241379310345e-05, + "logits/chosen": -0.3395247459411621, + "logits/rejected": -1.3865044116973877, + "logps/chosen": -324.23883056640625, + "logps/rejected": -543.4671630859375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.8300685882568359, + "rewards/margins": 44.21353530883789, + "rewards/rejected": -43.38346481323242, + "step": 703 + }, + { + "epoch": 0.77, + "learning_rate": 1.0965517241379311e-05, + "logits/chosen": -0.11953383684158325, + "logits/rejected": -1.4352549314498901, + "logps/chosen": -384.3127746582031, + "logps/rejected": -516.1475830078125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.3615190982818604, + "rewards/margins": 44.486629486083984, + "rewards/rejected": -41.12510681152344, + "step": 704 + }, + { + "epoch": 0.77, + "learning_rate": 1.0913793103448277e-05, + "logits/chosen": -0.11765387654304504, + "logits/rejected": -1.4176204204559326, + "logps/chosen": -391.2843933105469, + "logps/rejected": -551.655029296875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.346769332885742, + "rewards/margins": 49.20520782470703, + "rewards/rejected": -46.858436584472656, + "step": 705 + }, + { + "epoch": 0.77, + "learning_rate": 1.0862068965517242e-05, + "logits/chosen": -0.33923447132110596, + "logits/rejected": -1.3285080194473267, + "logps/chosen": -349.19921875, + "logps/rejected": -541.0517578125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.4373605251312256, + "rewards/margins": 43.785858154296875, + "rewards/rejected": -45.22321701049805, + "step": 706 + }, + { + "epoch": 0.77, + "learning_rate": 1.0810344827586207e-05, + "logits/chosen": -0.136460080742836, + "logits/rejected": -1.3332957029342651, + "logps/chosen": -421.43414306640625, + "logps/rejected": -533.1184692382812, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.373668909072876, + "rewards/margins": 48.273704528808594, + "rewards/rejected": -45.90003204345703, + "step": 707 + }, + { + "epoch": 0.77, + "learning_rate": 1.0758620689655173e-05, + "logits/chosen": -0.08225316554307938, + "logits/rejected": -1.5345730781555176, + "logps/chosen": -504.91265869140625, + "logps/rejected": -490.80535888671875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.046558380126953, + "rewards/margins": 47.151123046875, + "rewards/rejected": -43.10456466674805, + "step": 708 + }, + { + "epoch": 0.77, + "learning_rate": 1.070689655172414e-05, + "logits/chosen": -0.4365447461605072, + "logits/rejected": -1.3855762481689453, + "logps/chosen": -281.58013916015625, + "logps/rejected": -509.04608154296875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.8242043256759644, + "rewards/margins": 42.542762756347656, + "rewards/rejected": -43.366966247558594, + "step": 709 + }, + { + "epoch": 0.77, + "learning_rate": 1.0655172413793104e-05, + "logits/chosen": -0.2629587948322296, + "logits/rejected": -1.122794508934021, + "logps/chosen": -292.0431213378906, + "logps/rejected": -518.714111328125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.2017064094543457, + "rewards/margins": 40.38639450073242, + "rewards/rejected": -42.588096618652344, + "step": 710 + }, + { + "epoch": 0.78, + "learning_rate": 1.0603448275862069e-05, + "logits/chosen": -0.2471962720155716, + "logits/rejected": -1.4472237825393677, + "logps/chosen": -426.6012878417969, + "logps/rejected": -552.3489990234375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.5974886417388916, + "rewards/margins": 49.03648376464844, + "rewards/rejected": -46.438995361328125, + "step": 711 + }, + { + "epoch": 0.78, + "learning_rate": 1.0551724137931035e-05, + "logits/chosen": -0.09044022858142853, + "logits/rejected": -1.4647562503814697, + "logps/chosen": -424.2209777832031, + "logps/rejected": -516.6889038085938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.952216148376465, + "rewards/margins": 46.98312759399414, + "rewards/rejected": -43.03091049194336, + "step": 712 + }, + { + "epoch": 0.78, + "learning_rate": 1.0500000000000001e-05, + "logits/chosen": -0.2054508775472641, + "logits/rejected": -1.343733787536621, + "logps/chosen": -429.5415954589844, + "logps/rejected": -509.0820617675781, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.4331845045089722, + "rewards/margins": 44.145729064941406, + "rewards/rejected": -42.712547302246094, + "step": 713 + }, + { + "epoch": 0.78, + "learning_rate": 1.0448275862068967e-05, + "logits/chosen": -0.32011741399765015, + "logits/rejected": -1.4985260963439941, + "logps/chosen": -307.115966796875, + "logps/rejected": -482.08673095703125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.514482855796814, + "rewards/margins": 43.817840576171875, + "rewards/rejected": -42.30335998535156, + "step": 714 + }, + { + "epoch": 0.78, + "learning_rate": 1.0396551724137932e-05, + "logits/chosen": -0.253872275352478, + "logits/rejected": -1.4514174461364746, + "logps/chosen": -384.38958740234375, + "logps/rejected": -548.2704467773438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.331862688064575, + "rewards/margins": 48.48160934448242, + "rewards/rejected": -46.149749755859375, + "step": 715 + }, + { + "epoch": 0.78, + "learning_rate": 1.0344827586206897e-05, + "logits/chosen": -0.34388551115989685, + "logits/rejected": -0.9692472219467163, + "logps/chosen": -298.96734619140625, + "logps/rejected": -617.0269165039062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.927310943603516, + "rewards/margins": 44.458168029785156, + "rewards/rejected": -50.385475158691406, + "step": 716 + }, + { + "epoch": 0.78, + "learning_rate": 1.0293103448275863e-05, + "logits/chosen": -0.15216024219989777, + "logits/rejected": -1.239759922027588, + "logps/chosen": -349.53863525390625, + "logps/rejected": -593.9808959960938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.8680083751678467, + "rewards/margins": 48.948795318603516, + "rewards/rejected": -50.816802978515625, + "step": 717 + }, + { + "epoch": 0.78, + "learning_rate": 1.024137931034483e-05, + "logits/chosen": 0.0016151629388332367, + "logits/rejected": -1.4735568761825562, + "logps/chosen": -395.8976135253906, + "logps/rejected": -540.4542236328125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.16508412361145, + "rewards/margins": 47.68174362182617, + "rewards/rejected": -44.51665496826172, + "step": 718 + }, + { + "epoch": 0.78, + "learning_rate": 1.0189655172413794e-05, + "logits/chosen": -0.1629282683134079, + "logits/rejected": -1.3431519269943237, + "logps/chosen": -414.5015869140625, + "logps/rejected": -511.0880126953125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.656442642211914, + "rewards/margins": 45.010833740234375, + "rewards/rejected": -42.354393005371094, + "step": 719 + }, + { + "epoch": 0.79, + "learning_rate": 1.0137931034482758e-05, + "logits/chosen": -0.165948286652565, + "logits/rejected": -1.3963861465454102, + "logps/chosen": -370.53857421875, + "logps/rejected": -517.9864501953125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6973997354507446, + "rewards/margins": 45.15171813964844, + "rewards/rejected": -44.454322814941406, + "step": 720 + }, + { + "epoch": 0.79, + "learning_rate": 1.0086206896551725e-05, + "logits/chosen": -0.1827414333820343, + "logits/rejected": -1.461106300354004, + "logps/chosen": -382.1580505371094, + "logps/rejected": -511.6290588378906, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.0063939094543457, + "rewards/margins": 43.85320281982422, + "rewards/rejected": -42.84680938720703, + "step": 721 + }, + { + "epoch": 0.79, + "learning_rate": 1.0034482758620691e-05, + "logits/chosen": -0.197463721036911, + "logits/rejected": -1.405523657798767, + "logps/chosen": -356.38909912109375, + "logps/rejected": -516.2345581054688, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.6934216618537903, + "rewards/margins": 42.74033737182617, + "rewards/rejected": -43.433753967285156, + "step": 722 + }, + { + "epoch": 0.79, + "learning_rate": 9.982758620689656e-06, + "logits/chosen": -0.05039804056286812, + "logits/rejected": -1.37940514087677, + "logps/chosen": -410.9005126953125, + "logps/rejected": -526.2455444335938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.09383654594421387, + "rewards/margins": 44.6170768737793, + "rewards/rejected": -44.71091079711914, + "step": 723 + }, + { + "epoch": 0.79, + "learning_rate": 9.931034482758622e-06, + "logits/chosen": -0.07969538867473602, + "logits/rejected": -1.5281819105148315, + "logps/chosen": -509.1402893066406, + "logps/rejected": -570.6793212890625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 6.728723526000977, + "rewards/margins": 51.72552490234375, + "rewards/rejected": -44.996803283691406, + "step": 724 + }, + { + "epoch": 0.79, + "learning_rate": 9.879310344827587e-06, + "logits/chosen": -0.07072431594133377, + "logits/rejected": -1.546677827835083, + "logps/chosen": -464.2735595703125, + "logps/rejected": -509.17120361328125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.091634511947632, + "rewards/margins": 46.73294448852539, + "rewards/rejected": -43.64130783081055, + "step": 725 + }, + { + "epoch": 0.79, + "learning_rate": 9.827586206896553e-06, + "logits/chosen": -0.26471734046936035, + "logits/rejected": -1.5124938488006592, + "logps/chosen": -357.88262939453125, + "logps/rejected": -475.23046875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.5712640285491943, + "rewards/margins": 40.140602111816406, + "rewards/rejected": -40.71186828613281, + "step": 726 + }, + { + "epoch": 0.79, + "learning_rate": 9.775862068965518e-06, + "logits/chosen": -0.276339054107666, + "logits/rejected": -1.3366577625274658, + "logps/chosen": -386.89825439453125, + "logps/rejected": -560.8017578125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.2428956925868988, + "rewards/margins": 44.60403823852539, + "rewards/rejected": -44.84693908691406, + "step": 727 + }, + { + "epoch": 0.79, + "learning_rate": 9.724137931034484e-06, + "logits/chosen": -0.11000851541757584, + "logits/rejected": -1.6319886445999146, + "logps/chosen": -530.5902099609375, + "logps/rejected": -539.3771362304688, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.3030576705932617, + "rewards/margins": 46.28190612792969, + "rewards/rejected": -44.97885513305664, + "step": 728 + }, + { + "epoch": 0.8, + "learning_rate": 9.672413793103448e-06, + "logits/chosen": -0.2481529414653778, + "logits/rejected": -1.5020639896392822, + "logps/chosen": -371.010986328125, + "logps/rejected": -558.0028686523438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.233778476715088, + "rewards/margins": 49.5811653137207, + "rewards/rejected": -47.34738540649414, + "step": 729 + }, + { + "epoch": 0.8, + "learning_rate": 9.620689655172415e-06, + "logits/chosen": -0.3894887864589691, + "logits/rejected": -1.3705419301986694, + "logps/chosen": -364.73126220703125, + "logps/rejected": -544.0112915039062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.4287183284759521, + "rewards/margins": 46.11931610107422, + "rewards/rejected": -44.69059371948242, + "step": 730 + }, + { + "epoch": 0.8, + "learning_rate": 9.56896551724138e-06, + "logits/chosen": -0.03169269114732742, + "logits/rejected": -1.3399221897125244, + "logps/chosen": -487.8983154296875, + "logps/rejected": -521.6971435546875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.0384957790374756, + "rewards/margins": 47.08793640136719, + "rewards/rejected": -44.0494384765625, + "step": 731 + }, + { + "epoch": 0.8, + "learning_rate": 9.517241379310346e-06, + "logits/chosen": -0.17971675097942352, + "logits/rejected": -1.4211113452911377, + "logps/chosen": -436.1258544921875, + "logps/rejected": -565.4899291992188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.9572806358337402, + "rewards/margins": 48.67027282714844, + "rewards/rejected": -45.712989807128906, + "step": 732 + }, + { + "epoch": 0.8, + "learning_rate": 9.465517241379312e-06, + "logits/chosen": -0.08533386886119843, + "logits/rejected": -1.1883230209350586, + "logps/chosen": -339.646484375, + "logps/rejected": -544.335205078125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.090062141418457, + "rewards/margins": 43.51349639892578, + "rewards/rejected": -45.603553771972656, + "step": 733 + }, + { + "epoch": 0.8, + "learning_rate": 9.413793103448277e-06, + "logits/chosen": -0.16655822098255157, + "logits/rejected": -1.5006437301635742, + "logps/chosen": -513.9613037109375, + "logps/rejected": -541.0925903320312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.505112886428833, + "rewards/margins": 46.561466217041016, + "rewards/rejected": -43.056358337402344, + "step": 734 + }, + { + "epoch": 0.8, + "learning_rate": 9.362068965517241e-06, + "logits/chosen": -0.12692072987556458, + "logits/rejected": -1.4615023136138916, + "logps/chosen": -428.9164733886719, + "logps/rejected": -531.3856201171875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6445188522338867, + "rewards/margins": 41.926231384277344, + "rewards/rejected": -41.281715393066406, + "step": 735 + }, + { + "epoch": 0.8, + "learning_rate": 9.310344827586207e-06, + "logits/chosen": -0.30228492617607117, + "logits/rejected": -1.37641441822052, + "logps/chosen": -385.970947265625, + "logps/rejected": -553.395263671875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.8824379444122314, + "rewards/margins": 46.69935607910156, + "rewards/rejected": -45.816917419433594, + "step": 736 + }, + { + "epoch": 0.8, + "learning_rate": 9.258620689655174e-06, + "logits/chosen": -0.22073112428188324, + "logits/rejected": -1.2506929636001587, + "logps/chosen": -323.58209228515625, + "logps/rejected": -537.7647094726562, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.170397400856018, + "rewards/margins": 46.1264762878418, + "rewards/rejected": -44.95608139038086, + "step": 737 + }, + { + "epoch": 0.81, + "learning_rate": 9.206896551724138e-06, + "logits/chosen": -0.20875300467014313, + "logits/rejected": -1.174045205116272, + "logps/chosen": -364.25286865234375, + "logps/rejected": -519.7232666015625, + "loss": 0.0001, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.43882614374160767, + "rewards/margins": 42.18488311767578, + "rewards/rejected": -41.746063232421875, + "step": 738 + }, + { + "epoch": 0.81, + "learning_rate": 9.155172413793103e-06, + "logits/chosen": -0.13731160759925842, + "logits/rejected": -1.4550132751464844, + "logps/chosen": -461.5903625488281, + "logps/rejected": -532.6383666992188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.08963948488235474, + "rewards/margins": 42.90585708618164, + "rewards/rejected": -42.81622314453125, + "step": 739 + }, + { + "epoch": 0.81, + "learning_rate": 9.10344827586207e-06, + "logits/chosen": -0.1737978756427765, + "logits/rejected": -1.4691574573516846, + "logps/chosen": -377.02239990234375, + "logps/rejected": -519.2705078125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07581466436386108, + "rewards/margins": 43.39927673339844, + "rewards/rejected": -43.47509765625, + "step": 740 + }, + { + "epoch": 0.81, + "learning_rate": 9.051724137931036e-06, + "logits/chosen": -0.14953354001045227, + "logits/rejected": -1.5017184019088745, + "logps/chosen": -520.56201171875, + "logps/rejected": -525.4119262695312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.2810912132263184, + "rewards/margins": 47.63104248046875, + "rewards/rejected": -44.34995651245117, + "step": 741 + }, + { + "epoch": 0.81, + "learning_rate": 9e-06, + "logits/chosen": -0.10064885765314102, + "logits/rejected": -1.3716775178909302, + "logps/chosen": -370.49932861328125, + "logps/rejected": -557.8595581054688, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.030269622802734, + "rewards/margins": 50.669315338134766, + "rewards/rejected": -46.63904571533203, + "step": 742 + }, + { + "epoch": 0.81, + "learning_rate": 8.948275862068965e-06, + "logits/chosen": -0.10139290988445282, + "logits/rejected": -1.4602479934692383, + "logps/chosen": -392.13653564453125, + "logps/rejected": -521.1805419921875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 5.079219341278076, + "rewards/margins": 47.59113693237305, + "rewards/rejected": -42.51191711425781, + "step": 743 + }, + { + "epoch": 0.81, + "learning_rate": 8.896551724137931e-06, + "logits/chosen": -0.005460496060550213, + "logits/rejected": -1.4407155513763428, + "logps/chosen": -461.0459899902344, + "logps/rejected": -523.099853515625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.6384750604629517, + "rewards/margins": 42.25281524658203, + "rewards/rejected": -43.89128112792969, + "step": 744 + }, + { + "epoch": 0.81, + "learning_rate": 8.844827586206897e-06, + "logits/chosen": -0.24220603704452515, + "logits/rejected": -1.52848219871521, + "logps/chosen": -482.8728332519531, + "logps/rejected": -539.7685546875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.07667076587677002, + "rewards/margins": 43.8439826965332, + "rewards/rejected": -43.920654296875, + "step": 745 + }, + { + "epoch": 0.81, + "learning_rate": 8.793103448275864e-06, + "logits/chosen": -0.10701542347669601, + "logits/rejected": -1.2211958169937134, + "logps/chosen": -431.6956787109375, + "logps/rejected": -511.1591796875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.027044653892517, + "rewards/margins": 43.53398895263672, + "rewards/rejected": -42.50694274902344, + "step": 746 + }, + { + "epoch": 0.81, + "learning_rate": 8.741379310344828e-06, + "logits/chosen": -0.30449381470680237, + "logits/rejected": -1.4786876440048218, + "logps/chosen": -360.4140930175781, + "logps/rejected": -529.6007080078125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.6337039470672607, + "rewards/margins": 44.28141403198242, + "rewards/rejected": -42.647705078125, + "step": 747 + }, + { + "epoch": 0.82, + "learning_rate": 8.689655172413793e-06, + "logits/chosen": -0.1430250108242035, + "logits/rejected": -1.4525260925292969, + "logps/chosen": -398.2258605957031, + "logps/rejected": -542.1051025390625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.06671726703643799, + "rewards/margins": 46.824588775634766, + "rewards/rejected": -46.89130401611328, + "step": 748 + }, + { + "epoch": 0.82, + "learning_rate": 8.63793103448276e-06, + "logits/chosen": -0.1619974821805954, + "logits/rejected": -1.3705554008483887, + "logps/chosen": -302.81805419921875, + "logps/rejected": -550.9197998046875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.016142070293426514, + "rewards/margins": 47.28443908691406, + "rewards/rejected": -47.2682991027832, + "step": 749 + }, + { + "epoch": 0.82, + "learning_rate": 8.586206896551726e-06, + "logits/chosen": -0.19095589220523834, + "logits/rejected": -1.448203682899475, + "logps/chosen": -444.8843078613281, + "logps/rejected": -546.9640502929688, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4624607563018799, + "rewards/margins": 47.96385955810547, + "rewards/rejected": -47.50139617919922, + "step": 750 + }, + { + "epoch": 0.82, + "learning_rate": 8.53448275862069e-06, + "logits/chosen": -0.16919027268886566, + "logits/rejected": -1.374964714050293, + "logps/chosen": -416.65167236328125, + "logps/rejected": -532.4627685546875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6660584807395935, + "rewards/margins": 46.673011779785156, + "rewards/rejected": -46.00695037841797, + "step": 751 + }, + { + "epoch": 0.82, + "learning_rate": 8.482758620689655e-06, + "logits/chosen": -0.07386205345392227, + "logits/rejected": -1.4595537185668945, + "logps/chosen": -420.66900634765625, + "logps/rejected": -553.5128173828125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 5.008194923400879, + "rewards/margins": 52.118568420410156, + "rewards/rejected": -47.11037063598633, + "step": 752 + }, + { + "epoch": 0.82, + "learning_rate": 8.431034482758621e-06, + "logits/chosen": -0.20895949006080627, + "logits/rejected": -1.295670747756958, + "logps/chosen": -362.1947021484375, + "logps/rejected": -552.0540771484375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.658504843711853, + "rewards/margins": 46.098846435546875, + "rewards/rejected": -45.440338134765625, + "step": 753 + }, + { + "epoch": 0.82, + "learning_rate": 8.379310344827587e-06, + "logits/chosen": 0.02430650219321251, + "logits/rejected": -1.4669663906097412, + "logps/chosen": -471.47601318359375, + "logps/rejected": -534.8856201171875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6093757152557373, + "rewards/margins": 44.60679626464844, + "rewards/rejected": -43.99742126464844, + "step": 754 + }, + { + "epoch": 0.82, + "learning_rate": 8.327586206896552e-06, + "logits/chosen": -0.07655708491802216, + "logits/rejected": -1.1800837516784668, + "logps/chosen": -371.775146484375, + "logps/rejected": -575.125244140625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 6.643337726593018, + "rewards/margins": 54.822017669677734, + "rewards/rejected": -48.178680419921875, + "step": 755 + }, + { + "epoch": 0.82, + "learning_rate": 8.275862068965518e-06, + "logits/chosen": -0.16958683729171753, + "logits/rejected": -1.5856120586395264, + "logps/chosen": -450.3104248046875, + "logps/rejected": -530.606201171875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.2721757888793945, + "rewards/margins": 45.90708541870117, + "rewards/rejected": -43.634910583496094, + "step": 756 + }, + { + "epoch": 0.83, + "learning_rate": 8.224137931034483e-06, + "logits/chosen": -0.25436925888061523, + "logits/rejected": -1.495656967163086, + "logps/chosen": -453.14666748046875, + "logps/rejected": -509.3147888183594, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.9480234384536743, + "rewards/margins": 44.93329620361328, + "rewards/rejected": -43.98527145385742, + "step": 757 + }, + { + "epoch": 0.83, + "learning_rate": 8.17241379310345e-06, + "logits/chosen": -0.3505147099494934, + "logits/rejected": -1.2925671339035034, + "logps/chosen": -321.7222900390625, + "logps/rejected": -540.38720703125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.6912283897399902, + "rewards/margins": 46.32369613647461, + "rewards/rejected": -44.632469177246094, + "step": 758 + }, + { + "epoch": 0.83, + "learning_rate": 8.120689655172414e-06, + "logits/chosen": -0.03808639571070671, + "logits/rejected": -1.6043531894683838, + "logps/chosen": -429.830078125, + "logps/rejected": -489.75115966796875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 5.646089553833008, + "rewards/margins": 47.71070861816406, + "rewards/rejected": -42.06461715698242, + "step": 759 + }, + { + "epoch": 0.83, + "learning_rate": 8.06896551724138e-06, + "logits/chosen": -0.07932931929826736, + "logits/rejected": -1.5729631185531616, + "logps/chosen": -384.529541015625, + "logps/rejected": -537.7402954101562, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.7734086513519287, + "rewards/margins": 49.798828125, + "rewards/rejected": -47.02540969848633, + "step": 760 + }, + { + "epoch": 0.83, + "learning_rate": 8.017241379310345e-06, + "logits/chosen": -0.0787133127450943, + "logits/rejected": -1.3799351453781128, + "logps/chosen": -391.05712890625, + "logps/rejected": -552.1306762695312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.06786489486694336, + "rewards/margins": 43.98936462402344, + "rewards/rejected": -44.057228088378906, + "step": 761 + }, + { + "epoch": 0.83, + "learning_rate": 7.965517241379311e-06, + "logits/chosen": 0.007942818105220795, + "logits/rejected": -1.316530704498291, + "logps/chosen": -461.40142822265625, + "logps/rejected": -554.76318359375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.7176241874694824, + "rewards/margins": 50.235965728759766, + "rewards/rejected": -48.518341064453125, + "step": 762 + }, + { + "epoch": 0.83, + "learning_rate": 7.913793103448276e-06, + "logits/chosen": -0.14800256490707397, + "logits/rejected": -1.5657436847686768, + "logps/chosen": -393.4061584472656, + "logps/rejected": -546.5244750976562, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.175642490386963, + "rewards/margins": 50.06828689575195, + "rewards/rejected": -46.89263916015625, + "step": 763 + }, + { + "epoch": 0.83, + "learning_rate": 7.862068965517242e-06, + "logits/chosen": -0.13226521015167236, + "logits/rejected": -1.4478318691253662, + "logps/chosen": -422.7304382324219, + "logps/rejected": -542.8041381835938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.1135668754577637, + "rewards/margins": 49.9066276550293, + "rewards/rejected": -47.793052673339844, + "step": 764 + }, + { + "epoch": 0.83, + "learning_rate": 7.810344827586208e-06, + "logits/chosen": -0.003958722576498985, + "logits/rejected": -1.609676718711853, + "logps/chosen": -435.3834228515625, + "logps/rejected": -524.7167358398438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.061379045248031616, + "rewards/margins": 47.20934295654297, + "rewards/rejected": -47.14796447753906, + "step": 765 + }, + { + "epoch": 0.84, + "learning_rate": 7.758620689655173e-06, + "logits/chosen": -0.18092390894889832, + "logits/rejected": -1.2236413955688477, + "logps/chosen": -320.91802978515625, + "logps/rejected": -506.1944274902344, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6078252196311951, + "rewards/margins": 43.06869888305664, + "rewards/rejected": -42.460880279541016, + "step": 766 + }, + { + "epoch": 0.84, + "learning_rate": 7.706896551724138e-06, + "logits/chosen": -0.20149590075016022, + "logits/rejected": -1.1600701808929443, + "logps/chosen": -354.605712890625, + "logps/rejected": -552.8082275390625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.32997846603393555, + "rewards/margins": 43.7484016418457, + "rewards/rejected": -43.41842269897461, + "step": 767 + }, + { + "epoch": 0.84, + "learning_rate": 7.655172413793104e-06, + "logits/chosen": -0.13990965485572815, + "logits/rejected": -1.271227240562439, + "logps/chosen": -314.2335205078125, + "logps/rejected": -546.614013671875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.4497412443161011, + "rewards/margins": 47.33036804199219, + "rewards/rejected": -47.78010559082031, + "step": 768 + }, + { + "epoch": 0.84, + "learning_rate": 7.603448275862069e-06, + "logits/chosen": -0.42814818024635315, + "logits/rejected": -1.4132099151611328, + "logps/chosen": -341.91461181640625, + "logps/rejected": -564.9898071289062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.5869011878967285, + "rewards/margins": 50.5153923034668, + "rewards/rejected": -47.92848587036133, + "step": 769 + }, + { + "epoch": 0.84, + "learning_rate": 7.551724137931036e-06, + "logits/chosen": -0.1636469066143036, + "logits/rejected": -1.2765783071517944, + "logps/chosen": -387.31036376953125, + "logps/rejected": -545.6071166992188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.6056363582611084, + "rewards/margins": 44.79917907714844, + "rewards/rejected": -48.404815673828125, + "step": 770 + }, + { + "epoch": 0.84, + "learning_rate": 7.5e-06, + "logits/chosen": -0.2579754590988159, + "logits/rejected": -1.3234689235687256, + "logps/chosen": -305.4987487792969, + "logps/rejected": -521.1893310546875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.6613775491714478, + "rewards/margins": 42.09248352050781, + "rewards/rejected": -43.75386047363281, + "step": 771 + }, + { + "epoch": 0.84, + "learning_rate": 7.448275862068966e-06, + "logits/chosen": -0.11062826961278915, + "logits/rejected": -1.2727200984954834, + "logps/chosen": -425.83306884765625, + "logps/rejected": -566.6824340820312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.7593159079551697, + "rewards/margins": 45.5145263671875, + "rewards/rejected": -46.273841857910156, + "step": 772 + }, + { + "epoch": 0.84, + "learning_rate": 7.396551724137932e-06, + "logits/chosen": -0.2024325430393219, + "logits/rejected": -1.4240882396697998, + "logps/chosen": -498.54718017578125, + "logps/rejected": -526.6759033203125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.25721901655197144, + "rewards/margins": 42.7508544921875, + "rewards/rejected": -42.49363327026367, + "step": 773 + }, + { + "epoch": 0.84, + "learning_rate": 7.344827586206897e-06, + "logits/chosen": -0.06756836175918579, + "logits/rejected": -1.318863034248352, + "logps/chosen": -439.17828369140625, + "logps/rejected": -524.9141845703125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.8192152976989746, + "rewards/margins": 43.100868225097656, + "rewards/rejected": -44.920082092285156, + "step": 774 + }, + { + "epoch": 0.85, + "learning_rate": 7.293103448275862e-06, + "logits/chosen": -0.13676553964614868, + "logits/rejected": -1.509717345237732, + "logps/chosen": -467.790771484375, + "logps/rejected": -556.1195678710938, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4794842600822449, + "rewards/margins": 46.7752571105957, + "rewards/rejected": -46.2957763671875, + "step": 775 + }, + { + "epoch": 0.85, + "learning_rate": 7.241379310344828e-06, + "logits/chosen": -0.3739855885505676, + "logits/rejected": -1.4018502235412598, + "logps/chosen": -378.53436279296875, + "logps/rejected": -543.9190063476562, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.0305161476135254, + "rewards/margins": 46.968936920166016, + "rewards/rejected": -43.938419342041016, + "step": 776 + }, + { + "epoch": 0.85, + "learning_rate": 7.189655172413794e-06, + "logits/chosen": -0.032344087958335876, + "logits/rejected": -1.596992015838623, + "logps/chosen": -406.6452331542969, + "logps/rejected": -545.2965087890625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.9050942659378052, + "rewards/margins": 48.07621383666992, + "rewards/rejected": -47.17111587524414, + "step": 777 + }, + { + "epoch": 0.85, + "learning_rate": 7.137931034482759e-06, + "logits/chosen": -0.05962662398815155, + "logits/rejected": -1.5691879987716675, + "logps/chosen": -517.4493408203125, + "logps/rejected": -572.8405151367188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.989516496658325, + "rewards/margins": 51.501243591308594, + "rewards/rejected": -48.5117301940918, + "step": 778 + }, + { + "epoch": 0.85, + "learning_rate": 7.086206896551724e-06, + "logits/chosen": -0.19138425588607788, + "logits/rejected": -1.2675436735153198, + "logps/chosen": -372.99652099609375, + "logps/rejected": -520.8903198242188, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.659572124481201, + "rewards/margins": 47.659244537353516, + "rewards/rejected": -44.999671936035156, + "step": 779 + }, + { + "epoch": 0.85, + "learning_rate": 7.03448275862069e-06, + "logits/chosen": -0.3746905028820038, + "logits/rejected": -1.394024133682251, + "logps/chosen": -287.9949035644531, + "logps/rejected": -537.7847900390625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.1210591793060303, + "rewards/margins": 44.04398727416992, + "rewards/rejected": -46.1650505065918, + "step": 780 + }, + { + "epoch": 0.85, + "learning_rate": 6.982758620689656e-06, + "logits/chosen": -0.24698945879936218, + "logits/rejected": -1.3805230855941772, + "logps/chosen": -294.69512939453125, + "logps/rejected": -485.89373779296875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.2966182231903076, + "rewards/margins": 39.19174575805664, + "rewards/rejected": -40.488365173339844, + "step": 781 + }, + { + "epoch": 0.85, + "learning_rate": 6.931034482758622e-06, + "logits/chosen": -0.17452430725097656, + "logits/rejected": -1.5681843757629395, + "logps/chosen": -525.44287109375, + "logps/rejected": -519.7317504882812, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.254897117614746, + "rewards/margins": 48.073875427246094, + "rewards/rejected": -43.818973541259766, + "step": 782 + }, + { + "epoch": 0.85, + "learning_rate": 6.8793103448275865e-06, + "logits/chosen": -0.048231612890958786, + "logits/rejected": -1.5449762344360352, + "logps/chosen": -425.188720703125, + "logps/rejected": -548.519287109375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.5561108589172363, + "rewards/margins": 48.53321838378906, + "rewards/rejected": -45.97710418701172, + "step": 783 + }, + { + "epoch": 0.86, + "learning_rate": 6.827586206896552e-06, + "logits/chosen": -0.06258489936590195, + "logits/rejected": -1.4091614484786987, + "logps/chosen": -515.0226440429688, + "logps/rejected": -507.0038146972656, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.27601510286331177, + "rewards/margins": 41.263938903808594, + "rewards/rejected": -41.53995132446289, + "step": 784 + }, + { + "epoch": 0.86, + "learning_rate": 6.775862068965518e-06, + "logits/chosen": 0.01976817660033703, + "logits/rejected": -1.3316725492477417, + "logps/chosen": -394.07293701171875, + "logps/rejected": -545.0585327148438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.06031680107116699, + "rewards/margins": 47.525875091552734, + "rewards/rejected": -47.46555709838867, + "step": 785 + }, + { + "epoch": 0.86, + "learning_rate": 6.724137931034484e-06, + "logits/chosen": -0.27925077080726624, + "logits/rejected": -1.3859542608261108, + "logps/chosen": -384.59515380859375, + "logps/rejected": -555.54150390625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.498483419418335, + "rewards/margins": 47.778785705566406, + "rewards/rejected": -45.280303955078125, + "step": 786 + }, + { + "epoch": 0.86, + "learning_rate": 6.672413793103448e-06, + "logits/chosen": 0.09738397598266602, + "logits/rejected": -1.63230299949646, + "logps/chosen": -476.4755859375, + "logps/rejected": -541.860595703125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 6.147635459899902, + "rewards/margins": 49.31689453125, + "rewards/rejected": -43.16926574707031, + "step": 787 + }, + { + "epoch": 0.86, + "learning_rate": 6.620689655172414e-06, + "logits/chosen": -0.18144603073596954, + "logits/rejected": -1.3245290517807007, + "logps/chosen": -366.3008117675781, + "logps/rejected": -570.7501831054688, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.618462562561035, + "rewards/margins": 48.92848205566406, + "rewards/rejected": -46.310020446777344, + "step": 788 + }, + { + "epoch": 0.86, + "learning_rate": 6.56896551724138e-06, + "logits/chosen": -0.27109184861183167, + "logits/rejected": -1.202581524848938, + "logps/chosen": -357.2593078613281, + "logps/rejected": -600.5250244140625, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.0307457447052002, + "rewards/margins": 43.368961334228516, + "rewards/rejected": -42.338218688964844, + "step": 789 + }, + { + "epoch": 0.86, + "learning_rate": 6.517241379310346e-06, + "logits/chosen": -0.0993267223238945, + "logits/rejected": -1.4569131135940552, + "logps/chosen": -459.5248107910156, + "logps/rejected": -583.1766967773438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 4.424875736236572, + "rewards/margins": 51.83470153808594, + "rewards/rejected": -47.409820556640625, + "step": 790 + }, + { + "epoch": 0.86, + "learning_rate": 6.46551724137931e-06, + "logits/chosen": -0.02265375852584839, + "logits/rejected": -1.5002658367156982, + "logps/chosen": -480.3158264160156, + "logps/rejected": -527.55224609375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.9364004135131836, + "rewards/margins": 41.2583122253418, + "rewards/rejected": -44.19470977783203, + "step": 791 + }, + { + "epoch": 0.86, + "learning_rate": 6.4137931034482765e-06, + "logits/chosen": -0.1097775399684906, + "logits/rejected": -1.3407869338989258, + "logps/chosen": -472.06695556640625, + "logps/rejected": -560.4843139648438, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.07017156481742859, + "rewards/margins": 46.916927337646484, + "rewards/rejected": -46.84675598144531, + "step": 792 + }, + { + "epoch": 0.87, + "learning_rate": 6.362068965517242e-06, + "logits/chosen": -0.19321691989898682, + "logits/rejected": -1.552483081817627, + "logps/chosen": -423.1416015625, + "logps/rejected": -538.5962524414062, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5412538051605225, + "rewards/margins": 47.966957092285156, + "rewards/rejected": -47.42570495605469, + "step": 793 + }, + { + "epoch": 0.87, + "learning_rate": 6.310344827586208e-06, + "logits/chosen": -0.17814648151397705, + "logits/rejected": -1.6144907474517822, + "logps/chosen": -510.121826171875, + "logps/rejected": -568.7341918945312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.9851760864257812, + "rewards/margins": 48.46935272216797, + "rewards/rejected": -46.48417282104492, + "step": 794 + }, + { + "epoch": 0.87, + "learning_rate": 6.258620689655172e-06, + "logits/chosen": -0.1669325828552246, + "logits/rejected": -1.4748461246490479, + "logps/chosen": -524.6575317382812, + "logps/rejected": -524.4050903320312, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 7.138103485107422, + "rewards/margins": 49.07673645019531, + "rewards/rejected": -41.938636779785156, + "step": 795 + }, + { + "epoch": 0.87, + "learning_rate": 6.206896551724138e-06, + "logits/chosen": -0.31837576627731323, + "logits/rejected": -1.4339959621429443, + "logps/chosen": -382.4565734863281, + "logps/rejected": -521.4945068359375, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.1704509258270264, + "rewards/margins": 45.89916229248047, + "rewards/rejected": -43.72870635986328, + "step": 796 + }, + { + "epoch": 0.87, + "learning_rate": 6.155172413793104e-06, + "logits/chosen": -0.09191716462373734, + "logits/rejected": -1.3367663621902466, + "logps/chosen": -431.8583068847656, + "logps/rejected": -521.5662841796875, + "loss": 0.0009, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.916877031326294, + "rewards/margins": 43.596458435058594, + "rewards/rejected": -42.67958068847656, + "step": 797 + }, + { + "epoch": 0.87, + "learning_rate": 6.10344827586207e-06, + "logits/chosen": -0.3136548101902008, + "logits/rejected": -1.2592624425888062, + "logps/chosen": -357.0682067871094, + "logps/rejected": -587.6653442382812, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.1548173427581787, + "rewards/margins": 47.3561897277832, + "rewards/rejected": -48.511009216308594, + "step": 798 + }, + { + "epoch": 0.87, + "learning_rate": 6.051724137931035e-06, + "logits/chosen": -0.14841410517692566, + "logits/rejected": -1.623029351234436, + "logps/chosen": -554.7265625, + "logps/rejected": -536.6767578125, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.5121312141418457, + "rewards/margins": 46.72931671142578, + "rewards/rejected": -44.217185974121094, + "step": 799 + }, + { + "epoch": 0.87, + "learning_rate": 6e-06, + "logits/chosen": -0.3482857942581177, + "logits/rejected": -1.0176464319229126, + "logps/chosen": -282.9642028808594, + "logps/rejected": -562.8319091796875, + "loss": 0.0, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18101203441619873, + "rewards/margins": 46.256874084472656, + "rewards/rejected": -46.437889099121094, + "step": 800 + } + ], + "logging_steps": 1, + "max_steps": 916, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 100, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +}