{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9997172745264349,
  "eval_steps": 500,
  "global_step": 442,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0022618037885213456,
      "grad_norm": 73.88447935284657,
      "learning_rate": 1.7777777777777777e-08,
      "logits/chosen": -0.970178484916687,
      "logits/rejected": -0.9579144716262817,
      "logps/chosen": -1.656538963317871,
      "logps/rejected": -1.776021957397461,
      "loss": 5.7103,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -16.56538963317871,
      "rewards/margins": 1.1948291063308716,
      "rewards/rejected": -17.76021957397461,
      "step": 1
    },
    {
      "epoch": 0.004523607577042691,
      "grad_norm": 86.9125420208072,
      "learning_rate": 3.5555555555555554e-08,
      "logits/chosen": -1.0327715873718262,
      "logits/rejected": -1.023756980895996,
      "logps/chosen": -1.7529594898223877,
      "logps/rejected": -1.700197458267212,
      "loss": 6.1559,
      "rewards/accuracies": 0.5390625,
      "rewards/chosen": -17.52959442138672,
      "rewards/margins": -0.527620792388916,
      "rewards/rejected": -17.00197410583496,
      "step": 2
    },
    {
      "epoch": 0.006785411365564037,
      "grad_norm": 81.69530155846427,
      "learning_rate": 5.333333333333333e-08,
      "logits/chosen": -0.9820892810821533,
      "logits/rejected": -0.998577892780304,
      "logps/chosen": -1.7510223388671875,
      "logps/rejected": -1.9477208852767944,
      "loss": 6.4107,
      "rewards/accuracies": 0.5546875,
      "rewards/chosen": -17.510223388671875,
      "rewards/margins": 1.966986060142517,
      "rewards/rejected": -19.47720718383789,
      "step": 3
    },
    {
      "epoch": 0.009047215154085382,
      "grad_norm": 102.14679455248601,
      "learning_rate": 7.111111111111111e-08,
      "logits/chosen": -1.0096410512924194,
      "logits/rejected": -0.9931849241256714,
      "logps/chosen": -1.7210272550582886,
      "logps/rejected": -1.6721601486206055,
      "loss": 6.2871,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -17.21027183532715,
      "rewards/margins": -0.4886711835861206,
      "rewards/rejected": -16.721601486206055,
      "step": 4
    },
    {
      "epoch": 0.01130901894260673,
      "grad_norm": 80.06704613306383,
      "learning_rate": 8.888888888888888e-08,
      "logits/chosen": -1.0100579261779785,
      "logits/rejected": -0.998342752456665,
      "logps/chosen": -1.6671767234802246,
      "logps/rejected": -1.6990655660629272,
      "loss": 5.7732,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -16.671768188476562,
      "rewards/margins": 0.3188871145248413,
      "rewards/rejected": -16.99065399169922,
      "step": 5
    },
    {
      "epoch": 0.013570822731128074,
      "grad_norm": 74.95530274523003,
      "learning_rate": 1.0666666666666666e-07,
      "logits/chosen": -1.0750232934951782,
      "logits/rejected": -1.054511547088623,
      "logps/chosen": -1.7332895994186401,
      "logps/rejected": -1.6780965328216553,
      "loss": 6.2788,
      "rewards/accuracies": 0.5078125,
      "rewards/chosen": -17.332895278930664,
      "rewards/margins": -0.5519311428070068,
      "rewards/rejected": -16.780963897705078,
      "step": 6
    },
    {
      "epoch": 0.01583262651964942,
      "grad_norm": 39.99505319144089,
      "learning_rate": 1.2444444444444443e-07,
      "logits/chosen": -0.9987316131591797,
      "logits/rejected": -0.9925400614738464,
      "logps/chosen": -1.56162691116333,
      "logps/rejected": -1.926164150238037,
      "loss": 4.1206,
      "rewards/accuracies": 0.6328125,
      "rewards/chosen": -15.616266250610352,
      "rewards/margins": 3.6453747749328613,
      "rewards/rejected": -19.261642456054688,
      "step": 7
    },
    {
      "epoch": 0.018094430308170765,
      "grad_norm": 106.66642676745393,
      "learning_rate": 1.4222222222222222e-07,
      "logits/chosen": -0.9619507789611816,
      "logits/rejected": -0.9788931608200073,
      "logps/chosen": -1.6865566968917847,
      "logps/rejected": -1.7454805374145508,
      "loss": 5.9544,
      "rewards/accuracies": 0.5390625,
      "rewards/chosen": -16.86556625366211,
      "rewards/margins": 0.5892367959022522,
      "rewards/rejected": -17.454803466796875,
      "step": 8
    },
    {
      "epoch": 0.020356234096692113,
      "grad_norm": 57.905987384590865,
      "learning_rate": 1.6e-07,
      "logits/chosen": -1.0222256183624268,
      "logits/rejected": -1.0178791284561157,
      "logps/chosen": -1.5835094451904297,
      "logps/rejected": -1.796851634979248,
      "loss": 5.3604,
      "rewards/accuracies": 0.5546875,
      "rewards/chosen": -15.835092544555664,
      "rewards/margins": 2.1334214210510254,
      "rewards/rejected": -17.968515396118164,
      "step": 9
    },
    {
      "epoch": 0.02261803788521346,
      "grad_norm": 82.69531006046971,
      "learning_rate": 1.7777777777777776e-07,
      "logits/chosen": -1.0171338319778442,
      "logits/rejected": -1.0163377523422241,
      "logps/chosen": -1.7584372758865356,
      "logps/rejected": -1.6933101415634155,
      "loss": 6.0802,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -17.58437156677246,
      "rewards/margins": -0.6512712240219116,
      "rewards/rejected": -16.933101654052734,
      "step": 10
    },
    {
      "epoch": 0.024879841673734804,
      "grad_norm": 97.05523972814746,
      "learning_rate": 1.9555555555555555e-07,
      "logits/chosen": -1.0786405801773071,
      "logits/rejected": -1.0799434185028076,
      "logps/chosen": -1.732240915298462,
      "logps/rejected": -1.967947006225586,
      "loss": 5.3828,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -17.322410583496094,
      "rewards/margins": 2.357060432434082,
      "rewards/rejected": -19.679471969604492,
      "step": 11
    },
    {
      "epoch": 0.02714164546225615,
      "grad_norm": 94.06438233273298,
      "learning_rate": 2.133333333333333e-07,
      "logits/chosen": -0.9874241352081299,
      "logits/rejected": -0.969804584980011,
      "logps/chosen": -1.8421882390975952,
      "logps/rejected": -1.9192876815795898,
      "loss": 6.3133,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -18.42188262939453,
      "rewards/margins": 0.7709953188896179,
      "rewards/rejected": -19.1928768157959,
      "step": 12
    },
    {
      "epoch": 0.029403449250777494,
      "grad_norm": 94.45847571939335,
      "learning_rate": 2.3111111111111107e-07,
      "logits/chosen": -0.9826835989952087,
      "logits/rejected": -0.960513710975647,
      "logps/chosen": -1.771365761756897,
      "logps/rejected": -1.6903609037399292,
      "loss": 6.1689,
      "rewards/accuracies": 0.5546875,
      "rewards/chosen": -17.71365737915039,
      "rewards/margins": -0.8100481629371643,
      "rewards/rejected": -16.903610229492188,
      "step": 13
    },
    {
      "epoch": 0.03166525303929884,
      "grad_norm": 59.99069333682814,
      "learning_rate": 2.4888888888888886e-07,
      "logits/chosen": -0.9839141964912415,
      "logits/rejected": -0.9832520484924316,
      "logps/chosen": -1.4408010244369507,
      "logps/rejected": -1.6560081243515015,
      "loss": 4.5476,
      "rewards/accuracies": 0.6171875,
      "rewards/chosen": -14.40800952911377,
      "rewards/margins": 2.152069568634033,
      "rewards/rejected": -16.56007957458496,
      "step": 14
    },
    {
      "epoch": 0.033927056827820185,
      "grad_norm": 96.41950248980098,
      "learning_rate": 2.666666666666666e-07,
      "logits/chosen": -0.9852787852287292,
      "logits/rejected": -0.9823875427246094,
      "logps/chosen": -1.5907388925552368,
      "logps/rejected": -1.6886473894119263,
      "loss": 5.1372,
      "rewards/accuracies": 0.6171875,
      "rewards/chosen": -15.907387733459473,
      "rewards/margins": 0.9790847301483154,
      "rewards/rejected": -16.886472702026367,
      "step": 15
    },
    {
      "epoch": 0.03618886061634153,
      "grad_norm": 76.96404809338092,
      "learning_rate": 2.8444444444444443e-07,
      "logits/chosen": -1.022695541381836,
      "logits/rejected": -1.0211068391799927,
      "logps/chosen": -1.741645097732544,
      "logps/rejected": -1.709848403930664,
      "loss": 6.4049,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -17.416452407836914,
      "rewards/margins": -0.3179686665534973,
      "rewards/rejected": -17.09848403930664,
      "step": 16
    },
    {
      "epoch": 0.038450664404862875,
      "grad_norm": 93.85331038926209,
      "learning_rate": 3.022222222222222e-07,
      "logits/chosen": -1.0325472354888916,
      "logits/rejected": -1.0200681686401367,
      "logps/chosen": -1.5360885858535767,
      "logps/rejected": -1.4957702159881592,
      "loss": 6.2812,
      "rewards/accuracies": 0.5390625,
      "rewards/chosen": -15.360885620117188,
      "rewards/margins": -0.4031827449798584,
      "rewards/rejected": -14.957704544067383,
      "step": 17
    },
    {
      "epoch": 0.04071246819338423,
      "grad_norm": 98.20855483649616,
      "learning_rate": 3.2e-07,
      "logits/chosen": -1.0454903841018677,
      "logits/rejected": -1.0343623161315918,
      "logps/chosen": -1.9974663257598877,
      "logps/rejected": -2.022808074951172,
      "loss": 6.0793,
      "rewards/accuracies": 0.5234375,
      "rewards/chosen": -19.97466278076172,
      "rewards/margins": 0.2534167766571045,
      "rewards/rejected": -20.22808265686035,
      "step": 18
    },
    {
      "epoch": 0.04297427198190557,
      "grad_norm": 86.85908299567286,
      "learning_rate": 3.3777777777777777e-07,
      "logits/chosen": -1.0504282712936401,
      "logits/rejected": -1.042330265045166,
      "logps/chosen": -1.7138850688934326,
      "logps/rejected": -1.7518320083618164,
      "loss": 5.2537,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -17.13884925842285,
      "rewards/margins": 0.37946897745132446,
      "rewards/rejected": -17.51831817626953,
      "step": 19
    },
    {
      "epoch": 0.04523607577042692,
      "grad_norm": 40.41103844657731,
      "learning_rate": 3.5555555555555553e-07,
      "logits/chosen": -1.0436055660247803,
      "logits/rejected": -1.0474931001663208,
      "logps/chosen": -1.5377863645553589,
      "logps/rejected": -1.766308069229126,
      "loss": 4.3313,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -15.377862930297852,
      "rewards/margins": 2.2852182388305664,
      "rewards/rejected": -17.6630802154541,
      "step": 20
    },
    {
      "epoch": 0.04749787955894826,
      "grad_norm": 65.41047373950924,
      "learning_rate": 3.7333333333333334e-07,
      "logits/chosen": -0.9501165151596069,
      "logits/rejected": -0.953487753868103,
      "logps/chosen": -1.5590920448303223,
      "logps/rejected": -1.5690488815307617,
      "loss": 5.4623,
      "rewards/accuracies": 0.6015625,
      "rewards/chosen": -15.590919494628906,
      "rewards/margins": 0.09956827759742737,
      "rewards/rejected": -15.690486907958984,
      "step": 21
    },
    {
      "epoch": 0.04975968334746961,
      "grad_norm": 52.44214623062925,
      "learning_rate": 3.911111111111111e-07,
      "logits/chosen": -0.9958354830741882,
      "logits/rejected": -0.9834215641021729,
      "logps/chosen": -1.6224457025527954,
      "logps/rejected": -1.6867178678512573,
      "loss": 4.7122,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -16.224456787109375,
      "rewards/margins": 0.6427204608917236,
      "rewards/rejected": -16.867177963256836,
      "step": 22
    },
    {
      "epoch": 0.05202148713599095,
      "grad_norm": 70.21591683383166,
      "learning_rate": 4.0888888888888886e-07,
      "logits/chosen": -1.0022647380828857,
      "logits/rejected": -0.972631573677063,
      "logps/chosen": -1.6058869361877441,
      "logps/rejected": -1.6072971820831299,
      "loss": 5.8867,
      "rewards/accuracies": 0.5390625,
      "rewards/chosen": -16.058868408203125,
      "rewards/margins": 0.014102667570114136,
      "rewards/rejected": -16.07297134399414,
      "step": 23
    },
    {
      "epoch": 0.0542832909245123,
      "grad_norm": 51.78707493520344,
      "learning_rate": 4.266666666666666e-07,
      "logits/chosen": -1.0004748106002808,
      "logits/rejected": -1.0073304176330566,
      "logps/chosen": -1.4363174438476562,
      "logps/rejected": -1.5035033226013184,
      "loss": 5.1217,
      "rewards/accuracies": 0.5859375,
      "rewards/chosen": -14.363174438476562,
      "rewards/margins": 0.671859622001648,
      "rewards/rejected": -15.035033226013184,
      "step": 24
    },
    {
      "epoch": 0.05654509471303364,
      "grad_norm": 83.93302314997005,
      "learning_rate": 4.4444444444444444e-07,
      "logits/chosen": -0.9994463920593262,
      "logits/rejected": -0.9950385093688965,
      "logps/chosen": -1.7453137636184692,
      "logps/rejected": -1.7574050426483154,
      "loss": 5.5417,
      "rewards/accuracies": 0.5703125,
      "rewards/chosen": -17.453136444091797,
      "rewards/margins": 0.12091405689716339,
      "rewards/rejected": -17.574050903320312,
      "step": 25
    },
    {
      "epoch": 0.05880689850155499,
      "grad_norm": 78.39435935083853,
      "learning_rate": 4.6222222222222214e-07,
      "logits/chosen": -1.0201776027679443,
      "logits/rejected": -1.0055274963378906,
      "logps/chosen": -1.7500752210617065,
      "logps/rejected": -1.752750039100647,
      "loss": 5.4918,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -17.500751495361328,
      "rewards/margins": 0.02674958109855652,
      "rewards/rejected": -17.52750015258789,
      "step": 26
    },
    {
      "epoch": 0.061068702290076333,
      "grad_norm": 68.24007748722875,
      "learning_rate": 4.8e-07,
      "logits/chosen": -1.0300168991088867,
      "logits/rejected": -1.032236933708191,
      "logps/chosen": -1.7269227504730225,
      "logps/rejected": -1.6468346118927002,
      "loss": 6.2062,
      "rewards/accuracies": 0.5390625,
      "rewards/chosen": -17.269227981567383,
      "rewards/margins": -0.800881028175354,
      "rewards/rejected": -16.468345642089844,
      "step": 27
    },
    {
      "epoch": 0.06333050607859768,
      "grad_norm": 51.325729522476735,
      "learning_rate": 4.977777777777777e-07,
      "logits/chosen": -1.020127534866333,
      "logits/rejected": -1.00740647315979,
      "logps/chosen": -1.6155153512954712,
      "logps/rejected": -1.7134020328521729,
      "loss": 5.6067,
      "rewards/accuracies": 0.5234375,
      "rewards/chosen": -16.1551513671875,
      "rewards/margins": 0.9788689613342285,
      "rewards/rejected": -17.134021759033203,
      "step": 28
    },
    {
      "epoch": 0.06559230986711903,
      "grad_norm": 67.70299840427867,
      "learning_rate": 5.155555555555556e-07,
      "logits/chosen": -1.0417428016662598,
      "logits/rejected": -1.0501348972320557,
      "logps/chosen": -1.625864028930664,
      "logps/rejected": -1.672263264656067,
      "loss": 5.307,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": -16.25864028930664,
      "rewards/margins": 0.4639921486377716,
      "rewards/rejected": -16.722633361816406,
      "step": 29
    },
    {
      "epoch": 0.06785411365564037,
      "grad_norm": 88.41842890120783,
      "learning_rate": 5.333333333333332e-07,
      "logits/chosen": -1.012568473815918,
      "logits/rejected": -0.9970849752426147,
      "logps/chosen": -1.673133134841919,
      "logps/rejected": -1.7118630409240723,
      "loss": 6.1408,
      "rewards/accuracies": 0.5078125,
      "rewards/chosen": -16.73133087158203,
      "rewards/margins": 0.38729995489120483,
      "rewards/rejected": -17.11863136291504,
      "step": 30
    },
    {
      "epoch": 0.07011591744416172,
      "grad_norm": 79.26925757564649,
      "learning_rate": 5.511111111111111e-07,
      "logits/chosen": -1.0454754829406738,
      "logits/rejected": -1.031201720237732,
      "logps/chosen": -1.654517412185669,
      "logps/rejected": -1.6231170892715454,
      "loss": 5.7537,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -16.54517364501953,
      "rewards/margins": -0.314003586769104,
      "rewards/rejected": -16.231170654296875,
      "step": 31
    },
    {
      "epoch": 0.07237772123268306,
      "grad_norm": 62.138847724263975,
      "learning_rate": 5.688888888888889e-07,
      "logits/chosen": -1.0572967529296875,
      "logits/rejected": -1.0592782497406006,
      "logps/chosen": -1.5859274864196777,
      "logps/rejected": -1.5584020614624023,
      "loss": 5.6448,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": -15.859275817871094,
      "rewards/margins": -0.275254487991333,
      "rewards/rejected": -15.584020614624023,
      "step": 32
    },
    {
      "epoch": 0.07463952502120441,
      "grad_norm": 51.730594123658335,
      "learning_rate": 5.866666666666666e-07,
      "logits/chosen": -1.001430869102478,
      "logits/rejected": -0.9772145748138428,
      "logps/chosen": -1.556628704071045,
      "logps/rejected": -1.740290880203247,
      "loss": 4.6798,
      "rewards/accuracies": 0.5859375,
      "rewards/chosen": -15.56628704071045,
      "rewards/margins": 1.836620569229126,
      "rewards/rejected": -17.402908325195312,
      "step": 33
    },
    {
      "epoch": 0.07690132880972575,
      "grad_norm": 52.933499868170415,
      "learning_rate": 6.044444444444444e-07,
      "logits/chosen": -1.0453863143920898,
      "logits/rejected": -0.9934295415878296,
      "logps/chosen": -1.4905204772949219,
      "logps/rejected": -1.6541056632995605,
      "loss": 4.4742,
      "rewards/accuracies": 0.5703125,
      "rewards/chosen": -14.905204772949219,
      "rewards/margins": 1.6358494758605957,
      "rewards/rejected": -16.541053771972656,
      "step": 34
    },
    {
      "epoch": 0.0791631325982471,
      "grad_norm": 74.39925640560193,
      "learning_rate": 6.222222222222223e-07,
      "logits/chosen": -1.0696301460266113,
      "logits/rejected": -1.028558611869812,
      "logps/chosen": -1.660436987876892,
      "logps/rejected": -1.8180859088897705,
      "loss": 5.0488,
      "rewards/accuracies": 0.5546875,
      "rewards/chosen": -16.6043701171875,
      "rewards/margins": 1.576488733291626,
      "rewards/rejected": -18.18086051940918,
      "step": 35
    },
    {
      "epoch": 0.08142493638676845,
      "grad_norm": 60.67801960429597,
      "learning_rate": 6.4e-07,
      "logits/chosen": -0.9725692272186279,
      "logits/rejected": -0.9748388528823853,
      "logps/chosen": -1.5920709371566772,
      "logps/rejected": -1.7690041065216064,
      "loss": 4.7704,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -15.920707702636719,
      "rewards/margins": 1.7693322896957397,
      "rewards/rejected": -17.690040588378906,
      "step": 36
    },
    {
      "epoch": 0.08368674017528979,
      "grad_norm": 55.8652194329523,
      "learning_rate": 6.577777777777777e-07,
      "logits/chosen": -0.9925840497016907,
      "logits/rejected": -0.9903581142425537,
      "logps/chosen": -1.442215085029602,
      "logps/rejected": -1.468677282333374,
      "loss": 5.1667,
      "rewards/accuracies": 0.6015625,
      "rewards/chosen": -14.422150611877441,
      "rewards/margins": 0.26462242007255554,
      "rewards/rejected": -14.686773300170898,
      "step": 37
    },
    {
      "epoch": 0.08594854396381114,
      "grad_norm": 66.48789350704135,
      "learning_rate": 6.755555555555555e-07,
      "logits/chosen": -1.081889271736145,
      "logits/rejected": -1.0824934244155884,
      "logps/chosen": -1.6371712684631348,
      "logps/rejected": -1.6554210186004639,
      "loss": 5.278,
      "rewards/accuracies": 0.5703125,
      "rewards/chosen": -16.371713638305664,
      "rewards/margins": 0.182498037815094,
      "rewards/rejected": -16.554210662841797,
      "step": 38
    },
    {
      "epoch": 0.08821034775233248,
      "grad_norm": 60.77774068293974,
      "learning_rate": 6.933333333333333e-07,
      "logits/chosen": -1.0623948574066162,
      "logits/rejected": -1.074435830116272,
      "logps/chosen": -1.5994151830673218,
      "logps/rejected": -1.6381984949111938,
      "loss": 5.0061,
      "rewards/accuracies": 0.5546875,
      "rewards/chosen": -15.99415111541748,
      "rewards/margins": 0.38783279061317444,
      "rewards/rejected": -16.38198471069336,
      "step": 39
    },
    {
      "epoch": 0.09047215154085383,
      "grad_norm": 38.115575176568385,
      "learning_rate": 7.111111111111111e-07,
      "logits/chosen": -1.0157631635665894,
      "logits/rejected": -0.9986149668693542,
      "logps/chosen": -1.4888092279434204,
      "logps/rejected": -1.7236121892929077,
      "loss": 3.922,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -14.888091087341309,
      "rewards/margins": 2.348029613494873,
      "rewards/rejected": -17.236122131347656,
      "step": 40
    },
    {
      "epoch": 0.09273395532937517,
      "grad_norm": 38.36474699448423,
      "learning_rate": 7.288888888888888e-07,
      "logits/chosen": -1.060340404510498,
      "logits/rejected": -1.0315660238265991,
      "logps/chosen": -1.456924319267273,
      "logps/rejected": -1.6312799453735352,
      "loss": 4.4318,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -14.569242477416992,
      "rewards/margins": 1.7435554265975952,
      "rewards/rejected": -16.31279945373535,
      "step": 41
    },
    {
      "epoch": 0.09499575911789652,
      "grad_norm": 68.97554618193577,
      "learning_rate": 7.466666666666667e-07,
      "logits/chosen": -0.9593913555145264,
      "logits/rejected": -0.9225877523422241,
      "logps/chosen": -1.5931707620620728,
      "logps/rejected": -1.6982492208480835,
      "loss": 5.0405,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": -15.931707382202148,
      "rewards/margins": 1.0507832765579224,
      "rewards/rejected": -16.982492446899414,
      "step": 42
    },
    {
      "epoch": 0.09725756290641786,
      "grad_norm": 58.86006592276919,
      "learning_rate": 7.644444444444444e-07,
      "logits/chosen": -1.0317012071609497,
      "logits/rejected": -1.0032808780670166,
      "logps/chosen": -1.561445951461792,
      "logps/rejected": -1.6140797138214111,
      "loss": 5.0187,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -15.614459037780762,
      "rewards/margins": 0.5263394117355347,
      "rewards/rejected": -16.140798568725586,
      "step": 43
    },
    {
      "epoch": 0.09951936669493922,
      "grad_norm": 40.694422648420655,
      "learning_rate": 7.822222222222222e-07,
      "logits/chosen": -0.9743322730064392,
      "logits/rejected": -0.9567911624908447,
      "logps/chosen": -1.4267774820327759,
      "logps/rejected": -1.591205358505249,
      "loss": 4.1227,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -14.26777458190918,
      "rewards/margins": 1.644278883934021,
      "rewards/rejected": -15.912054061889648,
      "step": 44
    },
    {
      "epoch": 0.10178117048346055,
      "grad_norm": 40.8715446540072,
      "learning_rate": 8e-07,
      "logits/chosen": -1.0036640167236328,
      "logits/rejected": -0.9986906051635742,
      "logps/chosen": -1.4704385995864868,
      "logps/rejected": -1.6148722171783447,
      "loss": 4.5268,
      "rewards/accuracies": 0.6015625,
      "rewards/chosen": -14.704385757446289,
      "rewards/margins": 1.4443360567092896,
      "rewards/rejected": -16.14872169494629,
      "step": 45
    },
    {
      "epoch": 0.1040429742719819,
      "grad_norm": 64.76142197420904,
      "learning_rate": 7.999874759018868e-07,
      "logits/chosen": -1.0634511709213257,
      "logits/rejected": -1.0455323457717896,
      "logps/chosen": -1.7534921169281006,
      "logps/rejected": -1.9212398529052734,
      "loss": 4.5489,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -17.534921646118164,
      "rewards/margins": 1.6774773597717285,
      "rewards/rejected": -19.212400436401367,
      "step": 46
    },
    {
      "epoch": 0.10630477806050326,
      "grad_norm": 43.36406398315485,
      "learning_rate": 7.999499043918123e-07,
      "logits/chosen": -1.0629500150680542,
      "logits/rejected": -1.0646634101867676,
      "logps/chosen": -1.575823187828064,
      "logps/rejected": -1.6532174348831177,
      "loss": 4.9532,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -15.758233070373535,
      "rewards/margins": 0.7739435434341431,
      "rewards/rejected": -16.532176971435547,
      "step": 47
    },
    {
      "epoch": 0.1085665818490246,
      "grad_norm": 44.32249342002411,
      "learning_rate": 7.998872878225228e-07,
      "logits/chosen": -0.9744206666946411,
      "logits/rejected": -0.9684075117111206,
      "logps/chosen": -1.603161334991455,
      "logps/rejected": -1.7590126991271973,
      "loss": 4.35,
      "rewards/accuracies": 0.6015625,
      "rewards/chosen": -16.0316104888916,
      "rewards/margins": 1.5585150718688965,
      "rewards/rejected": -17.590126037597656,
      "step": 48
    },
    {
      "epoch": 0.11082838563754595,
      "grad_norm": 45.52706437580373,
      "learning_rate": 7.997996301150987e-07,
      "logits/chosen": -0.9781917929649353,
      "logits/rejected": -0.9805136919021606,
      "logps/chosen": -1.5446103811264038,
      "logps/rejected": -1.670082926750183,
      "loss": 4.6801,
      "rewards/accuracies": 0.5859375,
      "rewards/chosen": -15.446105003356934,
      "rewards/margins": 1.2547240257263184,
      "rewards/rejected": -16.70082664489746,
      "step": 49
    },
    {
      "epoch": 0.11309018942606729,
      "grad_norm": 48.545636464126815,
      "learning_rate": 7.996869367587088e-07,
      "logits/chosen": -0.9543410539627075,
      "logits/rejected": -0.9468324780464172,
      "logps/chosen": -1.639030933380127,
      "logps/rejected": -1.7508221864700317,
      "loss": 4.7132,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -16.390310287475586,
      "rewards/margins": 1.1179133653640747,
      "rewards/rejected": -17.508222579956055,
      "step": 50
    },
    {
      "epoch": 0.11535199321458864,
      "grad_norm": 43.1716994169158,
      "learning_rate": 7.99549214810266e-07,
      "logits/chosen": -0.957094669342041,
      "logits/rejected": -0.9495903849601746,
      "logps/chosen": -1.6492178440093994,
      "logps/rejected": -1.7528547048568726,
      "loss": 4.624,
      "rewards/accuracies": 0.6015625,
      "rewards/chosen": -16.49217987060547,
      "rewards/margins": 1.0363675355911255,
      "rewards/rejected": -17.528545379638672,
      "step": 51
    },
    {
      "epoch": 0.11761379700310998,
      "grad_norm": 44.767623959784785,
      "learning_rate": 7.993864728939867e-07,
      "logits/chosen": -0.9784611463546753,
      "logits/rejected": -0.9418229460716248,
      "logps/chosen": -1.6040714979171753,
      "logps/rejected": -1.7161141633987427,
      "loss": 4.8234,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -16.040714263916016,
      "rewards/margins": 1.1204277276992798,
      "rewards/rejected": -17.16114044189453,
      "step": 52
    },
    {
      "epoch": 0.11987560079163133,
      "grad_norm": 49.50003960371549,
      "learning_rate": 7.991987212008491e-07,
      "logits/chosen": -0.9920171499252319,
      "logits/rejected": -0.983026921749115,
      "logps/chosen": -1.6678504943847656,
      "logps/rejected": -1.9023908376693726,
      "loss": 4.2403,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -16.678503036499023,
      "rewards/margins": 2.345404624938965,
      "rewards/rejected": -19.023910522460938,
      "step": 53
    },
    {
      "epoch": 0.12213740458015267,
      "grad_norm": 53.488107569112316,
      "learning_rate": 7.989859714879565e-07,
      "logits/chosen": -1.0169801712036133,
      "logits/rejected": -0.9922108054161072,
      "logps/chosen": -1.6226848363876343,
      "logps/rejected": -1.7421677112579346,
      "loss": 4.7775,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": -16.226848602294922,
      "rewards/margins": 1.1948294639587402,
      "rewards/rejected": -17.42167854309082,
      "step": 54
    },
    {
      "epoch": 0.12439920836867402,
      "grad_norm": 47.80071251117602,
      "learning_rate": 7.987482370778005e-07,
      "logits/chosen": -0.9851608276367188,
      "logits/rejected": -0.9729430675506592,
      "logps/chosen": -1.675248622894287,
      "logps/rejected": -1.8124985694885254,
      "loss": 4.6785,
      "rewards/accuracies": 0.5703125,
      "rewards/chosen": -16.752485275268555,
      "rewards/margins": 1.372498869895935,
      "rewards/rejected": -18.124984741210938,
      "step": 55
    },
    {
      "epoch": 0.12666101215719536,
      "grad_norm": 54.191735218964155,
      "learning_rate": 7.984855328574262e-07,
      "logits/chosen": -0.8548079133033752,
      "logits/rejected": -0.8540121912956238,
      "logps/chosen": -1.6108894348144531,
      "logps/rejected": -1.752925992012024,
      "loss": 4.4245,
      "rewards/accuracies": 0.6171875,
      "rewards/chosen": -16.10889434814453,
      "rewards/margins": 1.4203639030456543,
      "rewards/rejected": -17.529258728027344,
      "step": 56
    },
    {
      "epoch": 0.1289228159457167,
      "grad_norm": 67.34737943017025,
      "learning_rate": 7.981978752775009e-07,
      "logits/chosen": -0.9092062711715698,
      "logits/rejected": -0.9146596789360046,
      "logps/chosen": -1.70232093334198,
      "logps/rejected": -1.8453538417816162,
      "loss": 4.6952,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -17.023208618164062,
      "rewards/margins": 1.4303276538848877,
      "rewards/rejected": -18.45353889465332,
      "step": 57
    },
    {
      "epoch": 0.13118461973423806,
      "grad_norm": 80.10334750091522,
      "learning_rate": 7.978852823512833e-07,
      "logits/chosen": -0.9658604264259338,
      "logits/rejected": -0.9477964639663696,
      "logps/chosen": -1.832795262336731,
      "logps/rejected": -1.9546037912368774,
      "loss": 5.0523,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -18.327951431274414,
      "rewards/margins": 1.2180850505828857,
      "rewards/rejected": -19.546037673950195,
      "step": 58
    },
    {
      "epoch": 0.1334464235227594,
      "grad_norm": 63.905912489444795,
      "learning_rate": 7.975477736534957e-07,
      "logits/chosen": -0.9663300514221191,
      "logits/rejected": -0.9616862535476685,
      "logps/chosen": -1.7907471656799316,
      "logps/rejected": -2.0022921562194824,
      "loss": 4.4589,
      "rewards/accuracies": 0.5859375,
      "rewards/chosen": -17.907470703125,
      "rewards/margins": 2.115452766418457,
      "rewards/rejected": -20.022924423217773,
      "step": 59
    },
    {
      "epoch": 0.13570822731128074,
      "grad_norm": 73.71974681164818,
      "learning_rate": 7.971853703190986e-07,
      "logits/chosen": -0.9496431350708008,
      "logits/rejected": -0.9395902752876282,
      "logps/chosen": -1.7495907545089722,
      "logps/rejected": -1.9543805122375488,
      "loss": 4.2887,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": -17.495908737182617,
      "rewards/margins": 2.047896146774292,
      "rewards/rejected": -19.543804168701172,
      "step": 60
    },
    {
      "epoch": 0.1379700310998021,
      "grad_norm": 55.31678343050937,
      "learning_rate": 7.967980950419664e-07,
      "logits/chosen": -0.8977174758911133,
      "logits/rejected": -0.9004334807395935,
      "logps/chosen": -1.683959722518921,
      "logps/rejected": -1.8546793460845947,
      "loss": 4.4567,
      "rewards/accuracies": 0.6640625,
      "rewards/chosen": -16.839595794677734,
      "rewards/margins": 1.7071975469589233,
      "rewards/rejected": -18.546794891357422,
      "step": 61
    },
    {
      "epoch": 0.14023183488832344,
      "grad_norm": 40.408238034098886,
      "learning_rate": 7.963859720734669e-07,
      "logits/chosen": -0.9545056223869324,
      "logits/rejected": -0.9535243511199951,
      "logps/chosen": -1.6104179620742798,
      "logps/rejected": -1.8246254920959473,
      "loss": 4.2358,
      "rewards/accuracies": 0.6328125,
      "rewards/chosen": -16.10417938232422,
      "rewards/margins": 2.142076253890991,
      "rewards/rejected": -18.24625587463379,
      "step": 62
    },
    {
      "epoch": 0.14249363867684478,
      "grad_norm": 39.63066442014652,
      "learning_rate": 7.959490272209427e-07,
      "logits/chosen": -0.9525138735771179,
      "logits/rejected": -0.9237796664237976,
      "logps/chosen": -1.6529213190078735,
      "logps/rejected": -1.9204621315002441,
      "loss": 3.7751,
      "rewards/accuracies": 0.6953125,
      "rewards/chosen": -16.52921485900879,
      "rewards/margins": 2.675405740737915,
      "rewards/rejected": -19.204618453979492,
      "step": 63
    },
    {
      "epoch": 0.14475544246536612,
      "grad_norm": 49.2257151600756,
      "learning_rate": 7.954872878460946e-07,
      "logits/chosen": -0.9712534546852112,
      "logits/rejected": -0.9322056770324707,
      "logps/chosen": -1.7236018180847168,
      "logps/rejected": -1.9554036855697632,
      "loss": 3.9722,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -17.23601722717285,
      "rewards/margins": 2.3180198669433594,
      "rewards/rejected": -19.554035186767578,
      "step": 64
    },
    {
      "epoch": 0.14701724625388748,
      "grad_norm": 56.40694891058032,
      "learning_rate": 7.950007828632691e-07,
      "logits/chosen": -0.9170076847076416,
      "logits/rejected": -0.9099224805831909,
      "logps/chosen": -1.8218014240264893,
      "logps/rejected": -2.114569902420044,
      "loss": 4.0631,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -18.218013763427734,
      "rewards/margins": 2.9276864528656006,
      "rewards/rejected": -21.145700454711914,
      "step": 65
    },
    {
      "epoch": 0.14927905004240882,
      "grad_norm": 54.86077177732332,
      "learning_rate": 7.944895427376465e-07,
      "logits/chosen": -0.9314538240432739,
      "logits/rejected": -0.9174329042434692,
      "logps/chosen": -1.799817681312561,
      "logps/rejected": -2.0671584606170654,
      "loss": 4.1606,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -17.99817657470703,
      "rewards/margins": 2.6734066009521484,
      "rewards/rejected": -20.67158317565918,
      "step": 66
    },
    {
      "epoch": 0.15154085383093016,
      "grad_norm": 40.53992007371899,
      "learning_rate": 7.939535994833345e-07,
      "logits/chosen": -0.8827767372131348,
      "logits/rejected": -0.8872506618499756,
      "logps/chosen": -1.6804907321929932,
      "logps/rejected": -1.9774214029312134,
      "loss": 3.9425,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -16.804906845092773,
      "rewards/margins": 2.969305992126465,
      "rewards/rejected": -19.774213790893555,
      "step": 67
    },
    {
      "epoch": 0.1538026576194515,
      "grad_norm": 51.36974889634882,
      "learning_rate": 7.933929866613628e-07,
      "logits/chosen": -0.9131999611854553,
      "logits/rejected": -0.9120370149612427,
      "logps/chosen": -1.746276617050171,
      "logps/rejected": -1.8965258598327637,
      "loss": 4.5149,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -17.462764739990234,
      "rewards/margins": 1.5024943351745605,
      "rewards/rejected": -18.965259552001953,
      "step": 68
    },
    {
      "epoch": 0.15606446140797287,
      "grad_norm": 50.79237381214548,
      "learning_rate": 7.928077393775808e-07,
      "logits/chosen": -0.887502133846283,
      "logits/rejected": -0.9122737050056458,
      "logps/chosen": -1.7796714305877686,
      "logps/rejected": -2.1390035152435303,
      "loss": 3.5203,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -17.796714782714844,
      "rewards/margins": 3.593320369720459,
      "rewards/rejected": -21.39003562927246,
      "step": 69
    },
    {
      "epoch": 0.1583262651964942,
      "grad_norm": 77.32537487087262,
      "learning_rate": 7.921978942804609e-07,
      "logits/chosen": -0.8942463397979736,
      "logits/rejected": -0.8875532150268555,
      "logps/chosen": -1.7755475044250488,
      "logps/rejected": -2.0664196014404297,
      "loss": 3.8556,
      "rewards/accuracies": 0.6640625,
      "rewards/chosen": -17.755474090576172,
      "rewards/margins": 2.908721446990967,
      "rewards/rejected": -20.664196014404297,
      "step": 70
    },
    {
      "epoch": 0.16058806898501554,
      "grad_norm": 62.6030871972903,
      "learning_rate": 7.915634895588021e-07,
      "logits/chosen": -0.9042387008666992,
      "logits/rejected": -0.8948806524276733,
      "logps/chosen": -1.8970236778259277,
      "logps/rejected": -2.0581259727478027,
      "loss": 4.513,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -18.970237731933594,
      "rewards/margins": 1.6110222339630127,
      "rewards/rejected": -20.581260681152344,
      "step": 71
    },
    {
      "epoch": 0.1628498727735369,
      "grad_norm": 61.94984380236303,
      "learning_rate": 7.909045649393394e-07,
      "logits/chosen": -0.9422912001609802,
      "logits/rejected": -0.9519181251525879,
      "logps/chosen": -1.7758185863494873,
      "logps/rejected": -1.9057157039642334,
      "loss": 4.5088,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -17.7581844329834,
      "rewards/margins": 1.2989716529846191,
      "rewards/rejected": -19.05715560913086,
      "step": 72
    },
    {
      "epoch": 0.16511167656205825,
      "grad_norm": 48.64018161030811,
      "learning_rate": 7.902211616842556e-07,
      "logits/chosen": -0.9227447509765625,
      "logits/rejected": -0.9130942821502686,
      "logps/chosen": -1.815474033355713,
      "logps/rejected": -2.099963426589966,
      "loss": 4.078,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -18.154739379882812,
      "rewards/margins": 2.84489369392395,
      "rewards/rejected": -20.9996337890625,
      "step": 73
    },
    {
      "epoch": 0.16737348035057958,
      "grad_norm": 58.56813481029154,
      "learning_rate": 7.89513322588598e-07,
      "logits/chosen": -0.8936547636985779,
      "logits/rejected": -0.8881016373634338,
      "logps/chosen": -1.8046737909317017,
      "logps/rejected": -2.0184521675109863,
      "loss": 3.8547,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -18.046737670898438,
      "rewards/margins": 2.1377835273742676,
      "rewards/rejected": -20.18452262878418,
      "step": 74
    },
    {
      "epoch": 0.16963528413910092,
      "grad_norm": 53.69434207416314,
      "learning_rate": 7.887810919775976e-07,
      "logits/chosen": -0.8241357207298279,
      "logits/rejected": -0.8239607810974121,
      "logps/chosen": -1.8949229717254639,
      "logps/rejected": -2.1345691680908203,
      "loss": 4.0357,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -18.949230194091797,
      "rewards/margins": 2.396462917327881,
      "rewards/rejected": -21.345691680908203,
      "step": 75
    },
    {
      "epoch": 0.1718970879276223,
      "grad_norm": 45.03380806243152,
      "learning_rate": 7.880245157038949e-07,
      "logits/chosen": -0.9108306169509888,
      "logits/rejected": -0.8901318907737732,
      "logps/chosen": -1.8972225189208984,
      "logps/rejected": -2.1310153007507324,
      "loss": 4.1242,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -18.972225189208984,
      "rewards/margins": 2.3379273414611816,
      "rewards/rejected": -21.310152053833008,
      "step": 76
    },
    {
      "epoch": 0.17415889171614363,
      "grad_norm": 77.27156743255722,
      "learning_rate": 7.872436411446671e-07,
      "logits/chosen": -0.9079168438911438,
      "logits/rejected": -0.9322432279586792,
      "logps/chosen": -1.9609839916229248,
      "logps/rejected": -2.1486620903015137,
      "loss": 4.4641,
      "rewards/accuracies": 0.6015625,
      "rewards/chosen": -19.609838485717773,
      "rewards/margins": 1.876783013343811,
      "rewards/rejected": -21.486618041992188,
      "step": 77
    },
    {
      "epoch": 0.17642069550466496,
      "grad_norm": 48.02529617691416,
      "learning_rate": 7.86438517198662e-07,
      "logits/chosen": -0.8568049073219299,
      "logits/rejected": -0.8431757688522339,
      "logps/chosen": -1.8570719957351685,
      "logps/rejected": -2.0501086711883545,
      "loss": 4.3324,
      "rewards/accuracies": 0.6484375,
      "rewards/chosen": -18.570720672607422,
      "rewards/margins": 1.9303661584854126,
      "rewards/rejected": -20.501087188720703,
      "step": 78
    },
    {
      "epoch": 0.1786824992931863,
      "grad_norm": 47.59975572650292,
      "learning_rate": 7.856091942831366e-07,
      "logits/chosen": -0.829656183719635,
      "logits/rejected": -0.8414878249168396,
      "logps/chosen": -1.829184651374817,
      "logps/rejected": -2.0620758533477783,
      "loss": 4.3785,
      "rewards/accuracies": 0.6640625,
      "rewards/chosen": -18.291847229003906,
      "rewards/margins": 2.3289124965667725,
      "rewards/rejected": -20.620759963989258,
      "step": 79
    },
    {
      "epoch": 0.18094430308170767,
      "grad_norm": 50.91304057566776,
      "learning_rate": 7.847557243306982e-07,
      "logits/chosen": -0.9277985095977783,
      "logits/rejected": -0.9156295657157898,
      "logps/chosen": -1.8894391059875488,
      "logps/rejected": -2.130729913711548,
      "loss": 3.9842,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -18.894390106201172,
      "rewards/margins": 2.412907361984253,
      "rewards/rejected": -21.30729866027832,
      "step": 80
    },
    {
      "epoch": 0.183206106870229,
      "grad_norm": 54.950758859746244,
      "learning_rate": 7.838781607860541e-07,
      "logits/chosen": -0.8941960334777832,
      "logits/rejected": -0.8841151595115662,
      "logps/chosen": -1.9732506275177002,
      "logps/rejected": -2.187934637069702,
      "loss": 3.7454,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -19.732505798339844,
      "rewards/margins": 2.146841526031494,
      "rewards/rejected": -21.87934684753418,
      "step": 81
    },
    {
      "epoch": 0.18546791065875035,
      "grad_norm": 49.11264696752389,
      "learning_rate": 7.82976558602664e-07,
      "logits/chosen": -0.9305046796798706,
      "logits/rejected": -0.9404510855674744,
      "logps/chosen": -1.898559808731079,
      "logps/rejected": -2.1002445220947266,
      "loss": 4.1062,
      "rewards/accuracies": 0.6640625,
      "rewards/chosen": -18.985597610473633,
      "rewards/margins": 2.016847848892212,
      "rewards/rejected": -21.0024471282959,
      "step": 82
    },
    {
      "epoch": 0.1877297144472717,
      "grad_norm": 76.5927273539599,
      "learning_rate": 7.820509742392988e-07,
      "logits/chosen": -0.919454038143158,
      "logits/rejected": -0.914941668510437,
      "logps/chosen": -2.0516703128814697,
      "logps/rejected": -2.226200819015503,
      "loss": 4.1924,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": -20.51670265197754,
      "rewards/margins": 1.745304822921753,
      "rewards/rejected": -22.262008666992188,
      "step": 83
    },
    {
      "epoch": 0.18999151823579305,
      "grad_norm": 89.96851686792246,
      "learning_rate": 7.811014656565054e-07,
      "logits/chosen": -0.9175388813018799,
      "logits/rejected": -0.9047884345054626,
      "logps/chosen": -1.9007177352905273,
      "logps/rejected": -2.289841890335083,
      "loss": 3.5515,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -19.007177352905273,
      "rewards/margins": 3.891240119934082,
      "rewards/rejected": -22.898418426513672,
      "step": 84
    },
    {
      "epoch": 0.1922533220243144,
      "grad_norm": 68.354941784998,
      "learning_rate": 7.801280923129773e-07,
      "logits/chosen": -0.9134962558746338,
      "logits/rejected": -0.915255069732666,
      "logps/chosen": -1.979697823524475,
      "logps/rejected": -2.148502826690674,
      "loss": 4.4942,
      "rewards/accuracies": 0.6328125,
      "rewards/chosen": -19.79697608947754,
      "rewards/margins": 1.6880521774291992,
      "rewards/rejected": -21.485031127929688,
      "step": 85
    },
    {
      "epoch": 0.19451512581283573,
      "grad_norm": 70.68741427366325,
      "learning_rate": 7.791309151618305e-07,
      "logits/chosen": -0.904114842414856,
      "logits/rejected": -0.9040198922157288,
      "logps/chosen": -2.137181282043457,
      "logps/rejected": -2.3369667530059814,
      "loss": 4.2346,
      "rewards/accuracies": 0.6015625,
      "rewards/chosen": -21.37181282043457,
      "rewards/margins": 1.9978532791137695,
      "rewards/rejected": -23.36966323852539,
      "step": 86
    },
    {
      "epoch": 0.1967769296013571,
      "grad_norm": 49.78117435262967,
      "learning_rate": 7.781099966467874e-07,
      "logits/chosen": -0.9545247554779053,
      "logits/rejected": -0.9480579495429993,
      "logps/chosen": -1.931373119354248,
      "logps/rejected": -2.1317574977874756,
      "loss": 3.7192,
      "rewards/accuracies": 0.6953125,
      "rewards/chosen": -19.313732147216797,
      "rewards/margins": 2.0038418769836426,
      "rewards/rejected": -21.317575454711914,
      "step": 87
    },
    {
      "epoch": 0.19903873338987843,
      "grad_norm": 87.19584039986934,
      "learning_rate": 7.770654006982664e-07,
      "logits/chosen": -0.9274868965148926,
      "logits/rejected": -0.907908022403717,
      "logps/chosen": -2.2163946628570557,
      "logps/rejected": -2.405592918395996,
      "loss": 4.4877,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -22.16394805908203,
      "rewards/margins": 1.8919786214828491,
      "rewards/rejected": -24.055925369262695,
      "step": 88
    },
    {
      "epoch": 0.20130053717839977,
      "grad_norm": 67.3834157436242,
      "learning_rate": 7.759971927293781e-07,
      "logits/chosen": -0.9331269264221191,
      "logits/rejected": -0.9273476600646973,
      "logps/chosen": -2.0026259422302246,
      "logps/rejected": -2.2201406955718994,
      "loss": 3.9806,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -20.02625846862793,
      "rewards/margins": 2.175147533416748,
      "rewards/rejected": -22.201404571533203,
      "step": 89
    },
    {
      "epoch": 0.2035623409669211,
      "grad_norm": 67.10853700406643,
      "learning_rate": 7.749054396318297e-07,
      "logits/chosen": -0.9318501353263855,
      "logits/rejected": -0.9156869649887085,
      "logps/chosen": -2.1561312675476074,
      "logps/rejected": -2.351287364959717,
      "loss": 4.0478,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": -21.56131362915039,
      "rewards/margins": 1.9515619277954102,
      "rewards/rejected": -23.512874603271484,
      "step": 90
    },
    {
      "epoch": 0.20582414475544247,
      "grad_norm": 80.66942714303244,
      "learning_rate": 7.737902097717356e-07,
      "logits/chosen": -0.9083584547042847,
      "logits/rejected": -0.9206920266151428,
      "logps/chosen": -2.1059789657592773,
      "logps/rejected": -2.380828857421875,
      "loss": 4.1678,
      "rewards/accuracies": 0.6171875,
      "rewards/chosen": -21.059789657592773,
      "rewards/margins": 2.748499870300293,
      "rewards/rejected": -23.80828857421875,
      "step": 91
    },
    {
      "epoch": 0.2080859485439638,
      "grad_norm": 69.60271028625591,
      "learning_rate": 7.726515729853367e-07,
      "logits/chosen": -0.9099538922309875,
      "logits/rejected": -0.9193957448005676,
      "logps/chosen": -2.098288059234619,
      "logps/rejected": -2.2589056491851807,
      "loss": 4.3746,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -20.98288345336914,
      "rewards/margins": 1.6061725616455078,
      "rewards/rejected": -22.58905601501465,
      "step": 92
    },
    {
      "epoch": 0.21034775233248515,
      "grad_norm": 73.95526859841975,
      "learning_rate": 7.714896005746272e-07,
      "logits/chosen": -0.9390743970870972,
      "logits/rejected": -0.9398510456085205,
      "logps/chosen": -2.198225975036621,
      "logps/rejected": -2.485823631286621,
      "loss": 3.6005,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": -21.982261657714844,
      "rewards/margins": 2.875973701477051,
      "rewards/rejected": -24.858234405517578,
      "step": 93
    },
    {
      "epoch": 0.21260955612100652,
      "grad_norm": 115.43649279446808,
      "learning_rate": 7.703043653028896e-07,
      "logits/chosen": -0.9868816137313843,
      "logits/rejected": -0.9943164587020874,
      "logps/chosen": -2.428180694580078,
      "logps/rejected": -2.6686596870422363,
      "loss": 4.1693,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -24.281808853149414,
      "rewards/margins": 2.4047892093658447,
      "rewards/rejected": -26.68659782409668,
      "step": 94
    },
    {
      "epoch": 0.21487135990952785,
      "grad_norm": 84.91161124024543,
      "learning_rate": 7.690959413901379e-07,
      "logits/chosen": -0.9326856732368469,
      "logits/rejected": -0.9249070882797241,
      "logps/chosen": -2.2586255073547363,
      "logps/rejected": -2.5268056392669678,
      "loss": 4.1,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -22.586254119873047,
      "rewards/margins": 2.6818017959594727,
      "rewards/rejected": -25.268056869506836,
      "step": 95
    },
    {
      "epoch": 0.2171331636980492,
      "grad_norm": 75.95382221779049,
      "learning_rate": 7.678644045084704e-07,
      "logits/chosen": -0.8716304302215576,
      "logits/rejected": -0.9014454483985901,
      "logps/chosen": -2.1552388668060303,
      "logps/rejected": -2.4034106731414795,
      "loss": 4.2664,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -21.55238914489746,
      "rewards/margins": 2.4817161560058594,
      "rewards/rejected": -24.034103393554688,
      "step": 96
    },
    {
      "epoch": 0.21939496748657053,
      "grad_norm": 71.70588534717525,
      "learning_rate": 7.666098317773308e-07,
      "logits/chosen": -0.9344345331192017,
      "logits/rejected": -0.9383649826049805,
      "logps/chosen": -2.3225467205047607,
      "logps/rejected": -2.5641982555389404,
      "loss": 3.6566,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -23.2254638671875,
      "rewards/margins": 2.416515827178955,
      "rewards/rejected": -25.641983032226562,
      "step": 97
    },
    {
      "epoch": 0.2216567712750919,
      "grad_norm": 63.70209243196811,
      "learning_rate": 7.653323017586789e-07,
      "logits/chosen": -0.9357935786247253,
      "logits/rejected": -0.9202609062194824,
      "logps/chosen": -2.091855525970459,
      "logps/rejected": -2.3324010372161865,
      "loss": 3.7794,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -20.91855239868164,
      "rewards/margins": 2.4054555892944336,
      "rewards/rejected": -23.32400894165039,
      "step": 98
    },
    {
      "epoch": 0.22391857506361323,
      "grad_norm": 85.72067726547178,
      "learning_rate": 7.640318944520711e-07,
      "logits/chosen": -0.9170993566513062,
      "logits/rejected": -0.9224846363067627,
      "logps/chosen": -2.2841103076934814,
      "logps/rejected": -2.5552611351013184,
      "loss": 3.5253,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -22.841102600097656,
      "rewards/margins": 2.7115087509155273,
      "rewards/rejected": -25.552610397338867,
      "step": 99
    },
    {
      "epoch": 0.22618037885213457,
      "grad_norm": 496.5312096722674,
      "learning_rate": 7.627086912896511e-07,
      "logits/chosen": -0.8243163824081421,
      "logits/rejected": -0.8487036824226379,
      "logps/chosen": -2.1957106590270996,
      "logps/rejected": -2.4948227405548096,
      "loss": 3.5183,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -21.95710563659668,
      "rewards/margins": 2.9911231994628906,
      "rewards/rejected": -24.948226928710938,
      "step": 100
    },
    {
      "epoch": 0.2284421826406559,
      "grad_norm": 70.0697669735273,
      "learning_rate": 7.613627751310499e-07,
      "logits/chosen": -0.95480877161026,
      "logits/rejected": -0.9551037549972534,
      "logps/chosen": -2.3536200523376465,
      "logps/rejected": -2.6055948734283447,
      "loss": 3.5656,
      "rewards/accuracies": 0.6953125,
      "rewards/chosen": -23.53619956970215,
      "rewards/margins": 2.5197482109069824,
      "rewards/rejected": -26.055946350097656,
      "step": 101
    },
    {
      "epoch": 0.23070398642917728,
      "grad_norm": 88.93036785903197,
      "learning_rate": 7.599942302581977e-07,
      "logits/chosen": -0.9218271374702454,
      "logits/rejected": -0.9325077533721924,
      "logps/chosen": -2.3475117683410645,
      "logps/rejected": -2.683833122253418,
      "loss": 3.4855,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -23.47511863708496,
      "rewards/margins": 3.363212823867798,
      "rewards/rejected": -26.83833122253418,
      "step": 102
    },
    {
      "epoch": 0.23296579021769862,
      "grad_norm": 90.0166632810772,
      "learning_rate": 7.586031423700457e-07,
      "logits/chosen": -0.9283229112625122,
      "logits/rejected": -0.9252837896347046,
      "logps/chosen": -2.3153982162475586,
      "logps/rejected": -2.582225799560547,
      "loss": 3.9243,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -23.153982162475586,
      "rewards/margins": 2.6682748794555664,
      "rewards/rejected": -25.822254180908203,
      "step": 103
    },
    {
      "epoch": 0.23522759400621995,
      "grad_norm": 82.46023897895928,
      "learning_rate": 7.571895985772e-07,
      "logits/chosen": -0.8776436448097229,
      "logits/rejected": -0.897273063659668,
      "logps/chosen": -2.323317050933838,
      "logps/rejected": -2.699765205383301,
      "loss": 3.1857,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -23.233171463012695,
      "rewards/margins": 3.7644832134246826,
      "rewards/rejected": -26.99765396118164,
      "step": 104
    },
    {
      "epoch": 0.23748939779474132,
      "grad_norm": 82.19454890744656,
      "learning_rate": 7.557536873964661e-07,
      "logits/chosen": -0.9673488736152649,
      "logits/rejected": -0.9688251614570618,
      "logps/chosen": -2.662144660949707,
      "logps/rejected": -2.884061098098755,
      "loss": 4.0931,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -26.62144660949707,
      "rewards/margins": 2.219163417816162,
      "rewards/rejected": -28.84061050415039,
      "step": 105
    },
    {
      "epoch": 0.23975120158326266,
      "grad_norm": 69.82415627225754,
      "learning_rate": 7.542954987453069e-07,
      "logits/chosen": -0.9264833927154541,
      "logits/rejected": -0.9302193522453308,
      "logps/chosen": -2.4886820316314697,
      "logps/rejected": -2.736279010772705,
      "loss": 3.7142,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -24.88681983947754,
      "rewards/margins": 2.475968837738037,
      "rewards/rejected": -27.362789154052734,
      "step": 106
    },
    {
      "epoch": 0.242013005371784,
      "grad_norm": 97.8985386292832,
      "learning_rate": 7.528151239362108e-07,
      "logits/chosen": -0.9315553903579712,
      "logits/rejected": -0.9390649199485779,
      "logps/chosen": -2.59812068939209,
      "logps/rejected": -2.882211685180664,
      "loss": 3.6407,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -25.981210708618164,
      "rewards/margins": 2.840907096862793,
      "rewards/rejected": -28.82211685180664,
      "step": 107
    },
    {
      "epoch": 0.24427480916030533,
      "grad_norm": 92.40475152000312,
      "learning_rate": 7.513126556709748e-07,
      "logits/chosen": -0.8942272067070007,
      "logits/rejected": -0.8834237456321716,
      "logps/chosen": -2.5529963970184326,
      "logps/rejected": -2.9440810680389404,
      "loss": 3.287,
      "rewards/accuracies": 0.6953125,
      "rewards/chosen": -25.529958724975586,
      "rewards/margins": 3.9108479022979736,
      "rewards/rejected": -29.440811157226562,
      "step": 108
    },
    {
      "epoch": 0.2465366129488267,
      "grad_norm": 84.45808750482125,
      "learning_rate": 7.497881880348984e-07,
      "logits/chosen": -0.853496789932251,
      "logits/rejected": -0.8538418412208557,
      "logps/chosen": -2.599656581878662,
      "logps/rejected": -2.9151229858398438,
      "loss": 3.72,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -25.996566772460938,
      "rewards/margins": 3.154664993286133,
      "rewards/rejected": -29.151229858398438,
      "step": 109
    },
    {
      "epoch": 0.24879841673734804,
      "grad_norm": 108.36449765897002,
      "learning_rate": 7.482418164908931e-07,
      "logits/chosen": -0.8664608001708984,
      "logits/rejected": -0.8705554604530334,
      "logps/chosen": -2.702484607696533,
      "logps/rejected": -2.9504551887512207,
      "loss": 3.7899,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -27.024845123291016,
      "rewards/margins": 2.479705572128296,
      "rewards/rejected": -29.504549026489258,
      "step": 110
    },
    {
      "epoch": 0.2510602205258694,
      "grad_norm": 97.25049221586009,
      "learning_rate": 7.466736378735035e-07,
      "logits/chosen": -0.8632520437240601,
      "logits/rejected": -0.8773136138916016,
      "logps/chosen": -2.784451484680176,
      "logps/rejected": -3.137439250946045,
      "loss": 3.5651,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -27.84451675415039,
      "rewards/margins": 3.529878616333008,
      "rewards/rejected": -31.374393463134766,
      "step": 111
    },
    {
      "epoch": 0.2533220243143907,
      "grad_norm": 89.03725814368798,
      "learning_rate": 7.450837503828439e-07,
      "logits/chosen": -0.8495292663574219,
      "logits/rejected": -0.8434633612632751,
      "logps/chosen": -2.8906612396240234,
      "logps/rejected": -3.3082571029663086,
      "loss": 3.3517,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -28.90660858154297,
      "rewards/margins": 4.175960063934326,
      "rewards/rejected": -33.08256912231445,
      "step": 112
    },
    {
      "epoch": 0.2555838281029121,
      "grad_norm": 83.1197248364895,
      "learning_rate": 7.43472253578449e-07,
      "logits/chosen": -0.8363775610923767,
      "logits/rejected": -0.8391093611717224,
      "logps/chosen": -2.605562925338745,
      "logps/rejected": -2.962904930114746,
      "loss": 3.435,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -26.055631637573242,
      "rewards/margins": 3.5734176635742188,
      "rewards/rejected": -29.629047393798828,
      "step": 113
    },
    {
      "epoch": 0.2578456318914334,
      "grad_norm": 96.33426382345725,
      "learning_rate": 7.418392483730389e-07,
      "logits/chosen": -0.8500136137008667,
      "logits/rejected": -0.8644757270812988,
      "logps/chosen": -2.8558506965637207,
      "logps/rejected": -3.2193422317504883,
      "loss": 3.3585,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -28.558509826660156,
      "rewards/margins": 3.634913444519043,
      "rewards/rejected": -32.19342041015625,
      "step": 114
    },
    {
      "epoch": 0.26010743567995476,
      "grad_norm": 87.46521771371307,
      "learning_rate": 7.401848370262012e-07,
      "logits/chosen": -0.8887827396392822,
      "logits/rejected": -0.8773460984230042,
      "logps/chosen": -2.8665177822113037,
      "logps/rejected": -3.1353914737701416,
      "loss": 3.7008,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -28.665176391601562,
      "rewards/margins": 2.688736915588379,
      "rewards/rejected": -31.353914260864258,
      "step": 115
    },
    {
      "epoch": 0.2623692394684761,
      "grad_norm": 104.59325034684498,
      "learning_rate": 7.385091231379856e-07,
      "logits/chosen": -0.8525288701057434,
      "logits/rejected": -0.854878842830658,
      "logps/chosen": -3.076798439025879,
      "logps/rejected": -3.424906015396118,
      "loss": 3.7519,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -30.767982482910156,
      "rewards/margins": 3.481076717376709,
      "rewards/rejected": -34.249061584472656,
      "step": 116
    },
    {
      "epoch": 0.26463104325699743,
      "grad_norm": 86.85796272452473,
      "learning_rate": 7.368122116424182e-07,
      "logits/chosen": -0.8492611646652222,
      "logits/rejected": -0.8522269129753113,
      "logps/chosen": -3.021807909011841,
      "logps/rejected": -3.3991785049438477,
      "loss": 3.4941,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -30.21807861328125,
      "rewards/margins": 3.773710250854492,
      "rewards/rejected": -33.991783142089844,
      "step": 117
    },
    {
      "epoch": 0.2668928470455188,
      "grad_norm": 111.1000685978278,
      "learning_rate": 7.350942088009289e-07,
      "logits/chosen": -0.8642144203186035,
      "logits/rejected": -0.8728247284889221,
      "logps/chosen": -3.0954854488372803,
      "logps/rejected": -3.433528423309326,
      "loss": 3.2773,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -30.954856872558594,
      "rewards/margins": 3.3804259300231934,
      "rewards/rejected": -34.33528137207031,
      "step": 118
    },
    {
      "epoch": 0.26915465083404017,
      "grad_norm": 142.32137319032904,
      "learning_rate": 7.333552221956986e-07,
      "logits/chosen": -0.9806150197982788,
      "logits/rejected": -0.9809490442276001,
      "logps/chosen": -3.2465367317199707,
      "logps/rejected": -3.631196975708008,
      "loss": 3.7568,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -32.46536636352539,
      "rewards/margins": 3.8466014862060547,
      "rewards/rejected": -36.31196975708008,
      "step": 119
    },
    {
      "epoch": 0.2714164546225615,
      "grad_norm": 106.21999349893953,
      "learning_rate": 7.315953607229217e-07,
      "logits/chosen": -0.8744773268699646,
      "logits/rejected": -0.8868200778961182,
      "logps/chosen": -3.3663530349731445,
      "logps/rejected": -3.7398698329925537,
      "loss": 3.3962,
      "rewards/accuracies": 0.6953125,
      "rewards/chosen": -33.66353225708008,
      "rewards/margins": 3.73516845703125,
      "rewards/rejected": -37.39870071411133,
      "step": 120
    },
    {
      "epoch": 0.27367825841108284,
      "grad_norm": 136.21663214062764,
      "learning_rate": 7.298147345859869e-07,
      "logits/chosen": -0.8941931128501892,
      "logits/rejected": -0.9127393364906311,
      "logps/chosen": -3.2252919673919678,
      "logps/rejected": -3.617177963256836,
      "loss": 3.541,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -32.2529182434082,
      "rewards/margins": 3.918860912322998,
      "rewards/rejected": -36.171783447265625,
      "step": 121
    },
    {
      "epoch": 0.2759400621996042,
      "grad_norm": 110.82874244685783,
      "learning_rate": 7.280134552885762e-07,
      "logits/chosen": -0.8409429788589478,
      "logits/rejected": -0.8534374237060547,
      "logps/chosen": -3.2517788410186768,
      "logps/rejected": -3.6886167526245117,
      "loss": 3.2942,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -32.517791748046875,
      "rewards/margins": 4.368377685546875,
      "rewards/rejected": -36.88616943359375,
      "step": 122
    },
    {
      "epoch": 0.2782018659881255,
      "grad_norm": 104.89134559764693,
      "learning_rate": 7.261916356276831e-07,
      "logits/chosen": -0.8516025543212891,
      "logits/rejected": -0.8547568917274475,
      "logps/chosen": -3.388638734817505,
      "logps/rejected": -3.9394216537475586,
      "loss": 2.8993,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -33.886390686035156,
      "rewards/margins": 5.507827281951904,
      "rewards/rejected": -39.39421463012695,
      "step": 123
    },
    {
      "epoch": 0.2804636697766469,
      "grad_norm": 97.54336648263664,
      "learning_rate": 7.243493896865486e-07,
      "logits/chosen": -0.8488821387290955,
      "logits/rejected": -0.8565881848335266,
      "logps/chosen": -3.166471481323242,
      "logps/rejected": -3.4654765129089355,
      "loss": 3.6222,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -31.664714813232422,
      "rewards/margins": 2.9900498390197754,
      "rewards/rejected": -34.65476608276367,
      "step": 124
    },
    {
      "epoch": 0.2827254735651682,
      "grad_norm": 172.00687462942722,
      "learning_rate": 7.224868328275169e-07,
      "logits/chosen": -0.8468244075775146,
      "logits/rejected": -0.8559240102767944,
      "logps/chosen": -3.2884538173675537,
      "logps/rejected": -3.7175629138946533,
      "loss": 3.5839,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -32.88453674316406,
      "rewards/margins": 4.291090965270996,
      "rewards/rejected": -37.175628662109375,
      "step": 125
    },
    {
      "epoch": 0.28498727735368956,
      "grad_norm": 174.35341145683532,
      "learning_rate": 7.206040816848126e-07,
      "logits/chosen": -0.852262020111084,
      "logits/rejected": -0.8540931940078735,
      "logps/chosen": -3.4291653633117676,
      "logps/rejected": -3.6988754272460938,
      "loss": 4.5346,
      "rewards/accuracies": 0.6953125,
      "rewards/chosen": -34.29165267944336,
      "rewards/margins": 2.6971054077148438,
      "rewards/rejected": -36.9887580871582,
      "step": 126
    },
    {
      "epoch": 0.2872490811422109,
      "grad_norm": 138.87306344888844,
      "learning_rate": 7.187012541572356e-07,
      "logits/chosen": -0.9179739952087402,
      "logits/rejected": -0.9147102236747742,
      "logps/chosen": -3.4903879165649414,
      "logps/rejected": -3.916646718978882,
      "loss": 3.4458,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -34.90387725830078,
      "rewards/margins": 4.2625837326049805,
      "rewards/rejected": -39.166465759277344,
      "step": 127
    },
    {
      "epoch": 0.28951088493073224,
      "grad_norm": 143.69937293237177,
      "learning_rate": 7.167784694007791e-07,
      "logits/chosen": -0.8328741192817688,
      "logits/rejected": -0.8579819798469543,
      "logps/chosen": -3.286675453186035,
      "logps/rejected": -3.76692533493042,
      "loss": 3.2917,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -32.866756439208984,
      "rewards/margins": 4.802499771118164,
      "rewards/rejected": -37.669254302978516,
      "step": 128
    },
    {
      "epoch": 0.2917726887192536,
      "grad_norm": 99.31418978127101,
      "learning_rate": 7.148358478211682e-07,
      "logits/chosen": -0.8990265130996704,
      "logits/rejected": -0.888577401638031,
      "logps/chosen": -3.4041242599487305,
      "logps/rejected": -3.8776817321777344,
      "loss": 2.8207,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -34.04124450683594,
      "rewards/margins": 4.735580921173096,
      "rewards/rejected": -38.77682113647461,
      "step": 129
    },
    {
      "epoch": 0.29403449250777497,
      "grad_norm": 102.38604318491103,
      "learning_rate": 7.128735110663187e-07,
      "logits/chosen": -0.8688659071922302,
      "logits/rejected": -0.8469004034996033,
      "logps/chosen": -3.0947279930114746,
      "logps/rejected": -3.5679235458374023,
      "loss": 3.3385,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": -30.947277069091797,
      "rewards/margins": 4.731960773468018,
      "rewards/rejected": -35.679237365722656,
      "step": 130
    },
    {
      "epoch": 0.2962962962962963,
      "grad_norm": 113.17524735613614,
      "learning_rate": 7.108915820187211e-07,
      "logits/chosen": -0.8502359986305237,
      "logits/rejected": -0.8502506017684937,
      "logps/chosen": -3.5472612380981445,
      "logps/rejected": -3.9762022495269775,
      "loss": 3.7675,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -35.47261047363281,
      "rewards/margins": 4.289410591125488,
      "rewards/rejected": -39.762020111083984,
      "step": 131
    },
    {
      "epoch": 0.29855810008481765,
      "grad_norm": 128.69199015701764,
      "learning_rate": 7.088901847877447e-07,
      "logits/chosen": -0.8379947543144226,
      "logits/rejected": -0.8463510274887085,
      "logps/chosen": -3.320732831954956,
      "logps/rejected": -3.5967845916748047,
      "loss": 4.558,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -33.20732498168945,
      "rewards/margins": 2.76051926612854,
      "rewards/rejected": -35.96784591674805,
      "step": 132
    },
    {
      "epoch": 0.300819903873339,
      "grad_norm": 122.95029199912098,
      "learning_rate": 7.068694447018658e-07,
      "logits/chosen": -0.8884780406951904,
      "logits/rejected": -0.8965522050857544,
      "logps/chosen": -3.27457857131958,
      "logps/rejected": -3.729426383972168,
      "loss": 3.4103,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -32.745792388916016,
      "rewards/margins": 4.5484747886657715,
      "rewards/rejected": -37.29426574707031,
      "step": 133
    },
    {
      "epoch": 0.3030817076618603,
      "grad_norm": 102.55252063015635,
      "learning_rate": 7.048294883008199e-07,
      "logits/chosen": -0.8615848422050476,
      "logits/rejected": -0.8659648895263672,
      "logps/chosen": -3.20782208442688,
      "logps/rejected": -3.573031425476074,
      "loss": 3.4383,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -32.07822036743164,
      "rewards/margins": 3.652094841003418,
      "rewards/rejected": -35.730316162109375,
      "step": 134
    },
    {
      "epoch": 0.3053435114503817,
      "grad_norm": 82.3321624317021,
      "learning_rate": 7.027704433276776e-07,
      "logits/chosen": -0.8441089391708374,
      "logits/rejected": -0.8458942174911499,
      "logps/chosen": -3.187887191772461,
      "logps/rejected": -3.677952289581299,
      "loss": 3.0593,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -31.878875732421875,
      "rewards/margins": 4.900643825531006,
      "rewards/rejected": -36.779518127441406,
      "step": 135
    },
    {
      "epoch": 0.307605315238903,
      "grad_norm": 188.5700545530925,
      "learning_rate": 7.006924387208452e-07,
      "logits/chosen": -0.8357016444206238,
      "logits/rejected": -0.8290910720825195,
      "logps/chosen": -3.086805820465088,
      "logps/rejected": -3.4287164211273193,
      "loss": 3.5074,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -30.868057250976562,
      "rewards/margins": 3.419107437133789,
      "rewards/rejected": -34.28716278076172,
      "step": 136
    },
    {
      "epoch": 0.30986711902742436,
      "grad_norm": 108.37290517834423,
      "learning_rate": 6.985956046059904e-07,
      "logits/chosen": -0.8443328142166138,
      "logits/rejected": -0.8402392268180847,
      "logps/chosen": -2.943553924560547,
      "logps/rejected": -3.3497257232666016,
      "loss": 3.691,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -29.43553924560547,
      "rewards/margins": 4.061717987060547,
      "rewards/rejected": -33.497257232666016,
      "step": 137
    },
    {
      "epoch": 0.31212892281594573,
      "grad_norm": 77.64006113050937,
      "learning_rate": 6.964800722878945e-07,
      "logits/chosen": -0.7976541519165039,
      "logits/rejected": -0.8123403787612915,
      "logps/chosen": -3.1234209537506104,
      "logps/rejected": -3.665281057357788,
      "loss": 3.0089,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -31.234210968017578,
      "rewards/margins": 5.418602466583252,
      "rewards/rejected": -36.65281295776367,
      "step": 138
    },
    {
      "epoch": 0.31439072660446704,
      "grad_norm": 84.56545363105515,
      "learning_rate": 6.943459742422287e-07,
      "logits/chosen": -0.8222934603691101,
      "logits/rejected": -0.8047088384628296,
      "logps/chosen": -3.0740787982940674,
      "logps/rejected": -3.537842035293579,
      "loss": 3.5096,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": -30.74079132080078,
      "rewards/margins": 4.63762903213501,
      "rewards/rejected": -35.37841796875,
      "step": 139
    },
    {
      "epoch": 0.3166525303929884,
      "grad_norm": 89.71503011819237,
      "learning_rate": 6.921934441072597e-07,
      "logits/chosen": -0.8978405594825745,
      "logits/rejected": -0.9020492434501648,
      "logps/chosen": -3.2600619792938232,
      "logps/rejected": -3.6400341987609863,
      "loss": 3.6164,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -32.60062026977539,
      "rewards/margins": 3.7997186183929443,
      "rewards/rejected": -36.40034484863281,
      "step": 140
    },
    {
      "epoch": 0.3189143341815098,
      "grad_norm": 105.19674463664295,
      "learning_rate": 6.900226166754807e-07,
      "logits/chosen": -0.8642122149467468,
      "logits/rejected": -0.8808465003967285,
      "logps/chosen": -3.301727533340454,
      "logps/rejected": -3.619366407394409,
      "loss": 3.953,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -33.017276763916016,
      "rewards/margins": 3.1763877868652344,
      "rewards/rejected": -36.19366455078125,
      "step": 141
    },
    {
      "epoch": 0.3211761379700311,
      "grad_norm": 107.67624941944278,
      "learning_rate": 6.8783362788517e-07,
      "logits/chosen": -0.8666509985923767,
      "logits/rejected": -0.8702815175056458,
      "logps/chosen": -3.2676572799682617,
      "logps/rejected": -3.6641600131988525,
      "loss": 4.0009,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -32.676578521728516,
      "rewards/margins": 3.9650261402130127,
      "rewards/rejected": -36.6416015625,
      "step": 142
    },
    {
      "epoch": 0.32343794175855245,
      "grad_norm": 99.32873615736939,
      "learning_rate": 6.856266148118796e-07,
      "logits/chosen": -0.8618003726005554,
      "logits/rejected": -0.8743412494659424,
      "logps/chosen": -3.03981614112854,
      "logps/rejected": -3.484009027481079,
      "loss": 3.265,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -30.398160934448242,
      "rewards/margins": 4.441934585571289,
      "rewards/rejected": -34.84009552001953,
      "step": 143
    },
    {
      "epoch": 0.3256997455470738,
      "grad_norm": 102.8416386415525,
      "learning_rate": 6.834017156598512e-07,
      "logits/chosen": -0.8412870764732361,
      "logits/rejected": -0.840274453163147,
      "logps/chosen": -3.2066195011138916,
      "logps/rejected": -3.617882490158081,
      "loss": 3.5015,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -32.06619644165039,
      "rewards/margins": 4.112632751464844,
      "rewards/rejected": -36.178829193115234,
      "step": 144
    },
    {
      "epoch": 0.3279615493355951,
      "grad_norm": 81.13201808680371,
      "learning_rate": 6.811590697533607e-07,
      "logits/chosen": -0.9234153628349304,
      "logits/rejected": -0.9473743438720703,
      "logps/chosen": -3.108992576599121,
      "logps/rejected": -3.5079336166381836,
      "loss": 3.3848,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -31.089927673339844,
      "rewards/margins": 3.9894118309020996,
      "rewards/rejected": -35.07933807373047,
      "step": 145
    },
    {
      "epoch": 0.3302233531241165,
      "grad_norm": 126.90983253589141,
      "learning_rate": 6.788988175279951e-07,
      "logits/chosen": -0.8728285431861877,
      "logits/rejected": -0.8665140271186829,
      "logps/chosen": -3.129547357559204,
      "logps/rejected": -3.460236072540283,
      "loss": 3.8778,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -31.295473098754883,
      "rewards/margins": 3.3068888187408447,
      "rewards/rejected": -34.602359771728516,
      "step": 146
    },
    {
      "epoch": 0.3324851569126378,
      "grad_norm": 91.06075264383888,
      "learning_rate": 6.766211005218577e-07,
      "logits/chosen": -0.8505449295043945,
      "logits/rejected": -0.8593109250068665,
      "logps/chosen": -3.05971622467041,
      "logps/rejected": -3.5910305976867676,
      "loss": 2.9708,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -30.59716033935547,
      "rewards/margins": 5.313144683837891,
      "rewards/rejected": -35.91030502319336,
      "step": 147
    },
    {
      "epoch": 0.33474696070115917,
      "grad_norm": 85.9903517839081,
      "learning_rate": 6.743260613667047e-07,
      "logits/chosen": -0.9269782900810242,
      "logits/rejected": -0.9214802980422974,
      "logps/chosen": -3.011857748031616,
      "logps/rejected": -3.465926170349121,
      "loss": 3.2708,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -30.11857795715332,
      "rewards/margins": 4.540684223175049,
      "rewards/rejected": -34.659263610839844,
      "step": 148
    },
    {
      "epoch": 0.33700876448968053,
      "grad_norm": 71.98194941268687,
      "learning_rate": 6.720138437790139e-07,
      "logits/chosen": -0.8807485103607178,
      "logits/rejected": -0.8728206157684326,
      "logps/chosen": -2.951597213745117,
      "logps/rejected": -3.371277332305908,
      "loss": 3.1961,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -29.515972137451172,
      "rewards/margins": 4.196802139282227,
      "rewards/rejected": -33.71277618408203,
      "step": 149
    },
    {
      "epoch": 0.33927056827820185,
      "grad_norm": 115.41856316684272,
      "learning_rate": 6.696845925509848e-07,
      "logits/chosen": -0.9250473976135254,
      "logits/rejected": -0.9204010963439941,
      "logps/chosen": -3.063525676727295,
      "logps/rejected": -3.4022319316864014,
      "loss": 3.6883,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -30.635255813598633,
      "rewards/margins": 3.387064218521118,
      "rewards/rejected": -34.02231979370117,
      "step": 150
    },
    {
      "epoch": 0.3415323720667232,
      "grad_norm": 101.95454524075308,
      "learning_rate": 6.673384535414718e-07,
      "logits/chosen": -0.9343738555908203,
      "logits/rejected": -0.9356783628463745,
      "logps/chosen": -3.1804661750793457,
      "logps/rejected": -3.466540575027466,
      "loss": 3.9923,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": -31.80466079711914,
      "rewards/margins": 2.860745906829834,
      "rewards/rejected": -34.665401458740234,
      "step": 151
    },
    {
      "epoch": 0.3437941758552446,
      "grad_norm": 94.85696940073606,
      "learning_rate": 6.649755736668511e-07,
      "logits/chosen": -0.8427945971488953,
      "logits/rejected": -0.8381005525588989,
      "logps/chosen": -2.847813367843628,
      "logps/rejected": -3.27738356590271,
      "loss": 2.7389,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -28.478132247924805,
      "rewards/margins": 4.295701026916504,
      "rewards/rejected": -32.77383804321289,
      "step": 152
    },
    {
      "epoch": 0.3460559796437659,
      "grad_norm": 92.66825859497754,
      "learning_rate": 6.625961008918192e-07,
      "logits/chosen": -0.8821796178817749,
      "logits/rejected": -0.8879325985908508,
      "logps/chosen": -2.928504467010498,
      "logps/rejected": -3.411726951599121,
      "loss": 2.8879,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -29.285043716430664,
      "rewards/margins": 4.832226753234863,
      "rewards/rejected": -34.117271423339844,
      "step": 153
    },
    {
      "epoch": 0.34831778343228725,
      "grad_norm": 104.67373787864423,
      "learning_rate": 6.602001842201289e-07,
      "logits/chosen": -0.8627166152000427,
      "logits/rejected": -0.8778930902481079,
      "logps/chosen": -2.948608160018921,
      "logps/rejected": -3.2540464401245117,
      "loss": 3.8502,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -29.486083984375,
      "rewards/margins": 3.0543811321258545,
      "rewards/rejected": -32.540462493896484,
      "step": 154
    },
    {
      "epoch": 0.3505795872208086,
      "grad_norm": 85.52036897226773,
      "learning_rate": 6.577879736852571e-07,
      "logits/chosen": -0.8922989368438721,
      "logits/rejected": -0.9067973494529724,
      "logps/chosen": -3.0298097133636475,
      "logps/rejected": -3.309849977493286,
      "loss": 3.789,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -30.298097610473633,
      "rewards/margins": 2.8003997802734375,
      "rewards/rejected": -33.0984992980957,
      "step": 155
    },
    {
      "epoch": 0.35284139100932993,
      "grad_norm": 95.84181793437827,
      "learning_rate": 6.553596203410112e-07,
      "logits/chosen": -0.8902891874313354,
      "logits/rejected": -0.8962793946266174,
      "logps/chosen": -3.0103700160980225,
      "logps/rejected": -3.539245843887329,
      "loss": 2.5342,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -30.103702545166016,
      "rewards/margins": 5.288753509521484,
      "rewards/rejected": -35.392459869384766,
      "step": 156
    },
    {
      "epoch": 0.3551031947978513,
      "grad_norm": 83.48045959211674,
      "learning_rate": 6.529152762520688e-07,
      "logits/chosen": -0.890064537525177,
      "logits/rejected": -0.8975638151168823,
      "logps/chosen": -3.166869878768921,
      "logps/rejected": -3.534972667694092,
      "loss": 3.3879,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -31.6686954498291,
      "rewards/margins": 3.681027889251709,
      "rewards/rejected": -35.34972381591797,
      "step": 157
    },
    {
      "epoch": 0.3573649985863726,
      "grad_norm": 104.36848999624989,
      "learning_rate": 6.504550944844558e-07,
      "logits/chosen": -0.8488032221794128,
      "logits/rejected": -0.8643382787704468,
      "logps/chosen": -3.09021258354187,
      "logps/rejected": -3.471135139465332,
      "loss": 3.4617,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -30.90212631225586,
      "rewards/margins": 3.8092260360717773,
      "rewards/rejected": -34.71135330200195,
      "step": 158
    },
    {
      "epoch": 0.359626802374894,
      "grad_norm": 104.97216908078047,
      "learning_rate": 6.479792290959613e-07,
      "logits/chosen": -0.8663011789321899,
      "logits/rejected": -0.8864620923995972,
      "logps/chosen": -3.1034600734710693,
      "logps/rejected": -3.6639652252197266,
      "loss": 3.03,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -31.03459930419922,
      "rewards/margins": 5.60504674911499,
      "rewards/rejected": -36.639652252197266,
      "step": 159
    },
    {
      "epoch": 0.36188860616341534,
      "grad_norm": 102.25000126815084,
      "learning_rate": 6.454878351264906e-07,
      "logits/chosen": -0.8334028720855713,
      "logits/rejected": -0.8324363231658936,
      "logps/chosen": -3.0704848766326904,
      "logps/rejected": -3.4776694774627686,
      "loss": 3.5175,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": -30.704849243164062,
      "rewards/margins": 4.07184362411499,
      "rewards/rejected": -34.776695251464844,
      "step": 160
    },
    {
      "epoch": 0.36415040995193665,
      "grad_norm": 109.80268605933277,
      "learning_rate": 6.429810685883565e-07,
      "logits/chosen": -0.8985482454299927,
      "logits/rejected": -0.8990384340286255,
      "logps/chosen": -3.2140891551971436,
      "logps/rejected": -3.6064000129699707,
      "loss": 3.1092,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -32.140892028808594,
      "rewards/margins": 3.9231081008911133,
      "rewards/rejected": -36.06399917602539,
      "step": 161
    },
    {
      "epoch": 0.366412213740458,
      "grad_norm": 108.55618372141429,
      "learning_rate": 6.404590864565088e-07,
      "logits/chosen": -0.8505939245223999,
      "logits/rejected": -0.8520810008049011,
      "logps/chosen": -3.1991541385650635,
      "logps/rejected": -3.4929707050323486,
      "loss": 3.8613,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -31.991540908813477,
      "rewards/margins": 2.9381637573242188,
      "rewards/rejected": -34.92970275878906,
      "step": 162
    },
    {
      "epoch": 0.3686740175289794,
      "grad_norm": 104.50273506595065,
      "learning_rate": 6.379220466587063e-07,
      "logits/chosen": -0.8530042171478271,
      "logits/rejected": -0.8485729694366455,
      "logps/chosen": -3.2691879272460938,
      "logps/rejected": -3.709401845932007,
      "loss": 3.0171,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -32.69187927246094,
      "rewards/margins": 4.4021406173706055,
      "rewards/rejected": -37.094017028808594,
      "step": 163
    },
    {
      "epoch": 0.3709358213175007,
      "grad_norm": 110.29027629310306,
      "learning_rate": 6.353701080656254e-07,
      "logits/chosen": -0.8432658910751343,
      "logits/rejected": -0.8815178275108337,
      "logps/chosen": -3.3604822158813477,
      "logps/rejected": -3.718573808670044,
      "loss": 3.4067,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -33.60482406616211,
      "rewards/margins": 3.580916166305542,
      "rewards/rejected": -37.18573760986328,
      "step": 164
    },
    {
      "epoch": 0.37319762510602206,
      "grad_norm": 89.50224289291396,
      "learning_rate": 6.32803430480913e-07,
      "logits/chosen": -0.8596310019493103,
      "logits/rejected": -0.8655349016189575,
      "logps/chosen": -3.311291456222534,
      "logps/rejected": -3.7834932804107666,
      "loss": 3.3279,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -33.1129150390625,
      "rewards/margins": 4.722017288208008,
      "rewards/rejected": -37.83493423461914,
      "step": 165
    },
    {
      "epoch": 0.3754594288945434,
      "grad_norm": 101.95377945602579,
      "learning_rate": 6.302221746311782e-07,
      "logits/chosen": -0.8723585605621338,
      "logits/rejected": -0.8533425331115723,
      "logps/chosen": -3.2129547595977783,
      "logps/rejected": -3.6323916912078857,
      "loss": 3.3934,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -32.129547119140625,
      "rewards/margins": 4.194368839263916,
      "rewards/rejected": -36.323917388916016,
      "step": 166
    },
    {
      "epoch": 0.37772123268306473,
      "grad_norm": 96.75845720827888,
      "learning_rate": 6.276265021559288e-07,
      "logits/chosen": -0.890640377998352,
      "logits/rejected": -0.8925571441650391,
      "logps/chosen": -3.5187129974365234,
      "logps/rejected": -3.8824410438537598,
      "loss": 3.7992,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -35.187129974365234,
      "rewards/margins": 3.637279987335205,
      "rewards/rejected": -38.82441329956055,
      "step": 167
    },
    {
      "epoch": 0.3799830364715861,
      "grad_norm": 87.40404880493536,
      "learning_rate": 6.250165755974487e-07,
      "logits/chosen": -0.8265157341957092,
      "logits/rejected": -0.835180938243866,
      "logps/chosen": -3.481316328048706,
      "logps/rejected": -3.9201066493988037,
      "loss": 3.0053,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -34.81316375732422,
      "rewards/margins": 4.387899398803711,
      "rewards/rejected": -39.20106506347656,
      "step": 168
    },
    {
      "epoch": 0.3822448402601074,
      "grad_norm": 103.05270142062696,
      "learning_rate": 6.223925583906192e-07,
      "logits/chosen": -0.8841734528541565,
      "logits/rejected": -0.8769547939300537,
      "logps/chosen": -3.5402333736419678,
      "logps/rejected": -4.006646156311035,
      "loss": 3.0164,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -35.4023323059082,
      "rewards/margins": 4.664129257202148,
      "rewards/rejected": -40.06645965576172,
      "step": 169
    },
    {
      "epoch": 0.3845066440486288,
      "grad_norm": 99.14006440661235,
      "learning_rate": 6.19754614852685e-07,
      "logits/chosen": -0.8786910772323608,
      "logits/rejected": -0.8795101046562195,
      "logps/chosen": -3.42849063873291,
      "logps/rejected": -3.857112407684326,
      "loss": 2.992,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -34.28490447998047,
      "rewards/margins": 4.286218166351318,
      "rewards/rejected": -38.57112503051758,
      "step": 170
    },
    {
      "epoch": 0.38676844783715014,
      "grad_norm": 101.36285623071498,
      "learning_rate": 6.171029101729644e-07,
      "logits/chosen": -0.8048484325408936,
      "logits/rejected": -0.8320644497871399,
      "logps/chosen": -3.6026647090911865,
      "logps/rejected": -4.107236385345459,
      "loss": 3.0985,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -36.02664566040039,
      "rewards/margins": 5.045715808868408,
      "rewards/rejected": -41.07236099243164,
      "step": 171
    },
    {
      "epoch": 0.38903025162567145,
      "grad_norm": 107.03781497013733,
      "learning_rate": 6.144376104025055e-07,
      "logits/chosen": -0.895035982131958,
      "logits/rejected": -0.8912597894668579,
      "logps/chosen": -3.4788260459899902,
      "logps/rejected": -3.9913995265960693,
      "loss": 2.8953,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -34.78825759887695,
      "rewards/margins": 5.125735759735107,
      "rewards/rejected": -39.91399383544922,
      "step": 172
    },
    {
      "epoch": 0.3912920554141928,
      "grad_norm": 111.20601005232534,
      "learning_rate": 6.117588824436873e-07,
      "logits/chosen": -0.9020571112632751,
      "logits/rejected": -0.9188283681869507,
      "logps/chosen": -3.605067491531372,
      "logps/rejected": -3.988225221633911,
      "loss": 3.5203,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -36.05067825317383,
      "rewards/margins": 3.8315775394439697,
      "rewards/rejected": -39.88224792480469,
      "step": 173
    },
    {
      "epoch": 0.3935538592027142,
      "grad_norm": 103.4654281484595,
      "learning_rate": 6.090668940397688e-07,
      "logits/chosen": -0.8591521978378296,
      "logits/rejected": -0.8686568737030029,
      "logps/chosen": -3.537184953689575,
      "logps/rejected": -4.049908638000488,
      "loss": 2.8987,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -35.371849060058594,
      "rewards/margins": 5.127236843109131,
      "rewards/rejected": -40.499088287353516,
      "step": 174
    },
    {
      "epoch": 0.3958156629912355,
      "grad_norm": 135.38904727694083,
      "learning_rate": 6.063618137643844e-07,
      "logits/chosen": -0.8473508358001709,
      "logits/rejected": -0.8524729609489441,
      "logps/chosen": -3.561941385269165,
      "logps/rejected": -4.00567626953125,
      "loss": 2.9943,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -35.61941146850586,
      "rewards/margins": 4.437344551086426,
      "rewards/rejected": -40.05675506591797,
      "step": 175
    },
    {
      "epoch": 0.39807746677975686,
      "grad_norm": 110.97675460831563,
      "learning_rate": 6.03643811010988e-07,
      "logits/chosen": -0.8901565670967102,
      "logits/rejected": -0.9126089811325073,
      "logps/chosen": -3.7035725116729736,
      "logps/rejected": -4.154231071472168,
      "loss": 2.9707,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -37.035728454589844,
      "rewards/margins": 4.506581783294678,
      "rewards/rejected": -41.54230499267578,
      "step": 176
    },
    {
      "epoch": 0.4003392705682782,
      "grad_norm": 124.1530845740034,
      "learning_rate": 6.009130559822453e-07,
      "logits/chosen": -0.9078172445297241,
      "logits/rejected": -0.921136736869812,
      "logps/chosen": -3.670597553253174,
      "logps/rejected": -3.9640297889709473,
      "loss": 4.0276,
      "rewards/accuracies": 0.6640625,
      "rewards/chosen": -36.70597457885742,
      "rewards/margins": 2.9343202114105225,
      "rewards/rejected": -39.64029312133789,
      "step": 177
    },
    {
      "epoch": 0.40260107435679954,
      "grad_norm": 112.28760201530416,
      "learning_rate": 5.981697196793758e-07,
      "logits/chosen": -0.9355087280273438,
      "logits/rejected": -0.9506640434265137,
      "logps/chosen": -3.7393956184387207,
      "logps/rejected": -4.199625015258789,
      "loss": 2.6604,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -37.39395523071289,
      "rewards/margins": 4.602291584014893,
      "rewards/rejected": -41.99625015258789,
      "step": 178
    },
    {
      "epoch": 0.4048628781453209,
      "grad_norm": 113.95118236789975,
      "learning_rate": 5.954139738914446e-07,
      "logits/chosen": -0.9104683995246887,
      "logits/rejected": -0.9266977310180664,
      "logps/chosen": -3.624338388442993,
      "logps/rejected": -4.1234588623046875,
      "loss": 3.0403,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -36.243385314941406,
      "rewards/margins": 4.991206169128418,
      "rewards/rejected": -41.23459243774414,
      "step": 179
    },
    {
      "epoch": 0.4071246819338422,
      "grad_norm": 116.42013410546693,
      "learning_rate": 5.92645991184605e-07,
      "logits/chosen": -0.8819618225097656,
      "logits/rejected": -0.9078726768493652,
      "logps/chosen": -3.8188674449920654,
      "logps/rejected": -4.311014652252197,
      "loss": 2.8079,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -38.18867492675781,
      "rewards/margins": 4.921470642089844,
      "rewards/rejected": -43.11014938354492,
      "step": 180
    },
    {
      "epoch": 0.4093864857223636,
      "grad_norm": 105.7185913445078,
      "learning_rate": 5.898659448912917e-07,
      "logits/chosen": -0.892175555229187,
      "logits/rejected": -0.9181370735168457,
      "logps/chosen": -3.6667990684509277,
      "logps/rejected": -4.1268720626831055,
      "loss": 3.2278,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -36.667991638183594,
      "rewards/margins": 4.600729465484619,
      "rewards/rejected": -41.26872253417969,
      "step": 181
    },
    {
      "epoch": 0.41164828951088495,
      "grad_norm": 113.58444674434676,
      "learning_rate": 5.870740090993676e-07,
      "logits/chosen": -0.9441367983818054,
      "logits/rejected": -0.9531031250953674,
      "logps/chosen": -3.8816134929656982,
      "logps/rejected": -4.417888641357422,
      "loss": 2.726,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -38.816131591796875,
      "rewards/margins": 5.362754821777344,
      "rewards/rejected": -44.178890228271484,
      "step": 182
    },
    {
      "epoch": 0.41391009329940626,
      "grad_norm": 132.693780202085,
      "learning_rate": 5.842703586412214e-07,
      "logits/chosen": -0.9231690168380737,
      "logits/rejected": -0.9451411962509155,
      "logps/chosen": -4.003310203552246,
      "logps/rejected": -4.392942428588867,
      "loss": 3.7509,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -40.033103942871094,
      "rewards/margins": 3.896320343017578,
      "rewards/rejected": -43.92942428588867,
      "step": 183
    },
    {
      "epoch": 0.4161718970879276,
      "grad_norm": 117.19432439519281,
      "learning_rate": 5.814551690828203e-07,
      "logits/chosen": -0.9079042673110962,
      "logits/rejected": -0.9319837093353271,
      "logps/chosen": -3.8635096549987793,
      "logps/rejected": -4.329761505126953,
      "loss": 2.9234,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -38.635093688964844,
      "rewards/margins": 4.662517547607422,
      "rewards/rejected": -43.29761505126953,
      "step": 184
    },
    {
      "epoch": 0.418433700876449,
      "grad_norm": 134.874370876785,
      "learning_rate": 5.786286167127155e-07,
      "logits/chosen": -0.946037769317627,
      "logits/rejected": -0.940789520740509,
      "logps/chosen": -3.8176631927490234,
      "logps/rejected": -4.328977584838867,
      "loss": 3.094,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -38.17662811279297,
      "rewards/margins": 5.1131415367126465,
      "rewards/rejected": -43.289772033691406,
      "step": 185
    },
    {
      "epoch": 0.4206955046649703,
      "grad_norm": 104.28377189007104,
      "learning_rate": 5.757908785310031e-07,
      "logits/chosen": -0.891565203666687,
      "logits/rejected": -0.90488600730896,
      "logps/chosen": -3.6350033283233643,
      "logps/rejected": -4.077045440673828,
      "loss": 3.2427,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -36.350032806396484,
      "rewards/margins": 4.420421600341797,
      "rewards/rejected": -40.770450592041016,
      "step": 186
    },
    {
      "epoch": 0.42295730845349166,
      "grad_norm": 90.6334711650778,
      "learning_rate": 5.729421322382399e-07,
      "logits/chosen": -0.8813575506210327,
      "logits/rejected": -0.902955174446106,
      "logps/chosen": -3.487731695175171,
      "logps/rejected": -4.006654739379883,
      "loss": 2.862,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -34.8773193359375,
      "rewards/margins": 5.1892290115356445,
      "rewards/rejected": -40.06654357910156,
      "step": 187
    },
    {
      "epoch": 0.42521911224201303,
      "grad_norm": 129.7937200032232,
      "learning_rate": 5.700825562243163e-07,
      "logits/chosen": -0.8732385039329529,
      "logits/rejected": -0.8936405777931213,
      "logps/chosen": -3.58575439453125,
      "logps/rejected": -4.113236427307129,
      "loss": 2.8991,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -35.8575439453125,
      "rewards/margins": 5.27481746673584,
      "rewards/rejected": -41.13235855102539,
      "step": 188
    },
    {
      "epoch": 0.42748091603053434,
      "grad_norm": 108.64585507889065,
      "learning_rate": 5.672123295572854e-07,
      "logits/chosen": -0.940618634223938,
      "logits/rejected": -0.9647533893585205,
      "logps/chosen": -3.6546216011047363,
      "logps/rejected": -4.055443286895752,
      "loss": 2.8708,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -36.54621887207031,
      "rewards/margins": 4.008213996887207,
      "rewards/rejected": -40.5544319152832,
      "step": 189
    },
    {
      "epoch": 0.4297427198190557,
      "grad_norm": 108.37528044511077,
      "learning_rate": 5.643316319721487e-07,
      "logits/chosen": -0.9153305292129517,
      "logits/rejected": -0.9312554001808167,
      "logps/chosen": -3.8006129264831543,
      "logps/rejected": -4.253608703613281,
      "loss": 3.5312,
      "rewards/accuracies": 0.6953125,
      "rewards/chosen": -38.00613021850586,
      "rewards/margins": 4.529964923858643,
      "rewards/rejected": -42.536094665527344,
      "step": 190
    },
    {
      "epoch": 0.432004523607577,
      "grad_norm": 100.72656998572613,
      "learning_rate": 5.614406438596026e-07,
      "logits/chosen": -0.9669473171234131,
      "logits/rejected": -0.9767682552337646,
      "logps/chosen": -3.853173017501831,
      "logps/rejected": -4.320462703704834,
      "loss": 3.3801,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -38.53173065185547,
      "rewards/margins": 4.672896385192871,
      "rewards/rejected": -43.20462417602539,
      "step": 191
    },
    {
      "epoch": 0.4342663273960984,
      "grad_norm": 108.78329119316179,
      "learning_rate": 5.585395462547406e-07,
      "logits/chosen": -0.9244773983955383,
      "logits/rejected": -0.9188146591186523,
      "logps/chosen": -3.638792037963867,
      "logps/rejected": -3.9765026569366455,
      "loss": 3.4023,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -36.38792037963867,
      "rewards/margins": 3.3771071434020996,
      "rewards/rejected": -39.76502990722656,
      "step": 192
    },
    {
      "epoch": 0.43652813118461975,
      "grad_norm": 111.88606555816419,
      "learning_rate": 5.55628520825718e-07,
      "logits/chosen": -0.9915658235549927,
      "logits/rejected": -0.9997342824935913,
      "logps/chosen": -3.658958911895752,
      "logps/rejected": -4.055312156677246,
      "loss": 3.3482,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -36.5895881652832,
      "rewards/margins": 3.963534355163574,
      "rewards/rejected": -40.553123474121094,
      "step": 193
    },
    {
      "epoch": 0.43878993497314106,
      "grad_norm": 115.18763957837935,
      "learning_rate": 5.527077498623752e-07,
      "logits/chosen": -0.9457052946090698,
      "logits/rejected": -0.9517892599105835,
      "logps/chosen": -3.640913963317871,
      "logps/rejected": -4.082710266113281,
      "loss": 2.9719,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -36.409141540527344,
      "rewards/margins": 4.417965412139893,
      "rewards/rejected": -40.82710266113281,
      "step": 194
    },
    {
      "epoch": 0.4410517387616624,
      "grad_norm": 102.71937253011347,
      "learning_rate": 5.497774162648228e-07,
      "logits/chosen": -0.9235740303993225,
      "logits/rejected": -0.9570806622505188,
      "logps/chosen": -3.5843145847320557,
      "logps/rejected": -4.175308704376221,
      "loss": 2.7626,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -35.84314727783203,
      "rewards/margins": 5.909941673278809,
      "rewards/rejected": -41.753082275390625,
      "step": 195
    },
    {
      "epoch": 0.4433135425501838,
      "grad_norm": 123.78104117915449,
      "learning_rate": 5.468377035319882e-07,
      "logits/chosen": -0.9676077365875244,
      "logits/rejected": -0.9691037535667419,
      "logps/chosen": -3.5839638710021973,
      "logps/rejected": -4.107885360717773,
      "loss": 3.0469,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -35.83964157104492,
      "rewards/margins": 5.239212989807129,
      "rewards/rejected": -41.078853607177734,
      "step": 196
    },
    {
      "epoch": 0.4455753463387051,
      "grad_norm": 121.58965966459272,
      "learning_rate": 5.438887957501248e-07,
      "logits/chosen": -0.8684402108192444,
      "logits/rejected": -0.8744467496871948,
      "logps/chosen": -3.645962715148926,
      "logps/rejected": -4.094875335693359,
      "loss": 3.1804,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -36.45962905883789,
      "rewards/margins": 4.489128112792969,
      "rewards/rejected": -40.948753356933594,
      "step": 197
    },
    {
      "epoch": 0.44783715012722647,
      "grad_norm": 109.22873019040438,
      "learning_rate": 5.409308775812844e-07,
      "logits/chosen": -0.9160696864128113,
      "logits/rejected": -0.9228438138961792,
      "logps/chosen": -3.8386149406433105,
      "logps/rejected": -4.307831287384033,
      "loss": 3.3267,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -38.38615036010742,
      "rewards/margins": 4.692164421081543,
      "rewards/rejected": -43.078311920166016,
      "step": 198
    },
    {
      "epoch": 0.45009895391574783,
      "grad_norm": 104.87483635159766,
      "learning_rate": 5.379641342517541e-07,
      "logits/chosen": -0.9663090705871582,
      "logits/rejected": -0.9763889908790588,
      "logps/chosen": -3.617103099822998,
      "logps/rejected": -4.147641658782959,
      "loss": 2.9717,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -36.17103576660156,
      "rewards/margins": 5.305381774902344,
      "rewards/rejected": -41.47641372680664,
      "step": 199
    },
    {
      "epoch": 0.45236075770426915,
      "grad_norm": 103.08645689813231,
      "learning_rate": 5.349887515404564e-07,
      "logits/chosen": -0.9189653992652893,
      "logits/rejected": -0.9432557225227356,
      "logps/chosen": -3.866921901702881,
      "logps/rejected": -4.4801025390625,
      "loss": 2.6106,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -38.669219970703125,
      "rewards/margins": 6.131805419921875,
      "rewards/rejected": -44.801025390625,
      "step": 200
    },
    {
      "epoch": 0.4546225614927905,
      "grad_norm": 116.35515985998616,
      "learning_rate": 5.320049157673163e-07,
      "logits/chosen": -0.8698301911354065,
      "logits/rejected": -0.8828994631767273,
      "logps/chosen": -3.784053325653076,
      "logps/rejected": -4.32690954208374,
      "loss": 2.8006,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -37.84053421020508,
      "rewards/margins": 5.428560256958008,
      "rewards/rejected": -43.26909637451172,
      "step": 201
    },
    {
      "epoch": 0.4568843652813118,
      "grad_norm": 109.47698928643487,
      "learning_rate": 5.290128137815938e-07,
      "logits/chosen": -0.9175432920455933,
      "logits/rejected": -0.9369080662727356,
      "logps/chosen": -4.0667266845703125,
      "logps/rejected": -4.628395080566406,
      "loss": 2.6128,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -40.66727066040039,
      "rewards/margins": 5.616676330566406,
      "rewards/rejected": -46.28395080566406,
      "step": 202
    },
    {
      "epoch": 0.4591461690698332,
      "grad_norm": 111.02996873736168,
      "learning_rate": 5.260126329501828e-07,
      "logits/chosen": -0.939166784286499,
      "logits/rejected": -0.9610799551010132,
      "logps/chosen": -3.9119553565979004,
      "logps/rejected": -4.589254379272461,
      "loss": 2.4693,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -39.11955261230469,
      "rewards/margins": 6.772988796234131,
      "rewards/rejected": -45.89254379272461,
      "step": 203
    },
    {
      "epoch": 0.46140797285835455,
      "grad_norm": 146.1981574496801,
      "learning_rate": 5.230045611458789e-07,
      "logits/chosen": -0.875460147857666,
      "logits/rejected": -0.9028068780899048,
      "logps/chosen": -3.9016761779785156,
      "logps/rejected": -4.389087677001953,
      "loss": 2.9617,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -39.016761779785156,
      "rewards/margins": 4.874115943908691,
      "rewards/rejected": -43.89087677001953,
      "step": 204
    },
    {
      "epoch": 0.46366977664687586,
      "grad_norm": 121.56241675915379,
      "learning_rate": 5.199887867356143e-07,
      "logits/chosen": -0.8924857378005981,
      "logits/rejected": -0.9020816087722778,
      "logps/chosen": -4.1306023597717285,
      "logps/rejected": -4.79485559463501,
      "loss": 2.5375,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -41.306026458740234,
      "rewards/margins": 6.642524242401123,
      "rewards/rejected": -47.94855499267578,
      "step": 205
    },
    {
      "epoch": 0.46593158043539723,
      "grad_norm": 122.508625760498,
      "learning_rate": 5.16965498568662e-07,
      "logits/chosen": -0.9368366599082947,
      "logits/rejected": -0.9245457053184509,
      "logps/chosen": -4.264636516571045,
      "logps/rejected": -5.027202129364014,
      "loss": 2.7563,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -42.646366119384766,
      "rewards/margins": 7.6256561279296875,
      "rewards/rejected": -50.27201843261719,
      "step": 206
    },
    {
      "epoch": 0.4681933842239186,
      "grad_norm": 131.11773288045427,
      "learning_rate": 5.139348859648098e-07,
      "logits/chosen": -0.9146727919578552,
      "logits/rejected": -0.9255930185317993,
      "logps/chosen": -3.9246838092803955,
      "logps/rejected": -4.490647315979004,
      "loss": 2.9053,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -39.2468376159668,
      "rewards/margins": 5.659629821777344,
      "rewards/rejected": -44.90647506713867,
      "step": 207
    },
    {
      "epoch": 0.4704551880124399,
      "grad_norm": 120.950641593037,
      "learning_rate": 5.10897138702506e-07,
      "logits/chosen": -0.8812559843063354,
      "logits/rejected": -0.8994000554084778,
      "logps/chosen": -4.020036220550537,
      "logps/rejected": -4.57065486907959,
      "loss": 3.3148,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -40.20035934448242,
      "rewards/margins": 5.506185531616211,
      "rewards/rejected": -45.70654296875,
      "step": 208
    },
    {
      "epoch": 0.4727169918009613,
      "grad_norm": 119.1724296691641,
      "learning_rate": 5.078524470069743e-07,
      "logits/chosen": -0.9579637050628662,
      "logits/rejected": -0.9650572538375854,
      "logps/chosen": -4.186748027801514,
      "logps/rejected": -4.810725688934326,
      "loss": 2.3755,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -41.86748123168945,
      "rewards/margins": 6.239780426025391,
      "rewards/rejected": -48.107261657714844,
      "step": 209
    },
    {
      "epoch": 0.47497879558948264,
      "grad_norm": 116.29036710628272,
      "learning_rate": 5.048010015383021e-07,
      "logits/chosen": -0.8580180406570435,
      "logits/rejected": -0.8542965650558472,
      "logps/chosen": -4.219040870666504,
      "logps/rejected": -4.9554338455200195,
      "loss": 2.3901,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -42.190399169921875,
      "rewards/margins": 7.363934516906738,
      "rewards/rejected": -49.55433654785156,
      "step": 210
    },
    {
      "epoch": 0.47724059937800395,
      "grad_norm": 121.1251025837884,
      "learning_rate": 5.01742993379502e-07,
      "logits/chosen": -0.912419319152832,
      "logits/rejected": -0.9371484518051147,
      "logps/chosen": -4.289198398590088,
      "logps/rejected": -4.923559188842773,
      "loss": 2.4793,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -42.89198684692383,
      "rewards/margins": 6.343609809875488,
      "rewards/rejected": -49.235595703125,
      "step": 211
    },
    {
      "epoch": 0.4795024031665253,
      "grad_norm": 143.77581690319334,
      "learning_rate": 4.986786140245446e-07,
      "logits/chosen": -0.8684762716293335,
      "logits/rejected": -0.8781230449676514,
      "logps/chosen": -4.23728084564209,
      "logps/rejected": -4.795536041259766,
      "loss": 3.0145,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -42.37281036376953,
      "rewards/margins": 5.582554340362549,
      "rewards/rejected": -47.95536422729492,
      "step": 212
    },
    {
      "epoch": 0.4817642069550466,
      "grad_norm": 128.57500022332647,
      "learning_rate": 4.956080553663687e-07,
      "logits/chosen": -0.9273526072502136,
      "logits/rejected": -0.9341423511505127,
      "logps/chosen": -4.209490776062012,
      "logps/rejected": -4.829947471618652,
      "loss": 2.8036,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -42.09490203857422,
      "rewards/margins": 6.204565048217773,
      "rewards/rejected": -48.299468994140625,
      "step": 213
    },
    {
      "epoch": 0.484026010743568,
      "grad_norm": 134.74973546376373,
      "learning_rate": 4.925315096848636e-07,
      "logits/chosen": -0.9303544759750366,
      "logits/rejected": -0.9426358938217163,
      "logps/chosen": -4.48342752456665,
      "logps/rejected": -5.268622875213623,
      "loss": 2.5133,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -44.83427047729492,
      "rewards/margins": 7.851953983306885,
      "rewards/rejected": -52.68622589111328,
      "step": 214
    },
    {
      "epoch": 0.48628781453208936,
      "grad_norm": 137.87557167889346,
      "learning_rate": 4.894491696348293e-07,
      "logits/chosen": -0.933896541595459,
      "logits/rejected": -0.9386736154556274,
      "logps/chosen": -4.23392915725708,
      "logps/rejected": -4.687985420227051,
      "loss": 3.0521,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -42.33929443359375,
      "rewards/margins": 4.5405592918396,
      "rewards/rejected": -46.87985610961914,
      "step": 215
    },
    {
      "epoch": 0.48854961832061067,
      "grad_norm": 139.11956890881316,
      "learning_rate": 4.863612282339116e-07,
      "logits/chosen": -0.8552120923995972,
      "logits/rejected": -0.8585663437843323,
      "logps/chosen": -4.637057781219482,
      "logps/rejected": -5.207608222961426,
      "loss": 3.0894,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -46.37057113647461,
      "rewards/margins": 5.705511093139648,
      "rewards/rejected": -52.07608413696289,
      "step": 216
    },
    {
      "epoch": 0.49081142210913203,
      "grad_norm": 144.47535678356647,
      "learning_rate": 4.832678788505161e-07,
      "logits/chosen": -0.8959632515907288,
      "logits/rejected": -0.8897783160209656,
      "logps/chosen": -4.5698347091674805,
      "logps/rejected": -5.178661346435547,
      "loss": 3.2773,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -45.69834518432617,
      "rewards/margins": 6.0882649421691895,
      "rewards/rejected": -51.7866096496582,
      "step": 217
    },
    {
      "epoch": 0.4930732258976534,
      "grad_norm": 183.85171049659183,
      "learning_rate": 4.801693151916985e-07,
      "logits/chosen": -0.9000855088233948,
      "logits/rejected": -0.9295123219490051,
      "logps/chosen": -4.563023567199707,
      "logps/rejected": -5.171204566955566,
      "loss": 2.7277,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -45.63024139404297,
      "rewards/margins": 6.08180570602417,
      "rewards/rejected": -51.71204376220703,
      "step": 218
    },
    {
      "epoch": 0.4953350296861747,
      "grad_norm": 144.8137273177042,
      "learning_rate": 4.770657312910354e-07,
      "logits/chosen": -0.9353141784667969,
      "logits/rejected": -0.938468337059021,
      "logps/chosen": -4.625380039215088,
      "logps/rejected": -5.1412482261657715,
      "loss": 3.2197,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -46.25379943847656,
      "rewards/margins": 5.158687114715576,
      "rewards/rejected": -51.41248321533203,
      "step": 219
    },
    {
      "epoch": 0.4975968334746961,
      "grad_norm": 163.41972624587643,
      "learning_rate": 4.739573214964729e-07,
      "logits/chosen": -0.8990980386734009,
      "logits/rejected": -0.9109318852424622,
      "logps/chosen": -4.455334663391113,
      "logps/rejected": -5.060534954071045,
      "loss": 2.939,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -44.5533447265625,
      "rewards/margins": 6.052001953125,
      "rewards/rejected": -50.60535430908203,
      "step": 220
    },
    {
      "epoch": 0.49985863726321744,
      "grad_norm": 170.1558238963429,
      "learning_rate": 4.7084428045815733e-07,
      "logits/chosen": -0.8906704187393188,
      "logits/rejected": -0.9073330163955688,
      "logps/chosen": -4.730157375335693,
      "logps/rejected": -5.305697441101074,
      "loss": 3.0027,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -47.301578521728516,
      "rewards/margins": 5.755396842956543,
      "rewards/rejected": -53.05697250366211,
      "step": 221
    },
    {
      "epoch": 0.5021204410517388,
      "grad_norm": 226.11746442069574,
      "learning_rate": 4.677268031162457e-07,
      "logits/chosen": -0.9023485779762268,
      "logits/rejected": -0.9136303067207336,
      "logps/chosen": -4.3702263832092285,
      "logps/rejected": -4.995909690856934,
      "loss": 2.9436,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -43.70226287841797,
      "rewards/margins": 6.2568359375,
      "rewards/rejected": -49.9590950012207,
      "step": 222
    },
    {
      "epoch": 0.5043822448402601,
      "grad_norm": 132.7984703986409,
      "learning_rate": 4.646050846886985e-07,
      "logits/chosen": -0.814538300037384,
      "logits/rejected": -0.8217288255691528,
      "logps/chosen": -4.306761264801025,
      "logps/rejected": -4.975722312927246,
      "loss": 2.4596,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -43.06761169433594,
      "rewards/margins": 6.689613342285156,
      "rewards/rejected": -49.75722885131836,
      "step": 223
    },
    {
      "epoch": 0.5066440486287814,
      "grad_norm": 147.9751731515365,
      "learning_rate": 4.6147932065905494e-07,
      "logits/chosen": -0.8536533117294312,
      "logits/rejected": -0.8585684299468994,
      "logps/chosen": -4.433886528015137,
      "logps/rejected": -4.914949893951416,
      "loss": 3.3481,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -44.3388671875,
      "rewards/margins": 4.810628890991211,
      "rewards/rejected": -49.14949417114258,
      "step": 224
    },
    {
      "epoch": 0.5089058524173028,
      "grad_norm": 149.36332124550927,
      "learning_rate": 4.5834970676419214e-07,
      "logits/chosen": -0.867747962474823,
      "logits/rejected": -0.8860857486724854,
      "logps/chosen": -4.323178768157959,
      "logps/rejected": -4.883927822113037,
      "loss": 2.8019,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -43.23178482055664,
      "rewards/margins": 5.607486248016357,
      "rewards/rejected": -48.83927917480469,
      "step": 225
    },
    {
      "epoch": 0.5111676562058242,
      "grad_norm": 163.9248977367859,
      "learning_rate": 4.552164389820673e-07,
      "logits/chosen": -0.7673951983451843,
      "logits/rejected": -0.7906842231750488,
      "logps/chosen": -4.370258331298828,
      "logps/rejected": -5.137817859649658,
      "loss": 2.539,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -43.70258331298828,
      "rewards/margins": 7.675594329833984,
      "rewards/rejected": -51.378177642822266,
      "step": 226
    },
    {
      "epoch": 0.5134294599943455,
      "grad_norm": 160.95245413332196,
      "learning_rate": 4.5207971351944605e-07,
      "logits/chosen": -0.8810624480247498,
      "logits/rejected": -0.8833796977996826,
      "logps/chosen": -4.283847808837891,
      "logps/rejected": -4.946835994720459,
      "loss": 3.3887,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -42.838478088378906,
      "rewards/margins": 6.629883289337158,
      "rewards/rejected": -49.46835708618164,
      "step": 227
    },
    {
      "epoch": 0.5156912637828668,
      "grad_norm": 150.5220812123222,
      "learning_rate": 4.489397267996157e-07,
      "logits/chosen": -0.8820953965187073,
      "logits/rejected": -0.8898295164108276,
      "logps/chosen": -4.304379463195801,
      "logps/rejected": -4.843184471130371,
      "loss": 2.96,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -43.04379653930664,
      "rewards/margins": 5.388051509857178,
      "rewards/rejected": -48.431846618652344,
      "step": 228
    },
    {
      "epoch": 0.5179530675713881,
      "grad_norm": 141.43842633471348,
      "learning_rate": 4.45796675450085e-07,
      "logits/chosen": -0.8514925241470337,
      "logits/rejected": -0.8414303064346313,
      "logps/chosen": -4.168148040771484,
      "logps/rejected": -4.858559608459473,
      "loss": 2.6745,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -41.68148422241211,
      "rewards/margins": 6.904117107391357,
      "rewards/rejected": -48.585601806640625,
      "step": 229
    },
    {
      "epoch": 0.5202148713599095,
      "grad_norm": 168.87995319696404,
      "learning_rate": 4.4265075629027126e-07,
      "logits/chosen": -0.7781671285629272,
      "logits/rejected": -0.7910118103027344,
      "logps/chosen": -4.419309139251709,
      "logps/rejected": -4.9767374992370605,
      "loss": 2.7029,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -44.193092346191406,
      "rewards/margins": 5.57428503036499,
      "rewards/rejected": -49.767372131347656,
      "step": 230
    },
    {
      "epoch": 0.5224766751484309,
      "grad_norm": 131.99524587680546,
      "learning_rate": 4.3950216631917563e-07,
      "logits/chosen": -0.8382211327552795,
      "logits/rejected": -0.8589369058609009,
      "logps/chosen": -4.271603584289551,
      "logps/rejected": -4.960786819458008,
      "loss": 2.6628,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -42.71603012084961,
      "rewards/margins": 6.891840934753418,
      "rewards/rejected": -49.607872009277344,
      "step": 231
    },
    {
      "epoch": 0.5247384789369522,
      "grad_norm": 134.3361120222247,
      "learning_rate": 4.3635110270304676e-07,
      "logits/chosen": -0.8264502286911011,
      "logits/rejected": -0.8120173811912537,
      "logps/chosen": -4.067740440368652,
      "logps/rejected": -4.706015586853027,
      "loss": 2.148,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -40.677398681640625,
      "rewards/margins": 6.382755279541016,
      "rewards/rejected": -47.06015396118164,
      "step": 232
    },
    {
      "epoch": 0.5270002827254736,
      "grad_norm": 152.61581673294296,
      "learning_rate": 4.331977627630339e-07,
      "logits/chosen": -0.7725579738616943,
      "logits/rejected": -0.760904848575592,
      "logps/chosen": -4.003568172454834,
      "logps/rejected": -4.687016487121582,
      "loss": 2.7551,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -40.03567886352539,
      "rewards/margins": 6.8344855308532715,
      "rewards/rejected": -46.87016296386719,
      "step": 233
    },
    {
      "epoch": 0.5292620865139949,
      "grad_norm": 147.10762801151967,
      "learning_rate": 4.300423439628313e-07,
      "logits/chosen": -0.7840836048126221,
      "logits/rejected": -0.8098961114883423,
      "logps/chosen": -4.061254024505615,
      "logps/rejected": -4.711936950683594,
      "loss": 2.5606,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -40.61254119873047,
      "rewards/margins": 6.506833076477051,
      "rewards/rejected": -47.11936950683594,
      "step": 234
    },
    {
      "epoch": 0.5315238903025162,
      "grad_norm": 132.90712688474918,
      "learning_rate": 4.268850438963118e-07,
      "logits/chosen": -0.817150354385376,
      "logits/rejected": -0.8263847827911377,
      "logps/chosen": -4.237096309661865,
      "logps/rejected": -4.880129814147949,
      "loss": 2.584,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -42.370967864990234,
      "rewards/margins": 6.430337429046631,
      "rewards/rejected": -48.80130386352539,
      "step": 235
    },
    {
      "epoch": 0.5337856940910376,
      "grad_norm": 138.4880606332674,
      "learning_rate": 4.2372606027515463e-07,
      "logits/chosen": -0.7787373661994934,
      "logits/rejected": -0.7952776551246643,
      "logps/chosen": -3.890188694000244,
      "logps/rejected": -4.451065540313721,
      "loss": 2.9659,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -38.90188980102539,
      "rewards/margins": 5.60876989364624,
      "rewards/rejected": -44.51065444946289,
      "step": 236
    },
    {
      "epoch": 0.536047497879559,
      "grad_norm": 144.57651076503535,
      "learning_rate": 4.2056559091646387e-07,
      "logits/chosen": -0.7900989055633545,
      "logits/rejected": -0.8283600211143494,
      "logps/chosen": -4.038181781768799,
      "logps/rejected": -4.595399856567383,
      "loss": 2.9693,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -40.38182067871094,
      "rewards/margins": 5.572176933288574,
      "rewards/rejected": -45.95399475097656,
      "step": 237
    },
    {
      "epoch": 0.5383093016680803,
      "grad_norm": 139.26003729586844,
      "learning_rate": 4.1740383373038116e-07,
      "logits/chosen": -0.7787960171699524,
      "logits/rejected": -0.7999230027198792,
      "logps/chosen": -3.868332862854004,
      "logps/rejected": -4.443919658660889,
      "loss": 2.8123,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -38.683326721191406,
      "rewards/margins": 5.75586462020874,
      "rewards/rejected": -44.43919372558594,
      "step": 238
    },
    {
      "epoch": 0.5405711054566016,
      "grad_norm": 142.51994348526011,
      "learning_rate": 4.1424098670769255e-07,
      "logits/chosen": -0.8288208246231079,
      "logits/rejected": -0.8396254777908325,
      "logps/chosen": -4.00309419631958,
      "logps/rejected": -4.4558329582214355,
      "loss": 3.2117,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -40.030941009521484,
      "rewards/margins": 4.5273871421813965,
      "rewards/rejected": -44.55833053588867,
      "step": 239
    },
    {
      "epoch": 0.542832909245123,
      "grad_norm": 137.83906726980342,
      "learning_rate": 4.1107724790743007e-07,
      "logits/chosen": -0.7743911147117615,
      "logits/rejected": -0.8078039884567261,
      "logps/chosen": -3.917076587677002,
      "logps/rejected": -4.433419227600098,
      "loss": 2.8325,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -39.17076110839844,
      "rewards/margins": 5.163426399230957,
      "rewards/rejected": -44.334190368652344,
      "step": 240
    },
    {
      "epoch": 0.5450947130336443,
      "grad_norm": 139.8500358924521,
      "learning_rate": 4.0791281544446947e-07,
      "logits/chosen": -0.8219509124755859,
      "logits/rejected": -0.8224381804466248,
      "logps/chosen": -3.893245220184326,
      "logps/rejected": -4.521663665771484,
      "loss": 2.3828,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -38.932456970214844,
      "rewards/margins": 6.284185409545898,
      "rewards/rejected": -45.21664047241211,
      "step": 241
    },
    {
      "epoch": 0.5473565168221657,
      "grad_norm": 126.08857645654825,
      "learning_rate": 4.0474788747712416e-07,
      "logits/chosen": -0.8421152830123901,
      "logits/rejected": -0.8481616973876953,
      "logps/chosen": -3.897510051727295,
      "logps/rejected": -4.381252765655518,
      "loss": 3.3288,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -38.975101470947266,
      "rewards/margins": 4.837425231933594,
      "rewards/rejected": -43.812522888183594,
      "step": 242
    },
    {
      "epoch": 0.549618320610687,
      "grad_norm": 115.43735691500179,
      "learning_rate": 4.0158266219473573e-07,
      "logits/chosen": -0.8440154194831848,
      "logits/rejected": -0.8574568033218384,
      "logps/chosen": -3.7257094383239746,
      "logps/rejected": -4.348711013793945,
      "loss": 2.4764,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -37.25709533691406,
      "rewards/margins": 6.2300190925598145,
      "rewards/rejected": -43.48711013793945,
      "step": 243
    },
    {
      "epoch": 0.5518801243992084,
      "grad_norm": 116.33902882228996,
      "learning_rate": 3.984173378052643e-07,
      "logits/chosen": -0.7851986885070801,
      "logits/rejected": -0.796712338924408,
      "logps/chosen": -3.6892523765563965,
      "logps/rejected": -4.342462539672852,
      "loss": 2.6619,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -36.89252471923828,
      "rewards/margins": 6.532103538513184,
      "rewards/rejected": -43.42462921142578,
      "step": 244
    },
    {
      "epoch": 0.5541419281877297,
      "grad_norm": 124.64167796525535,
      "learning_rate": 3.9525211252287585e-07,
      "logits/chosen": -0.8931095600128174,
      "logits/rejected": -0.9180840849876404,
      "logps/chosen": -3.963862419128418,
      "logps/rejected": -4.748945236206055,
      "loss": 2.5828,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -39.63862609863281,
      "rewards/margins": 7.850823879241943,
      "rewards/rejected": -47.48944854736328,
      "step": 245
    },
    {
      "epoch": 0.556403731976251,
      "grad_norm": 112.09055809017627,
      "learning_rate": 3.920871845555305e-07,
      "logits/chosen": -0.8313508629798889,
      "logits/rejected": -0.8461873531341553,
      "logps/chosen": -3.870025157928467,
      "logps/rejected": -4.447489261627197,
      "loss": 2.385,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -38.700252532958984,
      "rewards/margins": 5.774643898010254,
      "rewards/rejected": -44.47489547729492,
      "step": 246
    },
    {
      "epoch": 0.5586655357647724,
      "grad_norm": 139.45655157059815,
      "learning_rate": 3.8892275209256984e-07,
      "logits/chosen": -0.8623577356338501,
      "logits/rejected": -0.8599537014961243,
      "logps/chosen": -4.079935073852539,
      "logps/rejected": -4.5872697830200195,
      "loss": 2.7693,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -40.79934310913086,
      "rewards/margins": 5.073347568511963,
      "rewards/rejected": -45.87269592285156,
      "step": 247
    },
    {
      "epoch": 0.5609273395532938,
      "grad_norm": 128.5379031156273,
      "learning_rate": 3.8575901329230747e-07,
      "logits/chosen": -0.831263542175293,
      "logits/rejected": -0.8186984062194824,
      "logps/chosen": -4.078851222991943,
      "logps/rejected": -4.684617519378662,
      "loss": 2.9864,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -40.788516998291016,
      "rewards/margins": 6.057657241821289,
      "rewards/rejected": -46.84617614746094,
      "step": 248
    },
    {
      "epoch": 0.5631891433418151,
      "grad_norm": 137.16938367411643,
      "learning_rate": 3.8259616626961886e-07,
      "logits/chosen": -0.8066788911819458,
      "logits/rejected": -0.8286083936691284,
      "logps/chosen": -3.8074519634246826,
      "logps/rejected": -4.328085422515869,
      "loss": 2.4375,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -38.074520111083984,
      "rewards/margins": 5.206332206726074,
      "rewards/rejected": -43.280853271484375,
      "step": 249
    },
    {
      "epoch": 0.5654509471303364,
      "grad_norm": 114.38221061442538,
      "learning_rate": 3.794344090835362e-07,
      "logits/chosen": -0.836406946182251,
      "logits/rejected": -0.8421981334686279,
      "logps/chosen": -4.185178279876709,
      "logps/rejected": -4.7932024002075195,
      "loss": 2.8436,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -41.85177993774414,
      "rewards/margins": 6.080243110656738,
      "rewards/rejected": -47.93202590942383,
      "step": 250
    },
    {
      "epoch": 0.5677127509188578,
      "grad_norm": 173.94050832794733,
      "learning_rate": 3.7627393972484534e-07,
      "logits/chosen": -0.8843735456466675,
      "logits/rejected": -0.8974891901016235,
      "logps/chosen": -4.174497127532959,
      "logps/rejected": -4.667062282562256,
      "loss": 3.303,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": -41.74496841430664,
      "rewards/margins": 4.92565393447876,
      "rewards/rejected": -46.67062759399414,
      "step": 251
    },
    {
      "epoch": 0.5699745547073791,
      "grad_norm": 139.59908745062046,
      "learning_rate": 3.7311495610368823e-07,
      "logits/chosen": -0.8997682929039001,
      "logits/rejected": -0.9016697406768799,
      "logps/chosen": -4.343634128570557,
      "logps/rejected": -4.967972755432129,
      "loss": 2.7301,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -43.43634033203125,
      "rewards/margins": 6.243386268615723,
      "rewards/rejected": -49.679725646972656,
      "step": 252
    },
    {
      "epoch": 0.5722363584959005,
      "grad_norm": 120.92407808146834,
      "learning_rate": 3.699576560371689e-07,
      "logits/chosen": -0.8279405236244202,
      "logits/rejected": -0.8453271389007568,
      "logps/chosen": -4.558162689208984,
      "logps/rejected": -5.47041654586792,
      "loss": 2.1306,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -45.581626892089844,
      "rewards/margins": 9.122541427612305,
      "rewards/rejected": -54.70416259765625,
      "step": 253
    },
    {
      "epoch": 0.5744981622844219,
      "grad_norm": 144.72597313458348,
      "learning_rate": 3.66802237236966e-07,
      "logits/chosen": -0.8229357600212097,
      "logits/rejected": -0.8324556946754456,
      "logps/chosen": -4.652154445648193,
      "logps/rejected": -5.330699920654297,
      "loss": 2.4779,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -46.521549224853516,
      "rewards/margins": 6.78544807434082,
      "rewards/rejected": -53.30699920654297,
      "step": 254
    },
    {
      "epoch": 0.5767599660729432,
      "grad_norm": 138.6610214924037,
      "learning_rate": 3.636488972969532e-07,
      "logits/chosen": -0.8425090312957764,
      "logits/rejected": -0.8567550182342529,
      "logps/chosen": -4.528759479522705,
      "logps/rejected": -5.228953838348389,
      "loss": 2.7067,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -45.287593841552734,
      "rewards/margins": 7.001940727233887,
      "rewards/rejected": -52.2895393371582,
      "step": 255
    },
    {
      "epoch": 0.5790217698614645,
      "grad_norm": 149.23473531486215,
      "learning_rate": 3.604978336808244e-07,
      "logits/chosen": -0.9809710383415222,
      "logits/rejected": -0.9890305399894714,
      "logps/chosen": -4.413546085357666,
      "logps/rejected": -5.061186790466309,
      "loss": 2.7346,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -44.13545608520508,
      "rewards/margins": 6.476408958435059,
      "rewards/rejected": -50.61186981201172,
      "step": 256
    },
    {
      "epoch": 0.5812835736499858,
      "grad_norm": 151.69021123299453,
      "learning_rate": 3.5734924370972876e-07,
      "logits/chosen": -0.9168260097503662,
      "logits/rejected": -0.9385607242584229,
      "logps/chosen": -4.60618257522583,
      "logps/rejected": -5.185503959655762,
      "loss": 3.0486,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -46.06182861328125,
      "rewards/margins": 5.793212890625,
      "rewards/rejected": -51.855037689208984,
      "step": 257
    },
    {
      "epoch": 0.5835453774385072,
      "grad_norm": 168.20501802658688,
      "learning_rate": 3.5420332454991504e-07,
      "logits/chosen": -0.8639505505561829,
      "logits/rejected": -0.8759117126464844,
      "logps/chosen": -4.676448822021484,
      "logps/rejected": -5.332930564880371,
      "loss": 3.0441,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -46.76448440551758,
      "rewards/margins": 6.564816951751709,
      "rewards/rejected": -53.32930374145508,
      "step": 258
    },
    {
      "epoch": 0.5858071812270286,
      "grad_norm": 133.2402904508335,
      "learning_rate": 3.510602732003843e-07,
      "logits/chosen": -0.9353285431861877,
      "logits/rejected": -0.9522349834442139,
      "logps/chosen": -4.753300666809082,
      "logps/rejected": -5.492231369018555,
      "loss": 2.6279,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -47.53300476074219,
      "rewards/margins": 7.389303684234619,
      "rewards/rejected": -54.92231369018555,
      "step": 259
    },
    {
      "epoch": 0.5880689850155499,
      "grad_norm": 144.95585468548657,
      "learning_rate": 3.4792028648055396e-07,
      "logits/chosen": -0.9198298454284668,
      "logits/rejected": -0.9325035214424133,
      "logps/chosen": -4.450214385986328,
      "logps/rejected": -5.1093268394470215,
      "loss": 2.8402,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -44.502140045166016,
      "rewards/margins": 6.591127872467041,
      "rewards/rejected": -51.0932731628418,
      "step": 260
    },
    {
      "epoch": 0.5903307888040712,
      "grad_norm": 155.36455892363466,
      "learning_rate": 3.447835610179327e-07,
      "logits/chosen": -0.915900707244873,
      "logits/rejected": -0.9240118265151978,
      "logps/chosen": -4.484950065612793,
      "logps/rejected": -5.312657833099365,
      "loss": 2.5099,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -44.84950637817383,
      "rewards/margins": 8.277070999145508,
      "rewards/rejected": -53.12657928466797,
      "step": 261
    },
    {
      "epoch": 0.5925925925925926,
      "grad_norm": 178.5841356353186,
      "learning_rate": 3.416502932358079e-07,
      "logits/chosen": -0.9722024202346802,
      "logits/rejected": -0.9864451289176941,
      "logps/chosen": -4.724699974060059,
      "logps/rejected": -5.200839519500732,
      "loss": 3.191,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -47.24700164794922,
      "rewards/margins": 4.761399269104004,
      "rewards/rejected": -52.008399963378906,
      "step": 262
    },
    {
      "epoch": 0.5948543963811139,
      "grad_norm": 155.6491240961835,
      "learning_rate": 3.385206793409451e-07,
      "logits/chosen": -0.9002339839935303,
      "logits/rejected": -0.917601466178894,
      "logps/chosen": -4.058086395263672,
      "logps/rejected": -4.651598930358887,
      "loss": 2.657,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -40.58086395263672,
      "rewards/margins": 5.935131072998047,
      "rewards/rejected": -46.515995025634766,
      "step": 263
    },
    {
      "epoch": 0.5971162001696353,
      "grad_norm": 148.42975008344453,
      "learning_rate": 3.3539491531130163e-07,
      "logits/chosen": -0.9280572533607483,
      "logits/rejected": -0.9459517002105713,
      "logps/chosen": -4.205946922302246,
      "logps/rejected": -4.992376804351807,
      "loss": 2.4892,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -42.05946731567383,
      "rewards/margins": 7.864302635192871,
      "rewards/rejected": -49.923770904541016,
      "step": 264
    },
    {
      "epoch": 0.5993780039581567,
      "grad_norm": 140.98014037458591,
      "learning_rate": 3.3227319688375426e-07,
      "logits/chosen": -1.005509376525879,
      "logits/rejected": -0.996308445930481,
      "logps/chosen": -4.1070556640625,
      "logps/rejected": -4.702898979187012,
      "loss": 2.6008,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -41.070556640625,
      "rewards/margins": 5.958428382873535,
      "rewards/rejected": -47.02898406982422,
      "step": 265
    },
    {
      "epoch": 0.601639807746678,
      "grad_norm": 128.74217848218075,
      "learning_rate": 3.291557195418427e-07,
      "logits/chosen": -1.0023953914642334,
      "logits/rejected": -0.9917722940444946,
      "logps/chosen": -3.8998665809631348,
      "logps/rejected": -4.379611492156982,
      "loss": 3.1704,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -38.99866485595703,
      "rewards/margins": 4.797451496124268,
      "rewards/rejected": -43.796112060546875,
      "step": 266
    },
    {
      "epoch": 0.6039016115351993,
      "grad_norm": 145.05497368203444,
      "learning_rate": 3.260426785035272e-07,
      "logits/chosen": -0.9523136019706726,
      "logits/rejected": -0.9565008878707886,
      "logps/chosen": -3.9325990676879883,
      "logps/rejected": -4.465220928192139,
      "loss": 3.3689,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -39.325984954833984,
      "rewards/margins": 5.326223850250244,
      "rewards/rejected": -44.6522102355957,
      "step": 267
    },
    {
      "epoch": 0.6061634153237206,
      "grad_norm": 133.48758547591208,
      "learning_rate": 3.229342687089646e-07,
      "logits/chosen": -0.9732850193977356,
      "logits/rejected": -0.9786388278007507,
      "logps/chosen": -3.7646870613098145,
      "logps/rejected": -4.4038310050964355,
      "loss": 2.6359,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -37.646873474121094,
      "rewards/margins": 6.391435623168945,
      "rewards/rejected": -44.038307189941406,
      "step": 268
    },
    {
      "epoch": 0.608425219112242,
      "grad_norm": 156.96228832905817,
      "learning_rate": 3.1983068480830143e-07,
      "logits/chosen": -0.9513987302780151,
      "logits/rejected": -0.9469345808029175,
      "logps/chosen": -3.7457966804504395,
      "logps/rejected": -4.372827529907227,
      "loss": 2.6094,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -37.45797348022461,
      "rewards/margins": 6.270303249359131,
      "rewards/rejected": -43.728271484375,
      "step": 269
    },
    {
      "epoch": 0.6106870229007634,
      "grad_norm": 126.30153170952352,
      "learning_rate": 3.1673212114948387e-07,
      "logits/chosen": -0.9619816541671753,
      "logits/rejected": -0.9705020189285278,
      "logps/chosen": -3.638077974319458,
      "logps/rejected": -4.2800679206848145,
      "loss": 2.5696,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -36.380775451660156,
      "rewards/margins": 6.419899940490723,
      "rewards/rejected": -42.80067825317383,
      "step": 270
    },
    {
      "epoch": 0.6129488266892847,
      "grad_norm": 131.06230943967222,
      "learning_rate": 3.1363877176608845e-07,
      "logits/chosen": -0.9253988862037659,
      "logits/rejected": -0.9438823461532593,
      "logps/chosen": -3.546363592147827,
      "logps/rejected": -4.163539409637451,
      "loss": 2.721,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -35.46363830566406,
      "rewards/margins": 6.171757698059082,
      "rewards/rejected": -41.63539505004883,
      "step": 271
    },
    {
      "epoch": 0.615210630477806,
      "grad_norm": 119.36603698630678,
      "learning_rate": 3.1055083036517076e-07,
      "logits/chosen": -0.931835949420929,
      "logits/rejected": -0.9240102767944336,
      "logps/chosen": -3.519230604171753,
      "logps/rejected": -4.135827541351318,
      "loss": 2.5416,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -35.19230651855469,
      "rewards/margins": 6.165970325469971,
      "rewards/rejected": -41.3582763671875,
      "step": 272
    },
    {
      "epoch": 0.6174724342663274,
      "grad_norm": 157.21966352920222,
      "learning_rate": 3.074684903151364e-07,
      "logits/chosen": -0.8328728675842285,
      "logits/rejected": -0.8194759488105774,
      "logps/chosen": -3.2504658699035645,
      "logps/rejected": -3.7328827381134033,
      "loss": 2.4568,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -32.50465774536133,
      "rewards/margins": 4.824168682098389,
      "rewards/rejected": -37.32883071899414,
      "step": 273
    },
    {
      "epoch": 0.6197342380548487,
      "grad_norm": 125.59777868815232,
      "learning_rate": 3.0439194463363136e-07,
      "logits/chosen": -0.9033098816871643,
      "logits/rejected": -0.9113788604736328,
      "logps/chosen": -3.432678699493408,
      "logps/rejected": -3.9219419956207275,
      "loss": 3.0769,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -34.326786041259766,
      "rewards/margins": 4.892629146575928,
      "rewards/rejected": -39.219417572021484,
      "step": 274
    },
    {
      "epoch": 0.6219960418433701,
      "grad_norm": 113.53689202871962,
      "learning_rate": 3.0132138597545537e-07,
      "logits/chosen": -0.9494621753692627,
      "logits/rejected": -0.9803709983825684,
      "logps/chosen": -3.605886936187744,
      "logps/rejected": -4.303212642669678,
      "loss": 2.5638,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -36.058868408203125,
      "rewards/margins": 6.973256587982178,
      "rewards/rejected": -43.03212356567383,
      "step": 275
    },
    {
      "epoch": 0.6242578456318915,
      "grad_norm": 105.00410359380473,
      "learning_rate": 2.982570066204981e-07,
      "logits/chosen": -0.9540393948554993,
      "logits/rejected": -0.9732652306556702,
      "logps/chosen": -3.5351197719573975,
      "logps/rejected": -4.073846340179443,
      "loss": 2.8045,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -35.351200103759766,
      "rewards/margins": 5.387262344360352,
      "rewards/rejected": -40.738460540771484,
      "step": 276
    },
    {
      "epoch": 0.6265196494204128,
      "grad_norm": 154.48698784858473,
      "learning_rate": 2.951989984616979e-07,
      "logits/chosen": -0.8954455852508545,
      "logits/rejected": -0.9178181886672974,
      "logps/chosen": -3.763580799102783,
      "logps/rejected": -4.397812843322754,
      "loss": 3.2425,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -37.63581085205078,
      "rewards/margins": 6.342320919036865,
      "rewards/rejected": -43.97813034057617,
      "step": 277
    },
    {
      "epoch": 0.6287814532089341,
      "grad_norm": 124.9996609566805,
      "learning_rate": 2.9214755299302584e-07,
      "logits/chosen": -0.9186022877693176,
      "logits/rejected": -0.9265093803405762,
      "logps/chosen": -4.027862548828125,
      "logps/rejected": -4.721011638641357,
      "loss": 2.1795,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -40.278629302978516,
      "rewards/margins": 6.931490421295166,
      "rewards/rejected": -47.210121154785156,
      "step": 278
    },
    {
      "epoch": 0.6310432569974554,
      "grad_norm": 128.6732786280236,
      "learning_rate": 2.89102861297494e-07,
      "logits/chosen": -0.9596308469772339,
      "logits/rejected": -0.9828594923019409,
      "logps/chosen": -3.758955955505371,
      "logps/rejected": -4.320319175720215,
      "loss": 3.1683,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -37.589561462402344,
      "rewards/margins": 5.613630294799805,
      "rewards/rejected": -43.203189849853516,
      "step": 279
    },
    {
      "epoch": 0.6333050607859768,
      "grad_norm": 129.18886507394328,
      "learning_rate": 2.860651140351902e-07,
      "logits/chosen": -0.949332058429718,
      "logits/rejected": -0.9587225914001465,
      "logps/chosen": -3.851499557495117,
      "logps/rejected": -4.524806022644043,
      "loss": 2.8367,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -38.5150032043457,
      "rewards/margins": 6.733060359954834,
      "rewards/rejected": -45.2480583190918,
      "step": 280
    },
    {
      "epoch": 0.6355668645744982,
      "grad_norm": 121.51443077771003,
      "learning_rate": 2.830345014313381e-07,
      "logits/chosen": -0.882565975189209,
      "logits/rejected": -0.9311519265174866,
      "logps/chosen": -4.010669231414795,
      "logps/rejected": -4.751985549926758,
      "loss": 2.0959,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -40.106693267822266,
      "rewards/margins": 7.4131622314453125,
      "rewards/rejected": -47.51985549926758,
      "step": 281
    },
    {
      "epoch": 0.6378286683630195,
      "grad_norm": 124.30762639284417,
      "learning_rate": 2.800112132643856e-07,
      "logits/chosen": -0.9435937404632568,
      "logits/rejected": -0.9561968445777893,
      "logps/chosen": -4.011701583862305,
      "logps/rejected": -4.713750839233398,
      "loss": 2.6367,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -40.11701583862305,
      "rewards/margins": 7.020491600036621,
      "rewards/rejected": -47.13751220703125,
      "step": 282
    },
    {
      "epoch": 0.6400904721515408,
      "grad_norm": 126.70176291289812,
      "learning_rate": 2.7699543885412105e-07,
      "logits/chosen": -0.9590288400650024,
      "logits/rejected": -0.9676195383071899,
      "logps/chosen": -4.167572021484375,
      "logps/rejected": -4.936420917510986,
      "loss": 2.2448,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -41.67572021484375,
      "rewards/margins": 7.688486099243164,
      "rewards/rejected": -49.36420440673828,
      "step": 283
    },
    {
      "epoch": 0.6423522759400622,
      "grad_norm": 137.6588338492344,
      "learning_rate": 2.7398736704981725e-07,
      "logits/chosen": -0.9565585255622864,
      "logits/rejected": -0.9600222110748291,
      "logps/chosen": -4.261449337005615,
      "logps/rejected": -4.877924919128418,
      "loss": 2.4145,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -42.61449432373047,
      "rewards/margins": 6.164754867553711,
      "rewards/rejected": -48.77925109863281,
      "step": 284
    },
    {
      "epoch": 0.6446140797285835,
      "grad_norm": 142.52301773899467,
      "learning_rate": 2.709871862184063e-07,
      "logits/chosen": -0.9420406818389893,
      "logits/rejected": -0.9535735249519348,
      "logps/chosen": -4.204644680023193,
      "logps/rejected": -4.8350419998168945,
      "loss": 2.8569,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -42.046443939208984,
      "rewards/margins": 6.303977012634277,
      "rewards/rejected": -48.35041809082031,
      "step": 285
    },
    {
      "epoch": 0.6468758835171049,
      "grad_norm": 116.83614175305792,
      "learning_rate": 2.679950842326837e-07,
      "logits/chosen": -0.9828240275382996,
      "logits/rejected": -0.9906516671180725,
      "logps/chosen": -4.424816608428955,
      "logps/rejected": -5.189925193786621,
      "loss": 2.3291,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -44.248165130615234,
      "rewards/margins": 7.651082515716553,
      "rewards/rejected": -51.89924621582031,
      "step": 286
    },
    {
      "epoch": 0.6491376873056263,
      "grad_norm": 119.1863932565742,
      "learning_rate": 2.6501124845954363e-07,
      "logits/chosen": -0.9583408832550049,
      "logits/rejected": -0.9792217016220093,
      "logps/chosen": -4.437569618225098,
      "logps/rejected": -5.259509086608887,
      "loss": 2.0703,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -44.375701904296875,
      "rewards/margins": 8.219388961791992,
      "rewards/rejected": -52.595088958740234,
      "step": 287
    },
    {
      "epoch": 0.6513994910941476,
      "grad_norm": 135.34347421839638,
      "learning_rate": 2.62035865748246e-07,
      "logits/chosen": -0.942946195602417,
      "logits/rejected": -0.9585049748420715,
      "logps/chosen": -4.206583023071289,
      "logps/rejected": -4.9610395431518555,
      "loss": 2.3861,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -42.065826416015625,
      "rewards/margins": 7.544568061828613,
      "rewards/rejected": -49.61039733886719,
      "step": 288
    },
    {
      "epoch": 0.6536612948826689,
      "grad_norm": 135.7054533746377,
      "learning_rate": 2.5906912241871554e-07,
      "logits/chosen": -1.0347181558609009,
      "logits/rejected": -1.0334982872009277,
      "logps/chosen": -4.558216094970703,
      "logps/rejected": -5.3071675300598145,
      "loss": 2.4851,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -45.5821647644043,
      "rewards/margins": 7.489511013031006,
      "rewards/rejected": -53.07167434692383,
      "step": 289
    },
    {
      "epoch": 0.6559230986711903,
      "grad_norm": 146.88601624122214,
      "learning_rate": 2.561112042498753e-07,
      "logits/chosen": -0.898168683052063,
      "logits/rejected": -0.9251189231872559,
      "logps/chosen": -4.299272537231445,
      "logps/rejected": -4.885739326477051,
      "loss": 3.2414,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -42.99272155761719,
      "rewards/margins": 5.864667892456055,
      "rewards/rejected": -48.857391357421875,
      "step": 290
    },
    {
      "epoch": 0.6581849024597116,
      "grad_norm": 149.6582755236179,
      "learning_rate": 2.5316229646801195e-07,
      "logits/chosen": -0.9464926719665527,
      "logits/rejected": -0.9794542789459229,
      "logps/chosen": -4.87380313873291,
      "logps/rejected": -5.561539173126221,
      "loss": 2.6496,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -48.7380256652832,
      "rewards/margins": 6.877363204956055,
      "rewards/rejected": -55.61539077758789,
      "step": 291
    },
    {
      "epoch": 0.660446706248233,
      "grad_norm": 145.17127169522513,
      "learning_rate": 2.5022258373517714e-07,
      "logits/chosen": -1.0149282217025757,
      "logits/rejected": -1.0269863605499268,
      "logps/chosen": -4.728174686431885,
      "logps/rejected": -5.443430423736572,
      "loss": 2.3251,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -47.281742095947266,
      "rewards/margins": 7.152561187744141,
      "rewards/rejected": -54.43430709838867,
      "step": 292
    },
    {
      "epoch": 0.6627085100367544,
      "grad_norm": 163.61207833360584,
      "learning_rate": 2.4729225013762474e-07,
      "logits/chosen": -1.0522814989089966,
      "logits/rejected": -1.0629031658172607,
      "logps/chosen": -5.014071464538574,
      "logps/rejected": -5.679079532623291,
      "loss": 3.3606,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -50.140708923339844,
      "rewards/margins": 6.650084972381592,
      "rewards/rejected": -56.79079818725586,
      "step": 293
    },
    {
      "epoch": 0.6649703138252756,
      "grad_norm": 160.99675359530877,
      "learning_rate": 2.4437147917428203e-07,
      "logits/chosen": -0.9571595191955566,
      "logits/rejected": -0.9781152009963989,
      "logps/chosen": -4.976321220397949,
      "logps/rejected": -5.786618232727051,
      "loss": 2.5064,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -49.76321029663086,
      "rewards/margins": 8.102978706359863,
      "rewards/rejected": -57.866188049316406,
      "step": 294
    },
    {
      "epoch": 0.667232117613797,
      "grad_norm": 177.52031008687038,
      "learning_rate": 2.414604537452595e-07,
      "logits/chosen": -0.9503519535064697,
      "logits/rejected": -0.9574546813964844,
      "logps/chosen": -4.793954372406006,
      "logps/rejected": -5.45725679397583,
      "loss": 2.7098,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -47.939544677734375,
      "rewards/margins": 6.633028030395508,
      "rewards/rejected": -54.572574615478516,
      "step": 295
    },
    {
      "epoch": 0.6694939214023183,
      "grad_norm": 177.92862527487938,
      "learning_rate": 2.385593561403974e-07,
      "logits/chosen": -0.9907506704330444,
      "logits/rejected": -1.007781744003296,
      "logps/chosen": -4.650373935699463,
      "logps/rejected": -5.429416179656982,
      "loss": 2.1979,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -46.50373840332031,
      "rewards/margins": 7.790426254272461,
      "rewards/rejected": -54.29416275024414,
      "step": 296
    },
    {
      "epoch": 0.6717557251908397,
      "grad_norm": 139.27180823243322,
      "learning_rate": 2.3566836802785119e-07,
      "logits/chosen": -0.9892964363098145,
      "logits/rejected": -1.0037226676940918,
      "logps/chosen": -4.7973175048828125,
      "logps/rejected": -5.591341972351074,
      "loss": 2.467,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -47.973182678222656,
      "rewards/margins": 7.940241813659668,
      "rewards/rejected": -55.913421630859375,
      "step": 297
    },
    {
      "epoch": 0.6740175289793611,
      "grad_norm": 153.97560579826447,
      "learning_rate": 2.327876704427146e-07,
      "logits/chosen": -0.9428931474685669,
      "logits/rejected": -0.9553389549255371,
      "logps/chosen": -4.640042304992676,
      "logps/rejected": -5.214030742645264,
      "loss": 2.8868,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -46.40042495727539,
      "rewards/margins": 5.739879608154297,
      "rewards/rejected": -52.14030838012695,
      "step": 298
    },
    {
      "epoch": 0.6762793327678824,
      "grad_norm": 348.3808116966291,
      "learning_rate": 2.2991744377568358e-07,
      "logits/chosen": -0.9435532093048096,
      "logits/rejected": -0.9605557322502136,
      "logps/chosen": -4.827619552612305,
      "logps/rejected": -5.459317207336426,
      "loss": 2.9575,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -48.27620315551758,
      "rewards/margins": 6.316972255706787,
      "rewards/rejected": -54.593170166015625,
      "step": 299
    },
    {
      "epoch": 0.6785411365564037,
      "grad_norm": 179.28273430158572,
      "learning_rate": 2.270578677617601e-07,
      "logits/chosen": -1.0008050203323364,
      "logits/rejected": -0.9990091323852539,
      "logps/chosen": -4.594552993774414,
      "logps/rejected": -5.334099292755127,
      "loss": 3.3764,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -45.945526123046875,
      "rewards/margins": 7.395462512969971,
      "rewards/rejected": -53.34099197387695,
      "step": 300
    },
    {
      "epoch": 0.6808029403449251,
      "grad_norm": 151.0565233114858,
      "learning_rate": 2.242091214689971e-07,
      "logits/chosen": -0.9774994850158691,
      "logits/rejected": -1.0008808374404907,
      "logps/chosen": -4.770328044891357,
      "logps/rejected": -5.569509506225586,
      "loss": 2.2267,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -47.70328140258789,
      "rewards/margins": 7.991814613342285,
      "rewards/rejected": -55.69509506225586,
      "step": 301
    },
    {
      "epoch": 0.6830647441334464,
      "grad_norm": 159.89458772534127,
      "learning_rate": 2.2137138328728456e-07,
      "logits/chosen": -1.0418307781219482,
      "logits/rejected": -1.0276165008544922,
      "logps/chosen": -4.720486164093018,
      "logps/rejected": -5.296718120574951,
      "loss": 2.6692,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -47.20486068725586,
      "rewards/margins": 5.762320518493652,
      "rewards/rejected": -52.96717834472656,
      "step": 302
    },
    {
      "epoch": 0.6853265479219678,
      "grad_norm": 126.48151913696319,
      "learning_rate": 2.1854483091717974e-07,
      "logits/chosen": -1.0288279056549072,
      "logits/rejected": -1.0501220226287842,
      "logps/chosen": -4.512904167175293,
      "logps/rejected": -5.211638927459717,
      "loss": 2.3552,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -45.12904357910156,
      "rewards/margins": 6.987349987030029,
      "rewards/rejected": -52.116390228271484,
      "step": 303
    },
    {
      "epoch": 0.6875883517104892,
      "grad_norm": 135.94321383588283,
      "learning_rate": 2.1572964135877863e-07,
      "logits/chosen": -0.9905492067337036,
      "logits/rejected": -1.0089741945266724,
      "logps/chosen": -4.556299686431885,
      "logps/rejected": -5.186739921569824,
      "loss": 2.8414,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -45.5629997253418,
      "rewards/margins": 6.3044047355651855,
      "rewards/rejected": -51.867401123046875,
      "step": 304
    },
    {
      "epoch": 0.6898501554990104,
      "grad_norm": 125.82619317763663,
      "learning_rate": 2.1292599090063245e-07,
      "logits/chosen": -1.0182273387908936,
      "logits/rejected": -1.0373239517211914,
      "logps/chosen": -4.384622573852539,
      "logps/rejected": -5.199775695800781,
      "loss": 2.0894,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -43.84622573852539,
      "rewards/margins": 8.151532173156738,
      "rewards/rejected": -51.99775695800781,
      "step": 305
    },
    {
      "epoch": 0.6921119592875318,
      "grad_norm": 144.33359341912222,
      "learning_rate": 2.1013405510870824e-07,
      "logits/chosen": -0.9348523020744324,
      "logits/rejected": -0.9837851524353027,
      "logps/chosen": -4.506128311157227,
      "logps/rejected": -5.2762885093688965,
      "loss": 2.0643,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -45.06128692626953,
      "rewards/margins": 7.701597690582275,
      "rewards/rejected": -52.762882232666016,
      "step": 306
    },
    {
      "epoch": 0.6943737630760531,
      "grad_norm": 137.68267743107825,
      "learning_rate": 2.0735400881539494e-07,
      "logits/chosen": -0.9666372537612915,
      "logits/rejected": -0.9910269975662231,
      "logps/chosen": -4.676998138427734,
      "logps/rejected": -5.494973182678223,
      "loss": 2.4669,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -46.76997756958008,
      "rewards/margins": 8.179755210876465,
      "rewards/rejected": -54.949729919433594,
      "step": 307
    },
    {
      "epoch": 0.6966355668645745,
      "grad_norm": 136.55337779354605,
      "learning_rate": 2.0458602610855536e-07,
      "logits/chosen": -1.0127683877944946,
      "logits/rejected": -1.0208343267440796,
      "logps/chosen": -4.608272075653076,
      "logps/rejected": -5.278717517852783,
      "loss": 2.1998,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -46.08271789550781,
      "rewards/margins": 6.704453945159912,
      "rewards/rejected": -52.787174224853516,
      "step": 308
    },
    {
      "epoch": 0.6988973706530959,
      "grad_norm": 150.35850786814652,
      "learning_rate": 2.0183028032062422e-07,
      "logits/chosen": -0.9884274005889893,
      "logits/rejected": -1.007688045501709,
      "logps/chosen": -4.597423076629639,
      "logps/rejected": -5.278217315673828,
      "loss": 2.7826,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -45.97422790527344,
      "rewards/margins": 6.807945251464844,
      "rewards/rejected": -52.78217697143555,
      "step": 309
    },
    {
      "epoch": 0.7011591744416172,
      "grad_norm": 151.16788260358283,
      "learning_rate": 1.9908694401775473e-07,
      "logits/chosen": -1.014832854270935,
      "logits/rejected": -1.0308024883270264,
      "logps/chosen": -4.622461318969727,
      "logps/rejected": -5.307525634765625,
      "loss": 2.4588,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -46.22461700439453,
      "rewards/margins": 6.8506388664245605,
      "rewards/rejected": -53.075252532958984,
      "step": 310
    },
    {
      "epoch": 0.7034209782301385,
      "grad_norm": 143.10054441095096,
      "learning_rate": 1.9635618898901196e-07,
      "logits/chosen": -1.0075442790985107,
      "logits/rejected": -1.0087586641311646,
      "logps/chosen": -5.013138771057129,
      "logps/rejected": -5.722409725189209,
      "loss": 2.7131,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -50.13139343261719,
      "rewards/margins": 7.092708587646484,
      "rewards/rejected": -57.22409439086914,
      "step": 311
    },
    {
      "epoch": 0.7056827820186599,
      "grad_norm": 126.69180335404305,
      "learning_rate": 1.9363818623561565e-07,
      "logits/chosen": -0.9536194801330566,
      "logits/rejected": -0.9867286682128906,
      "logps/chosen": -4.637800693511963,
      "logps/rejected": -5.396078109741211,
      "loss": 2.3183,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -46.37800598144531,
      "rewards/margins": 7.582779884338379,
      "rewards/rejected": -53.960784912109375,
      "step": 312
    },
    {
      "epoch": 0.7079445858071812,
      "grad_norm": 120.13641288355163,
      "learning_rate": 1.9093310596023108e-07,
      "logits/chosen": -0.9519625902175903,
      "logits/rejected": -0.9601365923881531,
      "logps/chosen": -4.475955009460449,
      "logps/rejected": -5.384003639221191,
      "loss": 2.1502,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -44.759552001953125,
      "rewards/margins": 9.080483436584473,
      "rewards/rejected": -53.84003448486328,
      "step": 313
    },
    {
      "epoch": 0.7102063895957026,
      "grad_norm": 157.80538513211965,
      "learning_rate": 1.8824111755631274e-07,
      "logits/chosen": -1.0193778276443481,
      "logits/rejected": -1.0384585857391357,
      "logps/chosen": -4.5386857986450195,
      "logps/rejected": -5.142300605773926,
      "loss": 2.9757,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -45.38685989379883,
      "rewards/margins": 6.036149024963379,
      "rewards/rejected": -51.42300796508789,
      "step": 314
    },
    {
      "epoch": 0.712468193384224,
      "grad_norm": 181.18937775062142,
      "learning_rate": 1.8556238959749457e-07,
      "logits/chosen": -0.9791902303695679,
      "logits/rejected": -0.9818335771560669,
      "logps/chosen": -4.927301406860352,
      "logps/rejected": -5.432847023010254,
      "loss": 3.4906,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -49.273006439208984,
      "rewards/margins": 5.055453777313232,
      "rewards/rejected": -54.32846450805664,
      "step": 315
    },
    {
      "epoch": 0.7147299971727452,
      "grad_norm": 193.86258885290945,
      "learning_rate": 1.8289708982703562e-07,
      "logits/chosen": -0.9614027142524719,
      "logits/rejected": -0.936948299407959,
      "logps/chosen": -4.739333152770996,
      "logps/rejected": -5.496729373931885,
      "loss": 3.2191,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -47.3933219909668,
      "rewards/margins": 7.573972702026367,
      "rewards/rejected": -54.9672966003418,
      "step": 316
    },
    {
      "epoch": 0.7169918009612666,
      "grad_norm": 144.60961684057406,
      "learning_rate": 1.802453851473151e-07,
      "logits/chosen": -1.0016368627548218,
      "logits/rejected": -1.0029720067977905,
      "logps/chosen": -4.8493804931640625,
      "logps/rejected": -5.593348503112793,
      "loss": 2.2796,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -48.493804931640625,
      "rewards/margins": 7.4396843910217285,
      "rewards/rejected": -55.93348693847656,
      "step": 317
    },
    {
      "epoch": 0.719253604749788,
      "grad_norm": 131.40245309400345,
      "learning_rate": 1.7760744160938093e-07,
      "logits/chosen": -0.9554572701454163,
      "logits/rejected": -0.9705080986022949,
      "logps/chosen": -4.564050197601318,
      "logps/rejected": -5.474889278411865,
      "loss": 2.351,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -45.6405029296875,
      "rewards/margins": 9.108393669128418,
      "rewards/rejected": -54.748897552490234,
      "step": 318
    },
    {
      "epoch": 0.7215154085383093,
      "grad_norm": 155.43325768660566,
      "learning_rate": 1.7498342440255135e-07,
      "logits/chosen": -1.00507390499115,
      "logits/rejected": -1.0052752494812012,
      "logps/chosen": -4.724150657653809,
      "logps/rejected": -5.360733985900879,
      "loss": 2.5509,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -47.24150848388672,
      "rewards/margins": 6.365832805633545,
      "rewards/rejected": -53.60734176635742,
      "step": 319
    },
    {
      "epoch": 0.7237772123268307,
      "grad_norm": 131.09223849172065,
      "learning_rate": 1.7237349784407115e-07,
      "logits/chosen": -1.0048574209213257,
      "logits/rejected": -1.0041303634643555,
      "logps/chosen": -4.809683799743652,
      "logps/rejected": -5.5847930908203125,
      "loss": 2.2834,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -48.09683609008789,
      "rewards/margins": 7.751087665557861,
      "rewards/rejected": -55.847923278808594,
      "step": 320
    },
    {
      "epoch": 0.726039016115352,
      "grad_norm": 135.5800803898602,
      "learning_rate": 1.6977782536882178e-07,
      "logits/chosen": -0.934962809085846,
      "logits/rejected": -0.9414641857147217,
      "logps/chosen": -4.202589511871338,
      "logps/rejected": -5.056532859802246,
      "loss": 2.3171,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -42.02588653564453,
      "rewards/margins": 8.539436340332031,
      "rewards/rejected": -50.565330505371094,
      "step": 321
    },
    {
      "epoch": 0.7283008199038733,
      "grad_norm": 114.28778261159624,
      "learning_rate": 1.6719656951908708e-07,
      "logits/chosen": -0.9354251027107239,
      "logits/rejected": -0.961618185043335,
      "logps/chosen": -4.163588047027588,
      "logps/rejected": -4.967078685760498,
      "loss": 2.2743,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -41.63587951660156,
      "rewards/margins": 8.034907341003418,
      "rewards/rejected": -49.6707878112793,
      "step": 322
    },
    {
      "epoch": 0.7305626236923947,
      "grad_norm": 149.49164825917453,
      "learning_rate": 1.6462989193437453e-07,
      "logits/chosen": -1.0084627866744995,
      "logits/rejected": -1.0145008563995361,
      "logps/chosen": -4.514183521270752,
      "logps/rejected": -5.200974464416504,
      "loss": 2.7438,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -45.14183044433594,
      "rewards/margins": 6.867916584014893,
      "rewards/rejected": -52.00975036621094,
      "step": 323
    },
    {
      "epoch": 0.732824427480916,
      "grad_norm": 164.51017995267839,
      "learning_rate": 1.6207795334129365e-07,
      "logits/chosen": -0.9625239968299866,
      "logits/rejected": -0.969192385673523,
      "logps/chosen": -4.7873430252075195,
      "logps/rejected": -5.481894016265869,
      "loss": 2.7085,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -47.8734245300293,
      "rewards/margins": 6.945512771606445,
      "rewards/rejected": -54.81894302368164,
      "step": 324
    },
    {
      "epoch": 0.7350862312694374,
      "grad_norm": 176.57340901183855,
      "learning_rate": 1.5954091354349121e-07,
      "logits/chosen": -0.9980968832969666,
      "logits/rejected": -1.008570909500122,
      "logps/chosen": -4.490519046783447,
      "logps/rejected": -5.037813663482666,
      "loss": 3.1544,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -44.905189514160156,
      "rewards/margins": 5.4729437828063965,
      "rewards/rejected": -50.378135681152344,
      "step": 325
    },
    {
      "epoch": 0.7373480350579588,
      "grad_norm": 291.93760275647674,
      "learning_rate": 1.5701893141164364e-07,
      "logits/chosen": -0.9874584674835205,
      "logits/rejected": -0.9985488653182983,
      "logps/chosen": -4.682331085205078,
      "logps/rejected": -5.434433937072754,
      "loss": 2.9428,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -46.823307037353516,
      "rewards/margins": 7.52103853225708,
      "rewards/rejected": -54.34434509277344,
      "step": 326
    },
    {
      "epoch": 0.73960983884648,
      "grad_norm": 159.05107721522452,
      "learning_rate": 1.545121648735093e-07,
      "logits/chosen": -0.9645061492919922,
      "logits/rejected": -0.9795565605163574,
      "logps/chosen": -4.574826717376709,
      "logps/rejected": -5.205280303955078,
      "loss": 2.9854,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -45.748268127441406,
      "rewards/margins": 6.304535865783691,
      "rewards/rejected": -52.05281066894531,
      "step": 327
    },
    {
      "epoch": 0.7418716426350014,
      "grad_norm": 136.1527260388613,
      "learning_rate": 1.5202077090403863e-07,
      "logits/chosen": -0.99421226978302,
      "logits/rejected": -0.9890978336334229,
      "logps/chosen": -4.0757365226745605,
      "logps/rejected": -4.764214038848877,
      "loss": 2.4082,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -40.75736999511719,
      "rewards/margins": 6.884768486022949,
      "rewards/rejected": -47.64213562011719,
      "step": 328
    },
    {
      "epoch": 0.7441334464235227,
      "grad_norm": 148.6404714088356,
      "learning_rate": 1.495449055155443e-07,
      "logits/chosen": -0.9893301725387573,
      "logits/rejected": -0.9949851036071777,
      "logps/chosen": -4.401925086975098,
      "logps/rejected": -5.235118865966797,
      "loss": 2.1108,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -44.019248962402344,
      "rewards/margins": 8.331939697265625,
      "rewards/rejected": -52.3511848449707,
      "step": 329
    },
    {
      "epoch": 0.7463952502120441,
      "grad_norm": 146.90492941982063,
      "learning_rate": 1.4708472374793112e-07,
      "logits/chosen": -0.9245947599411011,
      "logits/rejected": -0.9359519481658936,
      "logps/chosen": -4.438968658447266,
      "logps/rejected": -4.909884452819824,
      "loss": 3.6505,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -44.389686584472656,
      "rewards/margins": 4.709160804748535,
      "rewards/rejected": -49.09885025024414,
      "step": 330
    },
    {
      "epoch": 0.7486570540005655,
      "grad_norm": 134.59594283680065,
      "learning_rate": 1.4464037965898878e-07,
      "logits/chosen": -0.9060376882553101,
      "logits/rejected": -0.9134998321533203,
      "logps/chosen": -4.321666717529297,
      "logps/rejected": -4.94952392578125,
      "loss": 2.7636,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -43.216670989990234,
      "rewards/margins": 6.278566360473633,
      "rewards/rejected": -49.4952392578125,
      "step": 331
    },
    {
      "epoch": 0.7509188577890868,
      "grad_norm": 135.57897482412505,
      "learning_rate": 1.4221202631474282e-07,
      "logits/chosen": -0.9183069467544556,
      "logits/rejected": -0.9207398891448975,
      "logps/chosen": -4.24176549911499,
      "logps/rejected": -4.887630462646484,
      "loss": 2.6764,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -42.417659759521484,
      "rewards/margins": 6.458644866943359,
      "rewards/rejected": -48.876304626464844,
      "step": 332
    },
    {
      "epoch": 0.7531806615776081,
      "grad_norm": 132.11305657314077,
      "learning_rate": 1.3979981577987113e-07,
      "logits/chosen": -0.9489963054656982,
      "logits/rejected": -0.9431209564208984,
      "logps/chosen": -3.9949569702148438,
      "logps/rejected": -4.739992141723633,
      "loss": 2.4723,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -39.94956588745117,
      "rewards/margins": 7.450351238250732,
      "rewards/rejected": -47.39991760253906,
      "step": 333
    },
    {
      "epoch": 0.7554424653661295,
      "grad_norm": 136.71421838648521,
      "learning_rate": 1.374038991081807e-07,
      "logits/chosen": -0.9887750148773193,
      "logits/rejected": -0.9911076426506042,
      "logps/chosen": -4.311697483062744,
      "logps/rejected": -4.892322540283203,
      "loss": 2.8695,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -43.11697769165039,
      "rewards/margins": 5.80624532699585,
      "rewards/rejected": -48.923221588134766,
      "step": 334
    },
    {
      "epoch": 0.7577042691546508,
      "grad_norm": 129.76539286228083,
      "learning_rate": 1.3502442633314882e-07,
      "logits/chosen": -0.9298081994056702,
      "logits/rejected": -0.9281089305877686,
      "logps/chosen": -3.8088793754577637,
      "logps/rejected": -4.399178981781006,
      "loss": 2.568,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -38.08879470825195,
      "rewards/margins": 5.902992248535156,
      "rewards/rejected": -43.991790771484375,
      "step": 335
    },
    {
      "epoch": 0.7599660729431722,
      "grad_norm": 133.48386572014678,
      "learning_rate": 1.3266154645852815e-07,
      "logits/chosen": -0.9325582981109619,
      "logits/rejected": -0.9300287961959839,
      "logps/chosen": -4.226827621459961,
      "logps/rejected": -4.8680830001831055,
      "loss": 2.4884,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -42.268280029296875,
      "rewards/margins": 6.4125542640686035,
      "rewards/rejected": -48.68083190917969,
      "step": 336
    },
    {
      "epoch": 0.7622278767316936,
      "grad_norm": 155.31497007928593,
      "learning_rate": 1.303154074490152e-07,
      "logits/chosen": -0.9856913685798645,
      "logits/rejected": -0.9596735239028931,
      "logps/chosen": -4.013852119445801,
      "logps/rejected": -4.661082744598389,
      "loss": 2.8335,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -40.13852310180664,
      "rewards/margins": 6.472311973571777,
      "rewards/rejected": -46.6108283996582,
      "step": 337
    },
    {
      "epoch": 0.7644896805202148,
      "grad_norm": 139.18511727180848,
      "learning_rate": 1.2798615622098616e-07,
      "logits/chosen": -0.9739874601364136,
      "logits/rejected": -0.9792473316192627,
      "logps/chosen": -3.9729883670806885,
      "logps/rejected": -4.684140205383301,
      "loss": 2.743,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -39.729881286621094,
      "rewards/margins": 7.111522674560547,
      "rewards/rejected": -46.841407775878906,
      "step": 338
    },
    {
      "epoch": 0.7667514843087362,
      "grad_norm": 132.3553794311643,
      "learning_rate": 1.2567393863329523e-07,
      "logits/chosen": -0.9450173377990723,
      "logits/rejected": -0.9746794104576111,
      "logps/chosen": -4.164608478546143,
      "logps/rejected": -4.965396404266357,
      "loss": 2.3621,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -41.646080017089844,
      "rewards/margins": 8.00788688659668,
      "rewards/rejected": -49.65396499633789,
      "step": 339
    },
    {
      "epoch": 0.7690132880972576,
      "grad_norm": 122.61541284883357,
      "learning_rate": 1.233788994781423e-07,
      "logits/chosen": -0.9901926517486572,
      "logits/rejected": -1.0221422910690308,
      "logps/chosen": -4.009914875030518,
      "logps/rejected": -4.677107334136963,
      "loss": 2.2977,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -40.099151611328125,
      "rewards/margins": 6.671916961669922,
      "rewards/rejected": -46.77106475830078,
      "step": 340
    },
    {
      "epoch": 0.7712750918857789,
      "grad_norm": 128.54731960578127,
      "learning_rate": 1.2110118247200468e-07,
      "logits/chosen": -0.98152756690979,
      "logits/rejected": -0.9877326488494873,
      "logps/chosen": -3.989288568496704,
      "logps/rejected": -4.7238030433654785,
      "loss": 2.0917,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -39.89288330078125,
      "rewards/margins": 7.3451457023620605,
      "rewards/rejected": -47.23802947998047,
      "step": 341
    },
    {
      "epoch": 0.7735368956743003,
      "grad_norm": 152.52119821185264,
      "learning_rate": 1.1884093024663933e-07,
      "logits/chosen": -0.9875534772872925,
      "logits/rejected": -0.9850603342056274,
      "logps/chosen": -3.8040857315063477,
      "logps/rejected": -4.616514682769775,
      "loss": 2.5956,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -38.04085922241211,
      "rewards/margins": 8.124288558959961,
      "rewards/rejected": -46.16515350341797,
      "step": 342
    },
    {
      "epoch": 0.7757986994628217,
      "grad_norm": 137.49550762814457,
      "learning_rate": 1.1659828434014886e-07,
      "logits/chosen": -0.9853401780128479,
      "logits/rejected": -0.9595975279808044,
      "logps/chosen": -3.8625781536102295,
      "logps/rejected": -4.667750835418701,
      "loss": 2.3451,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -38.62577819824219,
      "rewards/margins": 8.051729202270508,
      "rewards/rejected": -46.67750549316406,
      "step": 343
    },
    {
      "epoch": 0.7780605032513429,
      "grad_norm": 151.81630882151333,
      "learning_rate": 1.143733851881203e-07,
      "logits/chosen": -1.016958236694336,
      "logits/rejected": -1.0146615505218506,
      "logps/chosen": -4.202175617218018,
      "logps/rejected": -5.043542861938477,
      "loss": 2.3046,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -42.02175521850586,
      "rewards/margins": 8.413676261901855,
      "rewards/rejected": -50.4354248046875,
      "step": 344
    },
    {
      "epoch": 0.7803223070398643,
      "grad_norm": 132.32110407207458,
      "learning_rate": 1.1216637211483005e-07,
      "logits/chosen": -0.9708240032196045,
      "logits/rejected": -0.9722229242324829,
      "logps/chosen": -4.108154773712158,
      "logps/rejected": -4.720202922821045,
      "loss": 2.8093,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -41.08155059814453,
      "rewards/margins": 6.120481014251709,
      "rewards/rejected": -47.20203399658203,
      "step": 345
    },
    {
      "epoch": 0.7825841108283856,
      "grad_norm": 145.16817928123578,
      "learning_rate": 1.0997738332451936e-07,
      "logits/chosen": -0.9453762769699097,
      "logits/rejected": -0.94859778881073,
      "logps/chosen": -4.420743465423584,
      "logps/rejected": -5.103504180908203,
      "loss": 2.555,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -44.20743179321289,
      "rewards/margins": 6.827612400054932,
      "rewards/rejected": -51.03504943847656,
      "step": 346
    },
    {
      "epoch": 0.784845914616907,
      "grad_norm": 152.89585809443537,
      "learning_rate": 1.0780655589274031e-07,
      "logits/chosen": -1.0250661373138428,
      "logits/rejected": -1.000236988067627,
      "logps/chosen": -4.187366962432861,
      "logps/rejected": -4.821606636047363,
      "loss": 2.2597,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -41.8736686706543,
      "rewards/margins": 6.34239387512207,
      "rewards/rejected": -48.216064453125,
      "step": 347
    },
    {
      "epoch": 0.7871077184054284,
      "grad_norm": 127.0780346687704,
      "learning_rate": 1.056540257577712e-07,
      "logits/chosen": -0.9302332997322083,
      "logits/rejected": -0.9465296268463135,
      "logps/chosen": -4.749317646026611,
      "logps/rejected": -5.628003120422363,
      "loss": 1.8934,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -47.49317932128906,
      "rewards/margins": 8.786852836608887,
      "rewards/rejected": -56.280033111572266,
      "step": 348
    },
    {
      "epoch": 0.7893695221939496,
      "grad_norm": 126.51351161185914,
      "learning_rate": 1.0351992771210554e-07,
      "logits/chosen": -0.9633577466011047,
      "logits/rejected": -0.970734179019928,
      "logps/chosen": -4.292740821838379,
      "logps/rejected": -5.064752578735352,
      "loss": 2.5149,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": -42.92741012573242,
      "rewards/margins": 7.720117092132568,
      "rewards/rejected": -50.647525787353516,
      "step": 349
    },
    {
      "epoch": 0.791631325982471,
      "grad_norm": 166.01639039281858,
      "learning_rate": 1.0140439539400953e-07,
      "logits/chosen": -0.9338628649711609,
      "logits/rejected": -0.950715184211731,
      "logps/chosen": -4.245103359222412,
      "logps/rejected": -4.8726959228515625,
      "loss": 3.156,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -42.4510383605957,
      "rewards/margins": 6.275918483734131,
      "rewards/rejected": -48.72695541381836,
      "step": 350
    },
    {
      "epoch": 0.7938931297709924,
      "grad_norm": 131.0281285552435,
      "learning_rate": 9.930756127915488e-08,
      "logits/chosen": -0.9479851722717285,
      "logits/rejected": -0.9509535431861877,
      "logps/chosen": -4.242725372314453,
      "logps/rejected": -4.937259674072266,
      "loss": 2.4118,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -42.4272575378418,
      "rewards/margins": 6.945339679718018,
      "rewards/rejected": -49.37260055541992,
      "step": 351
    },
    {
      "epoch": 0.7961549335595137,
      "grad_norm": 159.53197135066353,
      "learning_rate": 9.722955667232242e-08,
      "logits/chosen": -0.9826103448867798,
      "logits/rejected": -0.9935145378112793,
      "logps/chosen": -4.630585670471191,
      "logps/rejected": -5.20355224609375,
      "loss": 3.0279,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -46.305850982666016,
      "rewards/margins": 5.729671955108643,
      "rewards/rejected": -52.0355224609375,
      "step": 352
    },
    {
      "epoch": 0.7984167373480351,
      "grad_norm": 131.6547821736773,
      "learning_rate": 9.517051169918016e-08,
      "logits/chosen": -0.9904384613037109,
      "logits/rejected": -0.9811457991600037,
      "logps/chosen": -4.245575904846191,
      "logps/rejected": -4.880057334899902,
      "loss": 3.1398,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -42.45575714111328,
      "rewards/margins": 6.344809532165527,
      "rewards/rejected": -48.800567626953125,
      "step": 353
    },
    {
      "epoch": 0.8006785411365565,
      "grad_norm": 142.9124100481491,
      "learning_rate": 9.313055529813412e-08,
      "logits/chosen": -0.9195039868354797,
      "logits/rejected": -0.9531816244125366,
      "logps/chosen": -4.369924545288086,
      "logps/rejected": -5.106759071350098,
      "loss": 2.3245,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -43.69923782348633,
      "rewards/margins": 7.368349552154541,
      "rewards/rejected": -51.067588806152344,
      "step": 354
    },
    {
      "epoch": 0.8029403449250777,
      "grad_norm": 141.5454480732975,
      "learning_rate": 9.110981521225532e-08,
      "logits/chosen": -0.9761883616447449,
      "logits/rejected": -0.9907628297805786,
      "logps/chosen": -4.390708923339844,
      "logps/rejected": -5.035253047943115,
      "loss": 2.7619,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -43.907081604003906,
      "rewards/margins": 6.445442199707031,
      "rewards/rejected": -50.35253143310547,
      "step": 355
    },
    {
      "epoch": 0.8052021487135991,
      "grad_norm": 141.24588926834747,
      "learning_rate": 8.910841798127884e-08,
      "logits/chosen": -0.932433545589447,
      "logits/rejected": -0.9380560517311096,
      "logps/chosen": -4.490683555603027,
      "logps/rejected": -5.1833906173706055,
      "loss": 2.4937,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -44.906837463378906,
      "rewards/margins": 6.927071571350098,
      "rewards/rejected": -51.83391189575195,
      "step": 356
    },
    {
      "epoch": 0.8074639525021204,
      "grad_norm": 153.489266818857,
      "learning_rate": 8.712648893368139e-08,
      "logits/chosen": -0.9466272592544556,
      "logits/rejected": -0.9827693700790405,
      "logps/chosen": -4.324895858764648,
      "logps/rejected": -5.1728057861328125,
      "loss": 2.4164,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -43.24895477294922,
      "rewards/margins": 8.479098320007324,
      "rewards/rejected": -51.72805404663086,
      "step": 357
    },
    {
      "epoch": 0.8097257562906418,
      "grad_norm": 134.69471486150493,
      "learning_rate": 8.516415217883186e-08,
      "logits/chosen": -0.9450441598892212,
      "logits/rejected": -0.9372915029525757,
      "logps/chosen": -4.20553731918335,
      "logps/rejected": -5.077816486358643,
      "loss": 2.1505,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": -42.05537796020508,
      "rewards/margins": 8.722793579101562,
      "rewards/rejected": -50.778167724609375,
      "step": 358
    },
    {
      "epoch": 0.8119875600791632,
      "grad_norm": 172.05066369257742,
      "learning_rate": 8.32215305992209e-08,
      "logits/chosen": -0.9893842935562134,
      "logits/rejected": -1.0069129467010498,
      "logps/chosen": -4.14910888671875,
      "logps/rejected": -4.812809944152832,
      "loss": 2.9886,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -41.491092681884766,
      "rewards/margins": 6.6370015144348145,
      "rewards/rejected": -48.12809371948242,
      "step": 359
    },
    {
      "epoch": 0.8142493638676844,
      "grad_norm": 118.45196699685957,
      "learning_rate": 8.129874584276448e-08,
      "logits/chosen": -0.9478542804718018,
      "logits/rejected": -0.9472507834434509,
      "logps/chosen": -4.2414398193359375,
      "logps/rejected": -5.106539726257324,
      "loss": 1.8634,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -42.41439437866211,
      "rewards/margins": 8.650999069213867,
      "rewards/rejected": -51.06539535522461,
      "step": 360
    },
    {
      "epoch": 0.8165111676562058,
      "grad_norm": 147.08542318664797,
      "learning_rate": 7.939591831518746e-08,
      "logits/chosen": -0.9485371112823486,
      "logits/rejected": -0.9716846942901611,
      "logps/chosen": -4.281550407409668,
      "logps/rejected": -4.866559028625488,
      "loss": 2.4179,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -42.81550979614258,
      "rewards/margins": 5.850086212158203,
      "rewards/rejected": -48.66558837890625,
      "step": 361
    },
    {
      "epoch": 0.8187729714447272,
      "grad_norm": 140.7832629501282,
      "learning_rate": 7.751316717248304e-08,
      "logits/chosen": -0.9519385695457458,
      "logits/rejected": -0.963306188583374,
      "logps/chosen": -4.716574668884277,
      "logps/rejected": -5.629094123840332,
      "loss": 2.4426,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -47.165740966796875,
      "rewards/margins": 9.125191688537598,
      "rewards/rejected": -56.29093933105469,
      "step": 362
    },
    {
      "epoch": 0.8210347752332485,
      "grad_norm": 161.95945831881926,
      "learning_rate": 7.565061031345142e-08,
      "logits/chosen": -0.9452296495437622,
      "logits/rejected": -0.9586248993873596,
      "logps/chosen": -4.896112442016602,
      "logps/rejected": -5.624301910400391,
      "loss": 2.4316,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -48.961124420166016,
      "rewards/margins": 7.281898498535156,
      "rewards/rejected": -56.24302673339844,
      "step": 363
    },
    {
      "epoch": 0.8232965790217699,
      "grad_norm": 192.2424118773627,
      "learning_rate": 7.380836437231686e-08,
      "logits/chosen": -0.9347036480903625,
      "logits/rejected": -0.9242918491363525,
      "logps/chosen": -4.409902095794678,
      "logps/rejected": -5.111190319061279,
      "loss": 2.5825,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -44.099021911621094,
      "rewards/margins": 7.012877941131592,
      "rewards/rejected": -51.111900329589844,
      "step": 364
    },
    {
      "epoch": 0.8255583828102913,
      "grad_norm": 124.6304811777762,
      "learning_rate": 7.198654471142371e-08,
      "logits/chosen": -0.9537335634231567,
      "logits/rejected": -0.9453788995742798,
      "logps/chosen": -4.486839771270752,
      "logps/rejected": -5.401633262634277,
      "loss": 2.3187,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -44.8683967590332,
      "rewards/margins": 9.14793872833252,
      "rewards/rejected": -54.016334533691406,
      "step": 365
    },
    {
      "epoch": 0.8278201865988125,
      "grad_norm": 163.83416320888907,
      "learning_rate": 7.01852654140132e-08,
      "logits/chosen": -1.0016599893569946,
      "logits/rejected": -1.0151605606079102,
      "logps/chosen": -4.924201965332031,
      "logps/rejected": -5.710483074188232,
      "loss": 2.5367,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -49.24201965332031,
      "rewards/margins": 7.862810134887695,
      "rewards/rejected": -57.104827880859375,
      "step": 366
    },
    {
      "epoch": 0.8300819903873339,
      "grad_norm": 144.1303882492354,
      "learning_rate": 6.840463927707833e-08,
      "logits/chosen": -0.9487999677658081,
      "logits/rejected": -0.9631211757659912,
      "logps/chosen": -4.902750015258789,
      "logps/rejected": -5.617919445037842,
      "loss": 2.5045,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -49.027503967285156,
      "rewards/margins": 7.1516900062561035,
      "rewards/rejected": -56.179195404052734,
      "step": 367
    },
    {
      "epoch": 0.8323437941758552,
      "grad_norm": 144.87253676271803,
      "learning_rate": 6.664477780430138e-08,
      "logits/chosen": -0.952880859375,
      "logits/rejected": -0.9492167234420776,
      "logps/chosen": -4.776792049407959,
      "logps/rejected": -5.370887756347656,
      "loss": 2.717,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -47.767921447753906,
      "rewards/margins": 5.940953731536865,
      "rewards/rejected": -53.7088737487793,
      "step": 368
    },
    {
      "epoch": 0.8346055979643766,
      "grad_norm": 184.09075370725176,
      "learning_rate": 6.49057911990711e-08,
      "logits/chosen": -0.9280183911323547,
      "logits/rejected": -0.9295321702957153,
      "logps/chosen": -4.6214423179626465,
      "logps/rejected": -5.288618087768555,
      "loss": 2.8896,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -46.21442413330078,
      "rewards/margins": 6.671757221221924,
      "rewards/rejected": -52.88618469238281,
      "step": 369
    },
    {
      "epoch": 0.836867401752898,
      "grad_norm": 156.0668928651448,
      "learning_rate": 6.318778835758189e-08,
      "logits/chosen": -0.9676793813705444,
      "logits/rejected": -0.9772645235061646,
      "logps/chosen": -4.79730224609375,
      "logps/rejected": -5.520888328552246,
      "loss": 2.0356,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -47.9730224609375,
      "rewards/margins": 7.235863208770752,
      "rewards/rejected": -55.20888137817383,
      "step": 370
    },
    {
      "epoch": 0.8391292055414192,
      "grad_norm": 181.31703942045965,
      "learning_rate": 6.149087686201433e-08,
      "logits/chosen": -0.9715522527694702,
      "logits/rejected": -0.9828546047210693,
      "logps/chosen": -4.468597888946533,
      "logps/rejected": -5.149814605712891,
      "loss": 3.3273,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -44.685977935791016,
      "rewards/margins": 6.812170505523682,
      "rewards/rejected": -51.498146057128906,
      "step": 371
    },
    {
      "epoch": 0.8413910093299406,
      "grad_norm": 157.64317964257543,
      "learning_rate": 5.98151629737988e-08,
      "logits/chosen": -0.9902997016906738,
      "logits/rejected": -0.9751971960067749,
      "logps/chosen": -4.583035469055176,
      "logps/rejected": -5.4460954666137695,
      "loss": 2.1937,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -45.83035659790039,
      "rewards/margins": 8.630597114562988,
      "rewards/rejected": -54.46095275878906,
      "step": 372
    },
    {
      "epoch": 0.843652813118462,
      "grad_norm": 126.70730972040158,
      "learning_rate": 5.816075162696097e-08,
      "logits/chosen": -0.9920557737350464,
      "logits/rejected": -1.0163335800170898,
      "logps/chosen": -4.634104251861572,
      "logps/rejected": -5.2726593017578125,
      "loss": 2.3016,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -46.341041564941406,
      "rewards/margins": 6.3855462074279785,
      "rewards/rejected": -52.72658920288086,
      "step": 373
    },
    {
      "epoch": 0.8459146169069833,
      "grad_norm": 136.57228607111298,
      "learning_rate": 5.6527746421551046e-08,
      "logits/chosen": -0.9222903847694397,
      "logits/rejected": -0.9167163372039795,
      "logps/chosen": -4.5692830085754395,
      "logps/rejected": -5.321811676025391,
      "loss": 2.4519,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -45.69283676147461,
      "rewards/margins": 7.525275230407715,
      "rewards/rejected": -53.218109130859375,
      "step": 374
    },
    {
      "epoch": 0.8481764206955047,
      "grad_norm": 147.12870026365698,
      "learning_rate": 5.4916249617156064e-08,
      "logits/chosen": -0.9463968873023987,
      "logits/rejected": -0.9552465677261353,
      "logps/chosen": -4.313798427581787,
      "logps/rejected": -4.9934186935424805,
      "loss": 2.6259,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -43.13798141479492,
      "rewards/margins": 6.796201705932617,
      "rewards/rejected": -49.93418502807617,
      "step": 375
    },
    {
      "epoch": 0.8504382244840261,
      "grad_norm": 161.79684082712205,
      "learning_rate": 5.332636212649646e-08,
      "logits/chosen": -0.9481571316719055,
      "logits/rejected": -0.9492475986480713,
      "logps/chosen": -4.502243995666504,
      "logps/rejected": -5.292466640472412,
      "loss": 2.0549,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -45.02243423461914,
      "rewards/margins": 7.902227878570557,
      "rewards/rejected": -52.924659729003906,
      "step": 376
    },
    {
      "epoch": 0.8527000282725473,
      "grad_norm": 158.20408842262404,
      "learning_rate": 5.17581835091069e-08,
      "logits/chosen": -0.940109372138977,
      "logits/rejected": -0.9614642858505249,
      "logps/chosen": -4.670576095581055,
      "logps/rejected": -5.383021831512451,
      "loss": 2.734,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -46.70576095581055,
      "rewards/margins": 7.124454021453857,
      "rewards/rejected": -53.83021545410156,
      "step": 377
    },
    {
      "epoch": 0.8549618320610687,
      "grad_norm": 131.07408134426578,
      "learning_rate": 5.02118119651016e-08,
      "logits/chosen": -0.9807331562042236,
      "logits/rejected": -0.98893803358078,
      "logps/chosen": -4.549108028411865,
      "logps/rejected": -5.304394721984863,
      "loss": 2.3562,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -45.49108123779297,
      "rewards/margins": 7.552868366241455,
      "rewards/rejected": -53.043949127197266,
      "step": 378
    },
    {
      "epoch": 0.85722363584959,
      "grad_norm": 159.82635061305217,
      "learning_rate": 4.868734432902526e-08,
      "logits/chosen": -1.0292322635650635,
      "logits/rejected": -1.0086599588394165,
      "logps/chosen": -4.577394485473633,
      "logps/rejected": -5.43131160736084,
      "loss": 2.9397,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -45.77394104003906,
      "rewards/margins": 8.539175987243652,
      "rewards/rejected": -54.313114166259766,
      "step": 379
    },
    {
      "epoch": 0.8594854396381114,
      "grad_norm": 153.5753175656312,
      "learning_rate": 4.7184876063789134e-08,
      "logits/chosen": -0.973901629447937,
      "logits/rejected": -0.9794867634773254,
      "logps/chosen": -4.038912296295166,
      "logps/rejected": -4.743659019470215,
      "loss": 2.5866,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -40.38911437988281,
      "rewards/margins": 7.0474653244018555,
      "rewards/rejected": -47.43658447265625,
      "step": 380
    },
    {
      "epoch": 0.8617472434266328,
      "grad_norm": 127.21695474983532,
      "learning_rate": 4.570450125469314e-08,
      "logits/chosen": -0.9439151287078857,
      "logits/rejected": -0.9651936292648315,
      "logps/chosen": -4.656189441680908,
      "logps/rejected": -5.580712795257568,
      "loss": 2.0476,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -46.561893463134766,
      "rewards/margins": 9.245238304138184,
      "rewards/rejected": -55.80712890625,
      "step": 381
    },
    {
      "epoch": 0.864009047215154,
      "grad_norm": 140.9365653388638,
      "learning_rate": 4.424631260353378e-08,
      "logits/chosen": -0.9873952865600586,
      "logits/rejected": -0.9998773336410522,
      "logps/chosen": -4.442070960998535,
      "logps/rejected": -5.129306316375732,
      "loss": 2.568,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -44.42071533203125,
      "rewards/margins": 6.872350215911865,
      "rewards/rejected": -51.293060302734375,
      "step": 382
    },
    {
      "epoch": 0.8662708510036754,
      "grad_norm": 123.99421688298398,
      "learning_rate": 4.281040142280008e-08,
      "logits/chosen": -1.0014389753341675,
      "logits/rejected": -1.0032625198364258,
      "logps/chosen": -4.253451347351074,
      "logps/rejected": -5.129884719848633,
      "loss": 1.7937,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -42.534515380859375,
      "rewards/margins": 8.764336585998535,
      "rewards/rejected": -51.298851013183594,
      "step": 383
    },
    {
      "epoch": 0.8685326547921968,
      "grad_norm": 147.9606819458139,
      "learning_rate": 4.1396857629954286e-08,
      "logits/chosen": -0.9735297560691833,
      "logits/rejected": -0.9905420541763306,
      "logps/chosen": -4.999640464782715,
      "logps/rejected": -5.769815921783447,
      "loss": 2.6449,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -49.99640655517578,
      "rewards/margins": 7.7017502784729,
      "rewards/rejected": -57.698158264160156,
      "step": 384
    },
    {
      "epoch": 0.8707944585807181,
      "grad_norm": 145.63899265924684,
      "learning_rate": 4.000576974180232e-08,
      "logits/chosen": -0.9300609827041626,
      "logits/rejected": -0.9512342214584351,
      "logps/chosen": -4.364050388336182,
      "logps/rejected": -5.139545917510986,
      "loss": 2.5334,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -43.6405029296875,
      "rewards/margins": 7.7549591064453125,
      "rewards/rejected": -51.39546203613281,
      "step": 385
    },
    {
      "epoch": 0.8730562623692395,
      "grad_norm": 134.08121981811036,
      "learning_rate": 3.8637224868950066e-08,
      "logits/chosen": -0.9386020302772522,
      "logits/rejected": -0.9424499273300171,
      "logps/chosen": -4.373361587524414,
      "logps/rejected": -5.010212421417236,
      "loss": 2.7981,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -43.73361587524414,
      "rewards/margins": 6.368506908416748,
      "rewards/rejected": -50.10212326049805,
      "step": 386
    },
    {
      "epoch": 0.8753180661577609,
      "grad_norm": 154.64601318433506,
      "learning_rate": 3.729130871034885e-08,
      "logits/chosen": -0.95456862449646,
      "logits/rejected": -0.9573556184768677,
      "logps/chosen": -4.52392578125,
      "logps/rejected": -5.276218891143799,
      "loss": 2.5976,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -45.239253997802734,
      "rewards/margins": 7.5229339599609375,
      "rewards/rejected": -52.76219177246094,
      "step": 387
    },
    {
      "epoch": 0.8775798699462821,
      "grad_norm": 154.26105591417908,
      "learning_rate": 3.596810554792888e-08,
      "logits/chosen": -0.9463018178939819,
      "logits/rejected": -0.954681396484375,
      "logps/chosen": -4.434272289276123,
      "logps/rejected": -5.201148509979248,
      "loss": 2.6181,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -44.34272003173828,
      "rewards/margins": 7.668770790100098,
      "rewards/rejected": -52.01148986816406,
      "step": 388
    },
    {
      "epoch": 0.8798416737348035,
      "grad_norm": 147.27634883503094,
      "learning_rate": 3.466769824132116e-08,
      "logits/chosen": -0.9583290219306946,
      "logits/rejected": -0.9514294862747192,
      "logps/chosen": -4.45271110534668,
      "logps/rejected": -5.209290981292725,
      "loss": 2.2404,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -44.52710723876953,
      "rewards/margins": 7.565799236297607,
      "rewards/rejected": -52.09291076660156,
      "step": 389
    },
    {
      "epoch": 0.8821034775233249,
      "grad_norm": 152.7393534346966,
      "learning_rate": 3.339016822266925e-08,
      "logits/chosen": -0.9181097149848938,
      "logits/rejected": -0.9469025135040283,
      "logps/chosen": -4.571746826171875,
      "logps/rejected": -5.5013909339904785,
      "loss": 1.7452,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": -45.717464447021484,
      "rewards/margins": 9.296442031860352,
      "rewards/rejected": -55.01390838623047,
      "step": 390
    },
    {
      "epoch": 0.8843652813118462,
      "grad_norm": 147.69164041581885,
      "learning_rate": 3.213559549152958e-08,
      "logits/chosen": -0.977473795413971,
      "logits/rejected": -0.9800957441329956,
      "logps/chosen": -4.443673610687256,
      "logps/rejected": -5.2254157066345215,
      "loss": 2.5852,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -44.436729431152344,
      "rewards/margins": 7.817424774169922,
      "rewards/rejected": -52.2541618347168,
      "step": 391
    },
    {
      "epoch": 0.8866270851003676,
      "grad_norm": 148.653528854335,
      "learning_rate": 3.090405860986203e-08,
      "logits/chosen": -0.9800371527671814,
      "logits/rejected": -1.016266942024231,
      "logps/chosen": -4.722820281982422,
      "logps/rejected": -5.609055519104004,
      "loss": 2.4299,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -47.22819900512695,
      "rewards/margins": 8.86235523223877,
      "rewards/rejected": -56.090553283691406,
      "step": 392
    },
    {
      "epoch": 0.8888888888888888,
      "grad_norm": 144.92734187701728,
      "learning_rate": 2.9695634697110315e-08,
      "logits/chosen": -0.9289108514785767,
      "logits/rejected": -0.9394564032554626,
      "logps/chosen": -4.453829765319824,
      "logps/rejected": -5.3411054611206055,
      "loss": 2.6235,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -44.53829574584961,
      "rewards/margins": 8.872754096984863,
      "rewards/rejected": -53.41105651855469,
      "step": 393
    },
    {
      "epoch": 0.8911506926774102,
      "grad_norm": 161.40695882406106,
      "learning_rate": 2.8510399425372766e-08,
      "logits/chosen": -0.9479678273200989,
      "logits/rejected": -0.9467705488204956,
      "logps/chosen": -4.576265811920166,
      "logps/rejected": -5.270918846130371,
      "loss": 2.6714,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -45.76266098022461,
      "rewards/margins": 6.946530342102051,
      "rewards/rejected": -52.70918655395508,
      "step": 394
    },
    {
      "epoch": 0.8934124964659316,
      "grad_norm": 162.16334658512758,
      "learning_rate": 2.734842701466329e-08,
      "logits/chosen": -0.9540742635726929,
      "logits/rejected": -0.9456802010536194,
      "logps/chosen": -4.940086364746094,
      "logps/rejected": -5.668696880340576,
      "loss": 2.2655,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -49.4008674621582,
      "rewards/margins": 7.286104202270508,
      "rewards/rejected": -56.686973571777344,
      "step": 395
    },
    {
      "epoch": 0.8956743002544529,
      "grad_norm": 154.19569156235414,
      "learning_rate": 2.6209790228264438e-08,
      "logits/chosen": -0.9714961647987366,
      "logits/rejected": -0.9679863452911377,
      "logps/chosen": -4.16938591003418,
      "logps/rejected": -4.939549446105957,
      "loss": 2.2384,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": -41.69386291503906,
      "rewards/margins": 7.701635837554932,
      "rewards/rejected": -49.39550018310547,
      "step": 396
    },
    {
      "epoch": 0.8979361040429743,
      "grad_norm": 149.68348291535114,
      "learning_rate": 2.5094560368170305e-08,
      "logits/chosen": -0.9434354305267334,
      "logits/rejected": -0.9568273425102234,
      "logps/chosen": -4.757465839385986,
      "logps/rejected": -5.512124061584473,
      "loss": 2.3075,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -47.57465744018555,
      "rewards/margins": 7.546583652496338,
      "rewards/rejected": -55.12124252319336,
      "step": 397
    },
    {
      "epoch": 0.9001979078314957,
      "grad_norm": 139.83871846925578,
      "learning_rate": 2.4002807270621893e-08,
      "logits/chosen": -0.9892213344573975,
      "logits/rejected": -0.9795714020729065,
      "logps/chosen": -4.536479473114014,
      "logps/rejected": -5.248366832733154,
      "loss": 2.4449,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -45.36479187011719,
      "rewards/margins": 7.118875026702881,
      "rewards/rejected": -52.48366928100586,
      "step": 398
    },
    {
      "epoch": 0.9024597116200169,
      "grad_norm": 166.9049138837263,
      "learning_rate": 2.293459930173354e-08,
      "logits/chosen": -0.9818339943885803,
      "logits/rejected": -0.9937196969985962,
      "logps/chosen": -4.615837574005127,
      "logps/rejected": -5.3938493728637695,
      "loss": 2.3729,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -46.15837478637695,
      "rewards/margins": 7.780113220214844,
      "rewards/rejected": -53.93849182128906,
      "step": 399
    },
    {
      "epoch": 0.9047215154085383,
      "grad_norm": 140.62439250035044,
      "learning_rate": 2.189000335321256e-08,
      "logits/chosen": -0.946580171585083,
      "logits/rejected": -0.9496131539344788,
      "logps/chosen": -4.469082355499268,
      "logps/rejected": -5.115677833557129,
      "loss": 2.9326,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -44.690818786621094,
      "rewards/margins": 6.465954780578613,
      "rewards/rejected": -51.156776428222656,
      "step": 400
    },
    {
      "epoch": 0.9069833191970597,
      "grad_norm": 160.13785787767887,
      "learning_rate": 2.086908483816954e-08,
      "logits/chosen": -0.9588191509246826,
      "logits/rejected": -0.9561234712600708,
      "logps/chosen": -4.833827495574951,
      "logps/rejected": -5.5546088218688965,
      "loss": 2.5806,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -48.33827590942383,
      "rewards/margins": 7.207812309265137,
      "rewards/rejected": -55.54608917236328,
      "step": 401
    },
    {
      "epoch": 0.909245122985581,
      "grad_norm": 143.62250331217595,
      "learning_rate": 1.9871907687022717e-08,
      "logits/chosen": -0.9328145980834961,
      "logits/rejected": -0.9461118578910828,
      "logps/chosen": -4.385026931762695,
      "logps/rejected": -5.071187496185303,
      "loss": 2.4165,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -43.85027313232422,
      "rewards/margins": 6.861605167388916,
      "rewards/rejected": -50.71187973022461,
      "step": 402
    },
    {
      "epoch": 0.9115069267741024,
      "grad_norm": 134.2795025444658,
      "learning_rate": 1.889853434349451e-08,
      "logits/chosen": -0.9710885286331177,
      "logits/rejected": -0.9686489105224609,
      "logps/chosen": -4.349798679351807,
      "logps/rejected": -5.178468227386475,
      "loss": 2.441,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -43.49799346923828,
      "rewards/margins": 8.286693572998047,
      "rewards/rejected": -51.78468322753906,
      "step": 403
    },
    {
      "epoch": 0.9137687305626236,
      "grad_norm": 164.30503356987847,
      "learning_rate": 1.7949025760701164e-08,
      "logits/chosen": -0.950221836566925,
      "logits/rejected": -0.9404230117797852,
      "logps/chosen": -4.71327543258667,
      "logps/rejected": -5.3966755867004395,
      "loss": 2.4213,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -47.132755279541016,
      "rewards/margins": 6.8340020179748535,
      "rewards/rejected": -53.96676254272461,
      "step": 404
    },
    {
      "epoch": 0.916030534351145,
      "grad_norm": 158.87149794688602,
      "learning_rate": 1.7023441397336023e-08,
      "logits/chosen": -0.973084568977356,
      "logits/rejected": -0.98138827085495,
      "logps/chosen": -4.376833438873291,
      "logps/rejected": -5.152012825012207,
      "loss": 2.1802,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -43.76833724975586,
      "rewards/margins": 7.751791954040527,
      "rewards/rejected": -51.52012634277344,
      "step": 405
    },
    {
      "epoch": 0.9182923381396664,
      "grad_norm": 136.71185170412178,
      "learning_rate": 1.6121839213945854e-08,
      "logits/chosen": -0.943014919757843,
      "logits/rejected": -0.9729979634284973,
      "logps/chosen": -4.3754682540893555,
      "logps/rejected": -5.180696487426758,
      "loss": 2.6478,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -43.75468826293945,
      "rewards/margins": 8.052278518676758,
      "rewards/rejected": -51.80696487426758,
      "step": 406
    },
    {
      "epoch": 0.9205541419281877,
      "grad_norm": 184.17485871176777,
      "learning_rate": 1.5244275669301777e-08,
      "logits/chosen": -0.9659703373908997,
      "logits/rejected": -0.9741247892379761,
      "logps/chosen": -4.513559818267822,
      "logps/rejected": -5.29529333114624,
      "loss": 2.6662,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -45.135597229003906,
      "rewards/margins": 7.817337512969971,
      "rewards/rejected": -52.95293426513672,
      "step": 407
    },
    {
      "epoch": 0.9228159457167091,
      "grad_norm": 153.29528491119703,
      "learning_rate": 1.4390805716863398e-08,
      "logits/chosen": -0.9317939281463623,
      "logits/rejected": -0.9431477785110474,
      "logps/chosen": -4.3477582931518555,
      "logps/rejected": -4.961559295654297,
      "loss": 3.0725,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -43.47758102416992,
      "rewards/margins": 6.138009071350098,
      "rewards/rejected": -49.615596771240234,
      "step": 408
    },
    {
      "epoch": 0.9250777495052305,
      "grad_norm": 147.4396268964287,
      "learning_rate": 1.3561482801337908e-08,
      "logits/chosen": -0.9190540909767151,
      "logits/rejected": -0.9394365549087524,
      "logps/chosen": -4.4133453369140625,
      "logps/rejected": -5.246624946594238,
      "loss": 2.8243,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -44.13344955444336,
      "rewards/margins": 8.332796096801758,
      "rewards/rejected": -52.46624755859375,
      "step": 409
    },
    {
      "epoch": 0.9273395532937517,
      "grad_norm": 147.41157160150556,
      "learning_rate": 1.2756358855332904e-08,
      "logits/chosen": -0.9719132781028748,
      "logits/rejected": -0.9848355054855347,
      "logps/chosen": -4.328832626342773,
      "logps/rejected": -4.96996545791626,
      "loss": 3.201,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -43.288326263427734,
      "rewards/margins": 6.4113287925720215,
      "rewards/rejected": -49.69965362548828,
      "step": 410
    },
    {
      "epoch": 0.9296013570822731,
      "grad_norm": 147.810411578744,
      "learning_rate": 1.1975484296105154e-08,
      "logits/chosen": -0.9302911758422852,
      "logits/rejected": -0.9466784596443176,
      "logps/chosen": -4.584578037261963,
      "logps/rejected": -5.3103132247924805,
      "loss": 2.8047,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -45.845787048339844,
      "rewards/margins": 7.2573466300964355,
      "rewards/rejected": -53.10313415527344,
      "step": 411
    },
    {
      "epoch": 0.9318631608707945,
      "grad_norm": 152.41484142524448,
      "learning_rate": 1.1218908022402374e-08,
      "logits/chosen": -0.928033173084259,
      "logits/rejected": -0.940939724445343,
      "logps/chosen": -4.2157416343688965,
      "logps/rejected": -5.0531816482543945,
      "loss": 2.3413,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -42.157413482666016,
      "rewards/margins": 8.374399185180664,
      "rewards/rejected": -50.53181457519531,
      "step": 412
    },
    {
      "epoch": 0.9341249646593158,
      "grad_norm": 159.91966152195383,
      "learning_rate": 1.0486677411402079e-08,
      "logits/chosen": -1.0264177322387695,
      "logits/rejected": -1.0172362327575684,
      "logps/chosen": -4.612265110015869,
      "logps/rejected": -5.4898295402526855,
      "loss": 2.7482,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -46.12265396118164,
      "rewards/margins": 8.77564525604248,
      "rewards/rejected": -54.89829635620117,
      "step": 413
    },
    {
      "epoch": 0.9363867684478372,
      "grad_norm": 147.1808101672586,
      "learning_rate": 9.778838315744353e-09,
      "logits/chosen": -0.9884222149848938,
      "logits/rejected": -0.9936903715133667,
      "logps/chosen": -4.626962661743164,
      "logps/rejected": -5.381559371948242,
      "loss": 2.2191,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -46.269622802734375,
      "rewards/margins": 7.545971393585205,
      "rewards/rejected": -53.81559371948242,
      "step": 414
    },
    {
      "epoch": 0.9386485722363584,
      "grad_norm": 158.74475786885856,
      "learning_rate": 9.095435060660595e-09,
      "logits/chosen": -0.9156569242477417,
      "logits/rejected": -0.9195461273193359,
      "logps/chosen": -4.509635925292969,
      "logps/rejected": -5.232195854187012,
      "loss": 2.663,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -45.09635543823242,
      "rewards/margins": 7.225605010986328,
      "rewards/rejected": -52.321964263916016,
      "step": 415
    },
    {
      "epoch": 0.9409103760248798,
      "grad_norm": 178.1295778190553,
      "learning_rate": 8.436510441197864e-09,
      "logits/chosen": -0.9454046487808228,
      "logits/rejected": -0.9776556491851807,
      "logps/chosen": -4.444124221801758,
      "logps/rejected": -5.162923336029053,
      "loss": 2.9006,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -44.44124221801758,
      "rewards/margins": 7.187990188598633,
      "rewards/rejected": -51.62923812866211,
      "step": 416
    },
    {
      "epoch": 0.9431721798134012,
      "grad_norm": 245.5564838113027,
      "learning_rate": 7.802105719539076e-09,
      "logits/chosen": -0.9548664689064026,
      "logits/rejected": -0.9679651260375977,
      "logps/chosen": -4.6692795753479,
      "logps/rejected": -5.344656944274902,
      "loss": 3.2472,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -46.69279861450195,
      "rewards/margins": 6.753774642944336,
      "rewards/rejected": -53.44656753540039,
      "step": 417
    },
    {
      "epoch": 0.9454339836019225,
      "grad_norm": 138.25589021962577,
      "learning_rate": 7.1922606224192e-09,
      "logits/chosen": -0.9730819463729858,
      "logits/rejected": -0.9867460131645203,
      "logps/chosen": -4.586428642272949,
      "logps/rejected": -5.304539680480957,
      "loss": 2.396,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -45.864295959472656,
      "rewards/margins": 7.181105613708496,
      "rewards/rejected": -53.0453987121582,
      "step": 418
    },
    {
      "epoch": 0.9476957873904439,
      "grad_norm": 155.31797780611356,
      "learning_rate": 6.6070133386372906e-09,
      "logits/chosen": -0.951643168926239,
      "logits/rejected": -0.9600397348403931,
      "logps/chosen": -4.466057777404785,
      "logps/rejected": -5.151131629943848,
      "loss": 2.7637,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -44.66057586669922,
      "rewards/margins": 6.850742340087891,
      "rewards/rejected": -51.51131820678711,
      "step": 419
    },
    {
      "epoch": 0.9499575911789653,
      "grad_norm": 149.2139797209097,
      "learning_rate": 6.046400516665384e-09,
      "logits/chosen": -0.9598467946052551,
      "logits/rejected": -0.9552338123321533,
      "logps/chosen": -4.396927833557129,
      "logps/rejected": -5.164035320281982,
      "loss": 2.9147,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -43.969276428222656,
      "rewards/margins": 7.671075344085693,
      "rewards/rejected": -51.640357971191406,
      "step": 420
    },
    {
      "epoch": 0.9522193949674865,
      "grad_norm": 128.23107358922502,
      "learning_rate": 5.510457262353396e-09,
      "logits/chosen": -1.0141513347625732,
      "logits/rejected": -1.0092058181762695,
      "logps/chosen": -4.431485176086426,
      "logps/rejected": -5.144137382507324,
      "loss": 2.3305,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -44.314849853515625,
      "rewards/margins": 7.126527309417725,
      "rewards/rejected": -51.44137954711914,
      "step": 421
    },
    {
      "epoch": 0.9544811987560079,
      "grad_norm": 137.41821363865725,
      "learning_rate": 4.9992171367309265e-09,
      "logits/chosen": -0.9783574342727661,
      "logits/rejected": -0.9639812111854553,
      "logps/chosen": -4.1773505210876465,
      "logps/rejected": -4.8878397941589355,
      "loss": 2.2812,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -41.773502349853516,
      "rewards/margins": 7.104894161224365,
      "rewards/rejected": -48.878395080566406,
      "step": 422
    },
    {
      "epoch": 0.9567430025445293,
      "grad_norm": 150.80335007353597,
      "learning_rate": 4.5127121539052955e-09,
      "logits/chosen": -0.9814745187759399,
      "logits/rejected": -0.9915533661842346,
      "logps/chosen": -4.722474098205566,
      "logps/rejected": -5.471231937408447,
      "loss": 2.4137,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -47.224735260009766,
      "rewards/margins": 7.487585544586182,
      "rewards/rejected": -54.712318420410156,
      "step": 423
    },
    {
      "epoch": 0.9590048063330506,
      "grad_norm": 190.30249728923863,
      "learning_rate": 4.050972779057327e-09,
      "logits/chosen": -0.8814293146133423,
      "logits/rejected": -0.8948724865913391,
      "logps/chosen": -4.208849906921387,
      "logps/rejected": -4.926727294921875,
      "loss": 2.7246,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -42.0885009765625,
      "rewards/margins": 7.178774833679199,
      "rewards/rejected": -49.267276763916016,
      "step": 424
    },
    {
      "epoch": 0.961266610121572,
      "grad_norm": 170.25417436856634,
      "learning_rate": 3.6140279265330477e-09,
      "logits/chosen": -0.9410693645477295,
      "logits/rejected": -0.9508557319641113,
      "logps/chosen": -4.573657035827637,
      "logps/rejected": -5.300447463989258,
      "loss": 2.6723,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -45.736572265625,
      "rewards/margins": 7.267903804779053,
      "rewards/rejected": -53.004478454589844,
      "step": 425
    },
    {
      "epoch": 0.9635284139100933,
      "grad_norm": 153.10970515350425,
      "learning_rate": 3.2019049580335853e-09,
      "logits/chosen": -0.9724865555763245,
      "logits/rejected": -0.9711145162582397,
      "logps/chosen": -4.299941062927246,
      "logps/rejected": -4.872091770172119,
      "loss": 3.2617,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -42.99940872192383,
      "rewards/margins": 5.721508979797363,
      "rewards/rejected": -48.720916748046875,
      "step": 426
    },
    {
      "epoch": 0.9657902176986146,
      "grad_norm": 135.53392350882962,
      "learning_rate": 2.814629680901337e-09,
      "logits/chosen": -0.9825209379196167,
      "logits/rejected": -1.0047634840011597,
      "logps/chosen": -4.572093486785889,
      "logps/rejected": -5.286868095397949,
      "loss": 2.4462,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -45.7209358215332,
      "rewards/margins": 7.1477484703063965,
      "rewards/rejected": -52.86868667602539,
      "step": 427
    },
    {
      "epoch": 0.968052021487136,
      "grad_norm": 142.7322468520036,
      "learning_rate": 2.4522263465041937e-09,
      "logits/chosen": -0.9486875534057617,
      "logits/rejected": -0.9619259238243103,
      "logps/chosen": -4.454163551330566,
      "logps/rejected": -5.211050033569336,
      "loss": 2.1732,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -44.54163360595703,
      "rewards/margins": 7.5688652992248535,
      "rewards/rejected": -52.11050033569336,
      "step": 428
    },
    {
      "epoch": 0.9703138252756573,
      "grad_norm": 136.69603531498632,
      "learning_rate": 2.114717648716713e-09,
      "logits/chosen": -0.9177216291427612,
      "logits/rejected": -0.9293065667152405,
      "logps/chosen": -4.46873140335083,
      "logps/rejected": -5.2916765213012695,
      "loss": 2.2091,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -44.687313079833984,
      "rewards/margins": 8.22945499420166,
      "rewards/rejected": -52.91676712036133,
      "step": 429
    },
    {
      "epoch": 0.9725756290641787,
      "grad_norm": 152.57420599911407,
      "learning_rate": 1.802124722499121e-09,
      "logits/chosen": -0.9466343522071838,
      "logits/rejected": -0.9572174549102783,
      "logps/chosen": -4.5352349281311035,
      "logps/rejected": -5.298283100128174,
      "loss": 2.5129,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -45.352352142333984,
      "rewards/margins": 7.6304826736450195,
      "rewards/rejected": -52.98283386230469,
      "step": 430
    },
    {
      "epoch": 0.9748374328527001,
      "grad_norm": 148.47683038194563,
      "learning_rate": 1.5144671425737499e-09,
      "logits/chosen": -0.93849116563797,
      "logits/rejected": -0.9434796571731567,
      "logps/chosen": -4.241235256195068,
      "logps/rejected": -4.951510906219482,
      "loss": 2.8765,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -42.412353515625,
      "rewards/margins": 7.102751731872559,
      "rewards/rejected": -49.515106201171875,
      "step": 431
    },
    {
      "epoch": 0.9770992366412213,
      "grad_norm": 156.65455975752164,
      "learning_rate": 1.251762922199484e-09,
      "logits/chosen": -0.9012454748153687,
      "logits/rejected": -0.9075654149055481,
      "logps/chosen": -4.4999308586120605,
      "logps/rejected": -5.314447402954102,
      "loss": 1.9835,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -44.999305725097656,
      "rewards/margins": 8.14516544342041,
      "rewards/rejected": -53.144466400146484,
      "step": 432
    },
    {
      "epoch": 0.9793610404297427,
      "grad_norm": 142.53224570983375,
      "learning_rate": 1.0140285120433744e-09,
      "logits/chosen": -0.973222017288208,
      "logits/rejected": -0.9794344305992126,
      "logps/chosen": -4.608139991760254,
      "logps/rejected": -5.329331874847412,
      "loss": 2.527,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -46.081398010253906,
      "rewards/margins": 7.211921691894531,
      "rewards/rejected": -53.29331970214844,
      "step": 433
    },
    {
      "epoch": 0.9816228442182641,
      "grad_norm": 162.55094493344626,
      "learning_rate": 8.012787991508396e-10,
      "logits/chosen": -0.9416544437408447,
      "logits/rejected": -0.9537829160690308,
      "logps/chosen": -4.446515083312988,
      "logps/rejected": -5.29661226272583,
      "loss": 2.8152,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -44.465152740478516,
      "rewards/margins": 8.50097370147705,
      "rewards/rejected": -52.966129302978516,
      "step": 434
    },
    {
      "epoch": 0.9838846480067854,
      "grad_norm": 145.96078846455643,
      "learning_rate": 6.135271060133007e-10,
      "logits/chosen": -0.9080808758735657,
      "logits/rejected": -0.9170538187026978,
      "logps/chosen": -4.4007487297058105,
      "logps/rejected": -5.0969038009643555,
      "loss": 2.9644,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -44.00749206542969,
      "rewards/margins": 6.96155309677124,
      "rewards/rejected": -50.96904373168945,
      "step": 435
    },
    {
      "epoch": 0.9861464517953068,
      "grad_norm": 128.81354346980564,
      "learning_rate": 4.50785189733871e-10,
      "logits/chosen": -0.9219609498977661,
      "logits/rejected": -0.9404686689376831,
      "logps/chosen": -4.3535308837890625,
      "logps/rejected": -5.156708240509033,
      "loss": 2.1384,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -43.53531265258789,
      "rewards/margins": 8.031768798828125,
      "rewards/rejected": -51.56707763671875,
      "step": 436
    },
    {
      "epoch": 0.988408255583828,
      "grad_norm": 144.04083799644908,
      "learning_rate": 3.1306324129118935e-10,
      "logits/chosen": -0.9294695854187012,
      "logits/rejected": -0.9402070045471191,
      "logps/chosen": -4.544949531555176,
      "logps/rejected": -5.239452362060547,
      "loss": 2.6014,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -45.449493408203125,
      "rewards/margins": 6.945026874542236,
      "rewards/rejected": -52.39452362060547,
      "step": 437
    },
    {
      "epoch": 0.9906700593723494,
      "grad_norm": 155.89550905311364,
      "learning_rate": 2.003698849011748e-10,
      "logits/chosen": -0.9938583374023438,
      "logits/rejected": -1.0007983446121216,
      "logps/chosen": -4.665548801422119,
      "logps/rejected": -5.291666507720947,
      "loss": 2.7518,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -46.65549087524414,
      "rewards/margins": 6.261169910430908,
      "rewards/rejected": -52.91666030883789,
      "step": 438
    },
    {
      "epoch": 0.9929318631608708,
      "grad_norm": 144.03429193625178,
      "learning_rate": 1.1271217747714779e-10,
      "logits/chosen": -0.957481324672699,
      "logits/rejected": -0.9813300371170044,
      "logps/chosen": -4.5693769454956055,
      "logps/rejected": -5.249251842498779,
      "loss": 2.7764,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -45.693763732910156,
      "rewards/margins": 6.798752784729004,
      "rewards/rejected": -52.49251937866211,
      "step": 439
    },
    {
      "epoch": 0.9951936669493922,
      "grad_norm": 144.95200313917013,
      "learning_rate": 5.0095608187739055e-11,
      "logits/chosen": -0.9488322734832764,
      "logits/rejected": -0.9575316309928894,
      "logps/chosen": -4.304515361785889,
      "logps/rejected": -4.995253086090088,
      "loss": 2.7284,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -43.0451545715332,
      "rewards/margins": 6.907373428344727,
      "rewards/rejected": -49.9525260925293,
      "step": 440
    },
    {
      "epoch": 0.9974554707379135,
      "grad_norm": 171.65037355117758,
      "learning_rate": 1.2524098113209092e-11,
      "logits/chosen": -0.9583221077919006,
      "logits/rejected": -0.9742617607116699,
      "logps/chosen": -4.497997283935547,
      "logps/rejected": -5.199645042419434,
      "loss": 2.9133,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -44.979976654052734,
      "rewards/margins": 7.016475677490234,
      "rewards/rejected": -51.99645233154297,
      "step": 441
    },
    {
      "epoch": 0.9997172745264349,
      "grad_norm": 135.08490682698582,
      "learning_rate": 0.0,
      "logits/chosen": -0.9516457319259644,
      "logits/rejected": -0.9772913455963135,
      "logps/chosen": -4.331707000732422,
      "logps/rejected": -5.04747200012207,
      "loss": 2.838,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -43.317073822021484,
      "rewards/margins": 7.15764856338501,
      "rewards/rejected": -50.47472381591797,
      "step": 442
    },
    {
      "epoch": 0.9997172745264349,
      "eval_logits/chosen": -0.9504738450050354,
      "eval_logits/rejected": -0.9573480486869812,
      "eval_logps/chosen": -4.477466106414795,
      "eval_logps/rejected": -5.212347030639648,
      "eval_loss": 2.455941915512085,
      "eval_rewards/accuracies": 0.8001337051391602,
      "eval_rewards/chosen": -44.774662017822266,
      "eval_rewards/margins": 7.348804473876953,
      "eval_rewards/rejected": -52.12346649169922,
      "eval_runtime": 100.7219,
      "eval_samples_per_second": 29.576,
      "eval_steps_per_second": 1.857,
      "step": 442
    },
    {
      "epoch": 0.9997172745264349,
      "step": 442,
      "total_flos": 134366991482880.0,
      "train_loss": 3.275575773208929,
      "train_runtime": 7589.4389,
      "train_samples_per_second": 7.456,
      "train_steps_per_second": 0.058
    }
  ],
  "logging_steps": 1.0,
  "max_steps": 442,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 134366991482880.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}