{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.981333333333333,
  "eval_steps": 500,
  "global_step": 351,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.042666666666666665,
      "grad_norm": 1961.1512451171875,
      "learning_rate": 6.944444444444445e-06,
      "log_odds_chosen": 3.233767032623291,
      "log_odds_ratio": -7.3191938400268555,
      "logps/chosen": -23.426687240600586,
      "logps/rejected": -26.660680770874023,
      "loss": 425.3708,
      "nll_loss": 9.728386878967285,
      "po_loss": 3.5644500255584717,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": -11.713343620300293,
      "rewards/margins": 1.6169971227645874,
      "rewards/rejected": -13.330340385437012,
      "step": 5
    },
    {
      "epoch": 0.08533333333333333,
      "grad_norm": 500.5514831542969,
      "learning_rate": 1.388888888888889e-05,
      "log_odds_chosen": 3.315504550933838,
      "log_odds_ratio": -6.866879463195801,
      "logps/chosen": -23.438692092895508,
      "logps/rejected": -26.755935668945312,
      "loss": 393.0612,
      "nll_loss": 8.955938339233398,
      "po_loss": 3.3272223472595215,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -11.719346046447754,
      "rewards/margins": 1.658623456954956,
      "rewards/rejected": -13.377967834472656,
      "step": 10
    },
    {
      "epoch": 0.128,
      "grad_norm": 514.0573120117188,
      "learning_rate": 2.0833333333333336e-05,
      "log_odds_chosen": 7.634564399719238,
      "log_odds_ratio": -6.296060562133789,
      "logps/chosen": -22.5439395904541,
      "logps/rejected": -30.179019927978516,
      "loss": 378.1973,
      "nll_loss": 8.766839981079102,
      "po_loss": 3.0518250465393066,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -11.27196979522705,
      "rewards/margins": 3.8175387382507324,
      "rewards/rejected": -15.089509963989258,
      "step": 15
    },
    {
      "epoch": 0.17066666666666666,
      "grad_norm": 3820.703125,
      "learning_rate": 2.777777777777778e-05,
      "log_odds_chosen": -1.3400392532348633,
      "log_odds_ratio": -8.175825119018555,
      "logps/chosen": -18.689817428588867,
      "logps/rejected": -17.35008430480957,
      "loss": 363.51,
      "nll_loss": 7.390625953674316,
      "po_loss": 3.9690635204315186,
      "rewards/accuracies": 0.484375,
      "rewards/chosen": -9.344908714294434,
      "rewards/margins": -0.6698670983314514,
      "rewards/rejected": -8.675042152404785,
      "step": 20
    },
    {
      "epoch": 0.21333333333333335,
      "grad_norm": 417.3852844238281,
      "learning_rate": 3.472222222222222e-05,
      "log_odds_chosen": -0.37303870916366577,
      "log_odds_ratio": -2.343538284301758,
      "logps/chosen": -5.883327484130859,
      "logps/rejected": -5.492794036865234,
      "loss": 134.8965,
      "nll_loss": 3.241476535797119,
      "po_loss": 0.9740395545959473,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -2.9416637420654297,
      "rewards/margins": -0.19526681303977966,
      "rewards/rejected": -2.746397018432617,
      "step": 25
    },
    {
      "epoch": 0.256,
      "grad_norm": 330.9832458496094,
      "learning_rate": 4.166666666666667e-05,
      "log_odds_chosen": 0.3532889187335968,
      "log_odds_ratio": -0.7218837141990662,
      "logps/chosen": -1.7468492984771729,
      "logps/rejected": -2.0471103191375732,
      "loss": 70.938,
      "nll_loss": 2.1013693809509277,
      "po_loss": 0.11544491350650787,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8734246492385864,
      "rewards/margins": 0.15013054013252258,
      "rewards/rejected": -1.0235551595687866,
      "step": 30
    },
    {
      "epoch": 0.2986666666666667,
      "grad_norm": 284.9355163574219,
      "learning_rate": 4.8611111111111115e-05,
      "log_odds_chosen": 0.4371984004974365,
      "log_odds_ratio": -0.6356752514839172,
      "logps/chosen": -1.627811074256897,
      "logps/rejected": -1.993390679359436,
      "loss": 63.8055,
      "nll_loss": 1.9257453680038452,
      "po_loss": 0.06817711889743805,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.8139055371284485,
      "rewards/margins": 0.18278983235359192,
      "rewards/rejected": -0.996695339679718,
      "step": 35
    },
    {
      "epoch": 0.3413333333333333,
      "grad_norm": 126.80204772949219,
      "learning_rate": 4.998010925565448e-05,
      "log_odds_chosen": 0.36739838123321533,
      "log_odds_ratio": -0.6945838928222656,
      "logps/chosen": -1.696966528892517,
      "logps/rejected": -2.019742488861084,
      "loss": 66.0958,
      "nll_loss": 1.972928762435913,
      "po_loss": 0.0925649031996727,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.8484832644462585,
      "rewards/margins": 0.16138817369937897,
      "rewards/rejected": -1.009871244430542,
      "step": 40
    },
    {
      "epoch": 0.384,
      "grad_norm": 734.975830078125,
      "learning_rate": 4.989935734988098e-05,
      "log_odds_chosen": 0.2258051633834839,
      "log_odds_ratio": -0.6970190405845642,
      "logps/chosen": -1.5770127773284912,
      "logps/rejected": -1.7601944208145142,
      "loss": 61.9986,
      "nll_loss": 1.8443195819854736,
      "po_loss": 0.09313799440860748,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.7885063886642456,
      "rewards/margins": 0.09159073978662491,
      "rewards/rejected": -0.8800972104072571,
      "step": 45
    },
    {
      "epoch": 0.4266666666666667,
      "grad_norm": 98.6443862915039,
      "learning_rate": 4.975670171853926e-05,
      "log_odds_chosen": 0.3061736226081848,
      "log_odds_ratio": -0.6558084487915039,
      "logps/chosen": -1.410742998123169,
      "logps/rejected": -1.6612850427627563,
      "loss": 56.1342,
      "nll_loss": 1.6897376775741577,
      "po_loss": 0.06445512920618057,
      "rewards/accuracies": 0.6343749761581421,
      "rewards/chosen": -0.7053714990615845,
      "rewards/margins": 0.1252710521221161,
      "rewards/rejected": -0.8306425213813782,
      "step": 50
    },
    {
      "epoch": 0.4693333333333333,
      "grad_norm": 138.6815948486328,
      "learning_rate": 4.9552497026005974e-05,
      "log_odds_chosen": 0.21871152520179749,
      "log_odds_ratio": -0.7383303046226501,
      "logps/chosen": -1.4634255170822144,
      "logps/rejected": -1.6147739887237549,
      "loss": 59.4312,
      "nll_loss": 1.7589048147201538,
      "po_loss": 0.09831906855106354,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.7317127585411072,
      "rewards/margins": 0.07567427307367325,
      "rewards/rejected": -0.8073869943618774,
      "step": 55
    },
    {
      "epoch": 0.512,
      "grad_norm": 102.89739227294922,
      "learning_rate": 4.928725095732169e-05,
      "log_odds_chosen": 0.401327520608902,
      "log_odds_ratio": -0.6062948107719421,
      "logps/chosen": -1.2786072492599487,
      "logps/rejected": -1.585089921951294,
      "loss": 51.5199,
      "nll_loss": 1.5874063968658447,
      "po_loss": 0.02259092405438423,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.6393036246299744,
      "rewards/margins": 0.15324126183986664,
      "rewards/rejected": -0.792544960975647,
      "step": 60
    },
    {
      "epoch": 0.5546666666666666,
      "grad_norm": 100.74688720703125,
      "learning_rate": 4.896162295600589e-05,
      "log_odds_chosen": 0.33866989612579346,
      "log_odds_ratio": -0.6249087452888489,
      "logps/chosen": -1.3379658460617065,
      "logps/rejected": -1.6066213846206665,
      "loss": 53.1437,
      "nll_loss": 1.6211912631988525,
      "po_loss": 0.03954792767763138,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6689829230308533,
      "rewards/margins": 0.13432787358760834,
      "rewards/rejected": -0.8033106923103333,
      "step": 65
    },
    {
      "epoch": 0.5973333333333334,
      "grad_norm": 107.1102523803711,
      "learning_rate": 4.8576422584576514e-05,
      "log_odds_chosen": 0.28569236397743225,
      "log_odds_ratio": -0.6388062238693237,
      "logps/chosen": -1.2627140283584595,
      "logps/rejected": -1.478611707687378,
      "loss": 51.3954,
      "nll_loss": 1.5717823505401611,
      "po_loss": 0.034325726330280304,
      "rewards/accuracies": 0.628125011920929,
      "rewards/chosen": -0.6313570141792297,
      "rewards/margins": 0.10794880241155624,
      "rewards/rejected": -0.739305853843689,
      "step": 70
    },
    {
      "epoch": 0.64,
      "grad_norm": 87.37519073486328,
      "learning_rate": 4.813260751184992e-05,
      "log_odds_chosen": 0.4192368984222412,
      "log_odds_ratio": -0.6039353609085083,
      "logps/chosen": -1.2572083473205566,
      "logps/rejected": -1.5851625204086304,
      "loss": 50.7092,
      "nll_loss": 1.5785938501358032,
      "po_loss": 0.00606974633410573,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.6286041736602783,
      "rewards/margins": 0.1639770269393921,
      "rewards/rejected": -0.7925812602043152,
      "step": 75
    },
    {
      "epoch": 0.6826666666666666,
      "grad_norm": 84.41800689697266,
      "learning_rate": 4.763128113202537e-05,
      "log_odds_chosen": 0.3978249132633209,
      "log_odds_ratio": -0.6257617473602295,
      "logps/chosen": -1.1973122358322144,
      "logps/rejected": -1.5119242668151855,
      "loss": 49.6894,
      "nll_loss": 1.5280753374099731,
      "po_loss": 0.024718867614865303,
      "rewards/accuracies": 0.6031249761581421,
      "rewards/chosen": -0.5986561179161072,
      "rewards/margins": 0.1573060303926468,
      "rewards/rejected": -0.7559621334075928,
      "step": 80
    },
    {
      "epoch": 0.7253333333333334,
      "grad_norm": 217.4127655029297,
      "learning_rate": 4.707368982147318e-05,
      "log_odds_chosen": 0.9764216542243958,
      "log_odds_ratio": -0.5953387022018433,
      "logps/chosen": -1.2593096494674683,
      "logps/rejected": -2.1358559131622314,
      "loss": 50.7417,
      "nll_loss": 1.5718034505844116,
      "po_loss": 0.013875389471650124,
      "rewards/accuracies": 0.628125011920929,
      "rewards/chosen": -0.6296548247337341,
      "rewards/margins": 0.43827304244041443,
      "rewards/rejected": -1.0679279565811157,
      "step": 85
    },
    {
      "epoch": 0.768,
      "grad_norm": 74.66828918457031,
      "learning_rate": 4.6461219840046654e-05,
      "log_odds_chosen": 0.7273199558258057,
      "log_odds_ratio": -0.6018415093421936,
      "logps/chosen": -1.213512897491455,
      "logps/rejected": -1.8320105075836182,
      "loss": 48.283,
      "nll_loss": 1.5248548984527588,
      "po_loss": -0.016011018306016922,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.6067564487457275,
      "rewards/margins": 0.30924877524375916,
      "rewards/rejected": -0.9160052537918091,
      "step": 90
    },
    {
      "epoch": 0.8106666666666666,
      "grad_norm": 96.17576599121094,
      "learning_rate": 4.579539388462173e-05,
      "log_odds_chosen": 0.968981921672821,
      "log_odds_ratio": -0.5852179527282715,
      "logps/chosen": -1.1610548496246338,
      "logps/rejected": -2.0138120651245117,
      "loss": 46.2603,
      "nll_loss": 1.5272886753082275,
      "po_loss": -0.08165426552295685,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.5805274248123169,
      "rewards/margins": 0.4263785481452942,
      "rewards/rejected": -1.0069060325622559,
      "step": 95
    },
    {
      "epoch": 0.8533333333333334,
      "grad_norm": 1037.95703125,
      "learning_rate": 4.5077867303432546e-05,
      "log_odds_chosen": 2.305642604827881,
      "log_odds_ratio": -0.6378474831581116,
      "logps/chosen": -1.2965327501296997,
      "logps/rejected": -3.484023332595825,
      "loss": 40.3825,
      "nll_loss": 1.5268816947937012,
      "po_loss": -0.2649289071559906,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.6482663750648499,
      "rewards/margins": 1.093745231628418,
      "rewards/rejected": -1.7420116662979126,
      "step": 100
    },
    {
      "epoch": 0.896,
      "grad_norm": 120.91621398925781,
      "learning_rate": 4.431042398061499e-05,
      "log_odds_chosen": 2.5239200592041016,
      "log_odds_ratio": -0.7032963633537292,
      "logps/chosen": -2.863506317138672,
      "logps/rejected": -5.296839714050293,
      "loss": 85.1499,
      "nll_loss": 2.635256767272949,
      "po_loss": 0.025676274672150612,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.431753158569336,
      "rewards/margins": 1.2166664600372314,
      "rewards/rejected": -2.6484198570251465,
      "step": 105
    },
    {
      "epoch": 0.9386666666666666,
      "grad_norm": 80.25611114501953,
      "learning_rate": 4.34949719011896e-05,
      "log_odds_chosen": 0.859009861946106,
      "log_odds_ratio": -0.5764500498771667,
      "logps/chosen": -1.229062557220459,
      "logps/rejected": -1.9831807613372803,
      "loss": 47.4796,
      "nll_loss": 1.5179859399795532,
      "po_loss": -0.03424710035324097,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.6145312786102295,
      "rewards/margins": 0.37705910205841064,
      "rewards/rejected": -0.9915903806686401,
      "step": 110
    },
    {
      "epoch": 0.9813333333333333,
      "grad_norm": 271.3537902832031,
      "learning_rate": 4.263353840751022e-05,
      "log_odds_chosen": 1.4788315296173096,
      "log_odds_ratio": -0.5846825242042542,
      "logps/chosen": -1.1537017822265625,
      "logps/rejected": -2.52790904045105,
      "loss": 44.319,
      "nll_loss": 1.4921709299087524,
      "po_loss": -0.10720314830541611,
      "rewards/accuracies": 0.6781250238418579,
      "rewards/chosen": -0.5768508911132812,
      "rewards/margins": 0.6871034502983093,
      "rewards/rejected": -1.263954520225525,
      "step": 115
    },
    {
      "epoch": 1.0170666666666666,
      "grad_norm": 146.54196166992188,
      "learning_rate": 4.172826515897146e-05,
      "log_odds_chosen": 2.831665515899658,
      "log_odds_ratio": -0.4861010015010834,
      "logps/chosen": -1.089508056640625,
      "logps/rejected": -3.7528676986694336,
      "loss": 33.7801,
      "nll_loss": 1.4263149499893188,
      "po_loss": -0.1658623069524765,
      "rewards/accuracies": 0.7723880410194397,
      "rewards/chosen": -0.5447540283203125,
      "rewards/margins": 1.3316797018051147,
      "rewards/rejected": -1.8764338493347168,
      "step": 120
    },
    {
      "epoch": 1.0597333333333334,
      "grad_norm": 1590.6824951171875,
      "learning_rate": 4.078140280750597e-05,
      "log_odds_chosen": 4.709661483764648,
      "log_odds_ratio": -0.41537848114967346,
      "logps/chosen": -1.5830024480819702,
      "logps/rejected": -6.075448036193848,
      "loss": 46.6655,
      "nll_loss": 1.6417958736419678,
      "po_loss": -0.18349842727184296,
      "rewards/accuracies": 0.8531249761581421,
      "rewards/chosen": -0.7915012240409851,
      "rewards/margins": 2.246222496032715,
      "rewards/rejected": -3.037724018096924,
      "step": 125
    },
    {
      "epoch": 1.1024,
      "grad_norm": 321.93255615234375,
      "learning_rate": 3.9795305402109195e-05,
      "log_odds_chosen": 3.6606578826904297,
      "log_odds_ratio": -0.4261302947998047,
      "logps/chosen": -1.2628165483474731,
      "logps/rejected": -4.708868980407715,
      "loss": 36.688,
      "nll_loss": 1.4487800598144531,
      "po_loss": -0.3022807240486145,
      "rewards/accuracies": 0.8218749761581421,
      "rewards/chosen": -0.6314082741737366,
      "rewards/margins": 1.7230265140533447,
      "rewards/rejected": -2.3544344902038574,
      "step": 130
    },
    {
      "epoch": 1.1450666666666667,
      "grad_norm": 156.33656311035156,
      "learning_rate": 3.8772424536302564e-05,
      "log_odds_chosen": 3.3216488361358643,
      "log_odds_ratio": -0.45871955156326294,
      "logps/chosen": -1.054512619972229,
      "logps/rejected": -4.166798114776611,
      "loss": 37.5147,
      "nll_loss": 1.3529837131500244,
      "po_loss": -0.18065081536769867,
      "rewards/accuracies": 0.778124988079071,
      "rewards/chosen": -0.5272563099861145,
      "rewards/margins": 1.5561425685882568,
      "rewards/rejected": -2.0833990573883057,
      "step": 135
    },
    {
      "epoch": 1.1877333333333333,
      "grad_norm": 126.58124542236328,
      "learning_rate": 3.771530325308579e-05,
      "log_odds_chosen": 3.002664089202881,
      "log_odds_ratio": -0.3991411030292511,
      "logps/chosen": -0.99451744556427,
      "logps/rejected": -3.7320716381073,
      "loss": 33.4168,
      "nll_loss": 1.2971866130828857,
      "po_loss": -0.2529115378856659,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.497258722782135,
      "rewards/margins": 1.3687771558761597,
      "rewards/rejected": -1.86603581905365,
      "step": 140
    },
    {
      "epoch": 1.2304,
      "grad_norm": 144.6908416748047,
      "learning_rate": 3.662656972253127e-05,
      "log_odds_chosen": 3.620790958404541,
      "log_odds_ratio": -0.39333289861679077,
      "logps/chosen": -1.1291520595550537,
      "logps/rejected": -4.468079090118408,
      "loss": 31.6104,
      "nll_loss": 1.2871015071868896,
      "po_loss": -0.2992752194404602,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5645760297775269,
      "rewards/margins": 1.6694637537002563,
      "rewards/rejected": -2.234039545059204,
      "step": 145
    },
    {
      "epoch": 1.2730666666666668,
      "grad_norm": 317.3659362792969,
      "learning_rate": 3.550893070773914e-05,
      "log_odds_chosen": 2.5045361518859863,
      "log_odds_ratio": -0.5062359571456909,
      "logps/chosen": -1.01883864402771,
      "logps/rejected": -3.3051059246063232,
      "loss": 35.2667,
      "nll_loss": 1.3030840158462524,
      "po_loss": -0.20100060105323792,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.509419322013855,
      "rewards/margins": 1.143133521080017,
      "rewards/rejected": -1.6525529623031616,
      "step": 150
    },
    {
      "epoch": 1.3157333333333332,
      "grad_norm": 99.71307373046875,
      "learning_rate": 3.436516483539781e-05,
      "log_odds_chosen": 2.3835015296936035,
      "log_odds_ratio": -0.4490797519683838,
      "logps/chosen": -0.961607813835144,
      "logps/rejected": -3.094728946685791,
      "loss": 36.1188,
      "nll_loss": 1.2786829471588135,
      "po_loss": -0.1499709188938141,
      "rewards/accuracies": 0.8343750238418579,
      "rewards/chosen": -0.480803906917572,
      "rewards/margins": 1.0665608644485474,
      "rewards/rejected": -1.5473644733428955,
      "step": 155
    },
    {
      "epoch": 1.3584,
      "grad_norm": 176.96144104003906,
      "learning_rate": 3.3198115687680115e-05,
      "log_odds_chosen": 2.804508924484253,
      "log_odds_ratio": -0.4551823139190674,
      "logps/chosen": -0.9649540781974792,
      "logps/rejected": -3.5283005237579346,
      "loss": 31.9084,
      "nll_loss": 1.2769290208816528,
      "po_loss": -0.2797916531562805,
      "rewards/accuracies": 0.7906249761581421,
      "rewards/chosen": -0.4824770390987396,
      "rewards/margins": 1.2816731929779053,
      "rewards/rejected": -1.7641502618789673,
      "step": 160
    },
    {
      "epoch": 1.4010666666666667,
      "grad_norm": 86.9504165649414,
      "learning_rate": 3.201068473265007e-05,
      "log_odds_chosen": 4.039158821105957,
      "log_odds_ratio": -0.41200175881385803,
      "logps/chosen": -1.1530981063842773,
      "logps/rejected": -4.936544418334961,
      "loss": 33.9241,
      "nll_loss": 1.339659571647644,
      "po_loss": -0.27953004837036133,
      "rewards/accuracies": 0.840624988079071,
      "rewards/chosen": -0.5765490531921387,
      "rewards/margins": 1.8917229175567627,
      "rewards/rejected": -2.4682722091674805,
      "step": 165
    },
    {
      "epoch": 1.4437333333333333,
      "grad_norm": 67.8962631225586,
      "learning_rate": 3.0805824110756064e-05,
      "log_odds_chosen": 2.8494575023651123,
      "log_odds_ratio": -0.4309585690498352,
      "logps/chosen": -1.0003385543823242,
      "logps/rejected": -3.5997538566589355,
      "loss": 32.9378,
      "nll_loss": 1.320733904838562,
      "po_loss": -0.2914286255836487,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.5001692771911621,
      "rewards/margins": 1.2997074127197266,
      "rewards/rejected": -1.7998769283294678,
      "step": 170
    },
    {
      "epoch": 1.4864,
      "grad_norm": 332.5802307128906,
      "learning_rate": 2.958652929534456e-05,
      "log_odds_chosen": 4.219790458679199,
      "log_odds_ratio": -0.3872655928134918,
      "logps/chosen": -0.9488153457641602,
      "logps/rejected": -4.870639324188232,
      "loss": 24.5,
      "nll_loss": 1.2430005073547363,
      "po_loss": -0.47737568616867065,
      "rewards/accuracies": 0.8656250238418579,
      "rewards/chosen": -0.4744076728820801,
      "rewards/margins": 1.9609121084213257,
      "rewards/rejected": -2.435319662094116,
      "step": 175
    },
    {
      "epoch": 1.5290666666666666,
      "grad_norm": 120.37972259521484,
      "learning_rate": 2.8355831645441388e-05,
      "log_odds_chosen": 3.8075263500213623,
      "log_odds_ratio": -0.4285067617893219,
      "logps/chosen": -1.0132713317871094,
      "logps/rejected": -4.545207500457764,
      "loss": 30.9088,
      "nll_loss": 1.2752387523651123,
      "po_loss": -0.3093392252922058,
      "rewards/accuracies": 0.840624988079071,
      "rewards/chosen": -0.5066356658935547,
      "rewards/margins": 1.7659679651260376,
      "rewards/rejected": -2.272603750228882,
      "step": 180
    },
    {
      "epoch": 1.5717333333333334,
      "grad_norm": 74.23609924316406,
      "learning_rate": 2.7116790869315582e-05,
      "log_odds_chosen": 3.5458972454071045,
      "log_odds_ratio": -0.3724256157875061,
      "logps/chosen": -0.9797577857971191,
      "logps/rejected": -4.209129810333252,
      "loss": 27.577,
      "nll_loss": 1.2894318103790283,
      "po_loss": -0.42764997482299805,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.48987889289855957,
      "rewards/margins": 1.6146857738494873,
      "rewards/rejected": -2.104564905166626,
      "step": 185
    },
    {
      "epoch": 1.6143999999999998,
      "grad_norm": 1912.4134521484375,
      "learning_rate": 2.587248741756253e-05,
      "log_odds_chosen": 4.569626808166504,
      "log_odds_ratio": -0.6492961049079895,
      "logps/chosen": -1.6978082656860352,
      "logps/rejected": -5.967839241027832,
      "loss": 47.8645,
      "nll_loss": 1.7877342700958252,
      "po_loss": -0.2919684648513794,
      "rewards/accuracies": 0.840624988079071,
      "rewards/chosen": -0.8489041328430176,
      "rewards/margins": 2.1350154876708984,
      "rewards/rejected": -2.983919620513916,
      "step": 190
    },
    {
      "epoch": 1.6570666666666667,
      "grad_norm": 246.9727020263672,
      "learning_rate": 2.4626014824618415e-05,
      "log_odds_chosen": 3.452259063720703,
      "log_odds_ratio": -0.4340329170227051,
      "logps/chosen": -1.348034143447876,
      "logps/rejected": -4.516729831695557,
      "loss": 39.5128,
      "nll_loss": 1.5523712635040283,
      "po_loss": -0.3175976276397705,
      "rewards/accuracies": 0.8218749761581421,
      "rewards/chosen": -0.674017071723938,
      "rewards/margins": 1.5843479633331299,
      "rewards/rejected": -2.2583649158477783,
      "step": 195
    },
    {
      "epoch": 1.6997333333333333,
      "grad_norm": 249.69097900390625,
      "learning_rate": 2.3380472017746202e-05,
      "log_odds_chosen": 4.080594539642334,
      "log_odds_ratio": -0.41809386014938354,
      "logps/chosen": -1.1225640773773193,
      "logps/rejected": -4.919617652893066,
      "loss": 22.4016,
      "nll_loss": 1.3972797393798828,
      "po_loss": -0.6972296237945557,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.5612820386886597,
      "rewards/margins": 1.8985265493392944,
      "rewards/rejected": -2.459808826446533,
      "step": 200
    },
    {
      "epoch": 1.7424,
      "grad_norm": 70.95199584960938,
      "learning_rate": 2.2138955612614207e-05,
      "log_odds_chosen": 3.4808452129364014,
      "log_odds_ratio": -0.4113942086696625,
      "logps/chosen": -0.9912225604057312,
      "logps/rejected": -4.186377048492432,
      "loss": 30.6187,
      "nll_loss": 1.2635384798049927,
      "po_loss": -0.30670270323753357,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.4956112802028656,
      "rewards/margins": 1.5975773334503174,
      "rewards/rejected": -2.093188524246216,
      "step": 205
    },
    {
      "epoch": 1.7850666666666668,
      "grad_norm": 77.6159896850586,
      "learning_rate": 2.090455221462156e-05,
      "log_odds_chosen": 3.126645803451538,
      "log_odds_ratio": -0.4194856286048889,
      "logps/chosen": -0.976054847240448,
      "logps/rejected": -3.822575807571411,
      "loss": 30.5635,
      "nll_loss": 1.2749359607696533,
      "po_loss": -0.319825679063797,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.488027423620224,
      "rewards/margins": 1.4232604503631592,
      "rewards/rejected": -1.9112879037857056,
      "step": 210
    },
    {
      "epoch": 1.8277333333333332,
      "grad_norm": 191.34779357910156,
      "learning_rate": 1.9680330745110954e-05,
      "log_odds_chosen": 4.950188636779785,
      "log_odds_ratio": -0.3956086039543152,
      "logps/chosen": -0.9748937487602234,
      "logps/rejected": -5.643929481506348,
      "loss": 23.1695,
      "nll_loss": 1.2857718467712402,
      "po_loss": -0.5617233514785767,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.4874468743801117,
      "rewards/margins": 2.33451771736145,
      "rewards/rejected": -2.821964740753174,
      "step": 215
    },
    {
      "epoch": 1.8704,
      "grad_norm": 327.7476501464844,
      "learning_rate": 1.8469334811546542e-05,
      "log_odds_chosen": 4.540267467498779,
      "log_odds_ratio": -0.39753258228302,
      "logps/chosen": -0.9387453198432922,
      "logps/rejected": -5.1981964111328125,
      "loss": 28.4212,
      "nll_loss": 1.252907633781433,
      "po_loss": -0.36474356055259705,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.4693726599216461,
      "rewards/margins": 2.129725217819214,
      "rewards/rejected": -2.5990982055664062,
      "step": 220
    },
    {
      "epoch": 1.9130666666666667,
      "grad_norm": 540.2483520507812,
      "learning_rate": 1.7274575140626318e-05,
      "log_odds_chosen": 5.64566707611084,
      "log_odds_ratio": -0.4446595311164856,
      "logps/chosen": -1.22698974609375,
      "logps/rejected": -6.585175514221191,
      "loss": 34.7211,
      "nll_loss": 1.3462177515029907,
      "po_loss": -0.26118165254592896,
      "rewards/accuracies": 0.8656250238418579,
      "rewards/chosen": -0.613494873046875,
      "rewards/margins": 2.6790928840637207,
      "rewards/rejected": -3.2925877571105957,
      "step": 225
    },
    {
      "epoch": 1.9557333333333333,
      "grad_norm": 401.98248291015625,
      "learning_rate": 1.609902209314108e-05,
      "log_odds_chosen": 4.817292213439941,
      "log_odds_ratio": -0.42455047369003296,
      "logps/chosen": -1.0640500783920288,
      "logps/rejected": -5.626768589019775,
      "loss": 31.062,
      "nll_loss": 1.342284917831421,
      "po_loss": -0.37159785628318787,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.5320250391960144,
      "rewards/margins": 2.2813591957092285,
      "rewards/rejected": -2.8133842945098877,
      "step": 230
    },
    {
      "epoch": 1.9984,
      "grad_norm": 91.74356079101562,
      "learning_rate": 1.4945598279189565e-05,
      "log_odds_chosen": 5.049031734466553,
      "log_odds_ratio": -0.406465619802475,
      "logps/chosen": -0.9556134939193726,
      "logps/rejected": -5.697612762451172,
      "loss": 25.4086,
      "nll_loss": 1.2838685512542725,
      "po_loss": -0.48984843492507935,
      "rewards/accuracies": 0.840624988079071,
      "rewards/chosen": -0.4778067469596863,
      "rewards/margins": 2.370999813079834,
      "rewards/rejected": -2.848806381225586,
      "step": 235
    },
    {
      "epoch": 2.034133333333333,
      "grad_norm": 84.14781188964844,
      "learning_rate": 1.3817171292109183e-05,
      "log_odds_chosen": 4.798027992248535,
      "log_odds_ratio": -0.3244837820529938,
      "logps/chosen": -0.8924720287322998,
      "logps/rejected": -5.292112827301025,
      "loss": 14.7486,
      "nll_loss": 1.1619552373886108,
      "po_loss": -0.6116347908973694,
      "rewards/accuracies": 0.9365671873092651,
      "rewards/chosen": -0.4462360143661499,
      "rewards/margins": 2.1998205184936523,
      "rewards/rejected": -2.6460564136505127,
      "step": 240
    },
    {
      "epoch": 2.0768,
      "grad_norm": 265.51214599609375,
      "learning_rate": 1.271654657918722e-05,
      "log_odds_chosen": 3.9462997913360596,
      "log_odds_ratio": -0.29830431938171387,
      "logps/chosen": -0.8560145497322083,
      "logps/rejected": -4.366959571838379,
      "loss": 21.2586,
      "nll_loss": 1.1650712490081787,
      "po_loss": -0.5007405281066895,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.4280072748661041,
      "rewards/margins": 1.7554724216461182,
      "rewards/rejected": -2.1834797859191895,
      "step": 245
    },
    {
      "epoch": 2.119466666666667,
      "grad_norm": 189.54098510742188,
      "learning_rate": 1.1646460466876783e-05,
      "log_odds_chosen": 5.587141036987305,
      "log_odds_ratio": -0.23089304566383362,
      "logps/chosen": -0.941892147064209,
      "logps/rejected": -6.023226737976074,
      "loss": 15.3802,
      "nll_loss": 1.1797395944595337,
      "po_loss": -0.6991080641746521,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.4709460735321045,
      "rewards/margins": 2.5406672954559326,
      "rewards/rejected": -3.011613368988037,
      "step": 250
    },
    {
      "epoch": 2.1621333333333332,
      "grad_norm": 198.67486572265625,
      "learning_rate": 1.0609573357858166e-05,
      "log_odds_chosen": 6.2472615242004395,
      "log_odds_ratio": -0.2708757519721985,
      "logps/chosen": -1.0158169269561768,
      "logps/rejected": -6.834556579589844,
      "loss": 14.1929,
      "nll_loss": 1.237670660018921,
      "po_loss": -0.7941441535949707,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.5079084634780884,
      "rewards/margins": 2.909369945526123,
      "rewards/rejected": -3.417278289794922,
      "step": 255
    },
    {
      "epoch": 2.2048,
      "grad_norm": 758.944580078125,
      "learning_rate": 9.608463116858542e-06,
      "log_odds_chosen": 6.66796350479126,
      "log_odds_ratio": -0.23688967525959015,
      "logps/chosen": -0.9843616485595703,
      "logps/rejected": -7.208956718444824,
      "loss": 11.9915,
      "nll_loss": 1.2095975875854492,
      "po_loss": -0.8348617553710938,
      "rewards/accuracies": 0.9593750238418579,
      "rewards/chosen": -0.49218082427978516,
      "rewards/margins": 3.112297534942627,
      "rewards/rejected": -3.604478359222412,
      "step": 260
    },
    {
      "epoch": 2.2474666666666665,
      "grad_norm": 504.9031982421875,
      "learning_rate": 8.645618661674142e-06,
      "log_odds_chosen": 6.688530921936035,
      "log_odds_ratio": -0.2643258273601532,
      "logps/chosen": -1.105387806892395,
      "logps/rejected": -7.340773582458496,
      "loss": 11.084,
      "nll_loss": 1.2117825746536255,
      "po_loss": -0.8654062151908875,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": -0.5526939034461975,
      "rewards/margins": 3.1176929473876953,
      "rewards/rejected": -3.670386791229248,
      "step": 265
    },
    {
      "epoch": 2.2901333333333334,
      "grad_norm": 250.27699279785156,
      "learning_rate": 7.723433775328384e-06,
      "log_odds_chosen": 6.858447074890137,
      "log_odds_ratio": -0.31602945923805237,
      "logps/chosen": -1.1505261659622192,
      "logps/rejected": -7.580657005310059,
      "loss": 20.3187,
      "nll_loss": 1.2361539602279663,
      "po_loss": -0.6011937260627747,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.5752630829811096,
      "rewards/margins": 3.2150657176971436,
      "rewards/rejected": -3.7903285026550293,
      "step": 270
    },
    {
      "epoch": 2.3327999999999998,
      "grad_norm": 3240.22802734375,
      "learning_rate": 6.844201154750177e-06,
      "log_odds_chosen": 6.9127960205078125,
      "log_odds_ratio": -0.28268226981163025,
      "logps/chosen": -1.3622684478759766,
      "logps/rejected": -7.855978965759277,
      "loss": 22.8393,
      "nll_loss": 1.4035627841949463,
      "po_loss": -0.6898348927497864,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.6811342239379883,
      "rewards/margins": 3.2468554973602295,
      "rewards/rejected": -3.9279894828796387,
      "step": 275
    },
    {
      "epoch": 2.3754666666666666,
      "grad_norm": 650.8209228515625,
      "learning_rate": 6.010106710768052e-06,
      "log_odds_chosen": 6.493264675140381,
      "log_odds_ratio": -0.34161344170570374,
      "logps/chosen": -1.386515498161316,
      "logps/rejected": -7.452689170837402,
      "loss": 19.5164,
      "nll_loss": 1.3803019523620605,
      "po_loss": -0.7704136967658997,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.693257749080658,
      "rewards/margins": 3.0330867767333984,
      "rewards/rejected": -3.726344585418701,
      "step": 280
    },
    {
      "epoch": 2.4181333333333335,
      "grad_norm": 507.415283203125,
      "learning_rate": 5.223224133591476e-06,
      "log_odds_chosen": 6.825043678283691,
      "log_odds_ratio": -0.2423882931470871,
      "logps/chosen": -1.1171207427978516,
      "logps/rejected": -7.483119964599609,
      "loss": 19.53,
      "nll_loss": 1.2847764492034912,
      "po_loss": -0.6744629144668579,
      "rewards/accuracies": 0.9468749761581421,
      "rewards/chosen": -0.5585603713989258,
      "rewards/margins": 3.1829993724823,
      "rewards/rejected": -3.7415599822998047,
      "step": 285
    },
    {
      "epoch": 2.4608,
      "grad_norm": 789.3963623046875,
      "learning_rate": 4.4855097372902135e-06,
      "log_odds_chosen": 7.850038051605225,
      "log_odds_ratio": -0.26526254415512085,
      "logps/chosen": -1.4054704904556274,
      "logps/rejected": -8.81872844696045,
      "loss": 18.6059,
      "nll_loss": 1.3957431316375732,
      "po_loss": -0.8143098950386047,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.7027352452278137,
      "rewards/margins": 3.7066292762756348,
      "rewards/rejected": -4.409364223480225,
      "step": 290
    },
    {
      "epoch": 2.5034666666666667,
      "grad_norm": 1187.58837890625,
      "learning_rate": 3.798797596089351e-06,
      "log_odds_chosen": 8.998720169067383,
      "log_odds_ratio": -0.218130424618721,
      "logps/chosen": -1.247434377670288,
      "logps/rejected": -9.765419960021973,
      "loss": 10.3413,
      "nll_loss": 1.355464220046997,
      "po_loss": -1.032299280166626,
      "rewards/accuracies": 0.9593750238418579,
      "rewards/chosen": -0.623717188835144,
      "rewards/margins": 4.2589921951293945,
      "rewards/rejected": -4.882709980010986,
      "step": 295
    },
    {
      "epoch": 2.5461333333333336,
      "grad_norm": 593.267578125,
      "learning_rate": 3.164794984571759e-06,
      "log_odds_chosen": 6.977963924407959,
      "log_odds_ratio": -0.3458719849586487,
      "logps/chosen": -1.6929105520248413,
      "logps/rejected": -8.215627670288086,
      "loss": 27.4312,
      "nll_loss": 1.5623621940612793,
      "po_loss": -0.7051381468772888,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": -0.8464552760124207,
      "rewards/margins": 3.2613587379455566,
      "rewards/rejected": -4.107813835144043,
      "step": 300
    },
    {
      "epoch": 2.5888,
      "grad_norm": 938.1863403320312,
      "learning_rate": 2.58507813312448e-06,
      "log_odds_chosen": 6.914087772369385,
      "log_odds_ratio": -0.28258031606674194,
      "logps/chosen": -1.2223470211029053,
      "logps/rejected": -7.694061279296875,
      "loss": 21.9591,
      "nll_loss": 1.3453009128570557,
      "po_loss": -0.6590788960456848,
      "rewards/accuracies": 0.909375011920929,
      "rewards/chosen": -0.6111735105514526,
      "rewards/margins": 3.2358570098876953,
      "rewards/rejected": -3.8470306396484375,
      "step": 305
    },
    {
      "epoch": 2.6314666666666664,
      "grad_norm": 221.02281188964844,
      "learning_rate": 2.0610883091816525e-06,
      "log_odds_chosen": 5.710585594177246,
      "log_odds_ratio": -0.2850475609302521,
      "logps/chosen": -1.021317720413208,
      "logps/rejected": -6.243259906768799,
      "loss": 16.8653,
      "nll_loss": 1.247992753982544,
      "po_loss": -0.7209508419036865,
      "rewards/accuracies": 0.9281250238418579,
      "rewards/chosen": -0.510658860206604,
      "rewards/margins": 2.610970973968506,
      "rewards/rejected": -3.1216299533843994,
      "step": 310
    },
    {
      "epoch": 2.6741333333333333,
      "grad_norm": 475.2201843261719,
      "learning_rate": 1.59412823400657e-06,
      "log_odds_chosen": 5.025318145751953,
      "log_odds_ratio": -0.25998514890670776,
      "logps/chosen": -1.0306951999664307,
      "logps/rejected": -5.601127624511719,
      "loss": 14.5714,
      "nll_loss": 1.1835569143295288,
      "po_loss": -0.7282010316848755,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.5153475999832153,
      "rewards/margins": 2.2852160930633545,
      "rewards/rejected": -2.8005638122558594,
      "step": 315
    },
    {
      "epoch": 2.7168,
      "grad_norm": 337.47796630859375,
      "learning_rate": 1.1853588439213442e-06,
      "log_odds_chosen": 6.4345574378967285,
      "log_odds_ratio": -0.23376791179180145,
      "logps/chosen": -1.014891266822815,
      "logps/rejected": -7.0141282081604,
      "loss": 16.8373,
      "nll_loss": 1.2330230474472046,
      "po_loss": -0.7068589329719543,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.5074456334114075,
      "rewards/margins": 2.9996187686920166,
      "rewards/rejected": -3.5070641040802,
      "step": 320
    },
    {
      "epoch": 2.7594666666666665,
      "grad_norm": 251.52114868164062,
      "learning_rate": 8.357964040363209e-07,
      "log_odds_chosen": 7.006105899810791,
      "log_odds_ratio": -0.3340076804161072,
      "logps/chosen": -1.0801838636398315,
      "logps/rejected": -7.584368705749512,
      "loss": 7.9524,
      "nll_loss": 1.2427351474761963,
      "po_loss": -0.9942227602005005,
      "rewards/accuracies": 0.9281250238418579,
      "rewards/chosen": -0.5400919318199158,
      "rewards/margins": 3.2520930767059326,
      "rewards/rejected": -3.792184352874756,
      "step": 325
    },
    {
      "epoch": 2.8021333333333334,
      "grad_norm": 500.19580078125,
      "learning_rate": 5.463099816548579e-07,
      "log_odds_chosen": 7.333872318267822,
      "log_odds_ratio": -0.251155823469162,
      "logps/chosen": -1.1577497720718384,
      "logps/rejected": -8.027981758117676,
      "loss": 19.0267,
      "nll_loss": 1.2874268293380737,
      "po_loss": -0.6928423047065735,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.5788748860359192,
      "rewards/margins": 3.4351158142089844,
      "rewards/rejected": -4.013990879058838,
      "step": 330
    },
    {
      "epoch": 2.8448,
      "grad_norm": 827.9666748046875,
      "learning_rate": 3.1761928563510955e-07,
      "log_odds_chosen": 5.787890434265137,
      "log_odds_ratio": -0.2869322896003723,
      "logps/chosen": -1.0971204042434692,
      "logps/rejected": -6.4105658531188965,
      "loss": 12.9881,
      "nll_loss": 1.2217962741851807,
      "po_loss": -0.8159177899360657,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.5485602021217346,
      "rewards/margins": 2.6567225456237793,
      "rewards/rejected": -3.2052829265594482,
      "step": 335
    },
    {
      "epoch": 2.8874666666666666,
      "grad_norm": 292.38897705078125,
      "learning_rate": 1.5029287708036854e-07,
      "log_odds_chosen": 6.05283260345459,
      "log_odds_ratio": -0.24598774313926697,
      "logps/chosen": -1.0574235916137695,
      "logps/rejected": -6.646490573883057,
      "loss": 7.9611,
      "nll_loss": 1.1931092739105225,
      "po_loss": -0.9443262219429016,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.5287117958068848,
      "rewards/margins": 2.7945332527160645,
      "rewards/rejected": -3.3232452869415283,
      "step": 340
    },
    {
      "epoch": 2.9301333333333335,
      "grad_norm": 387.1372985839844,
      "learning_rate": 4.474675580662113e-08,
      "log_odds_chosen": 6.334747791290283,
      "log_odds_ratio": -0.32026782631874084,
      "logps/chosen": -1.1003960371017456,
      "logps/rejected": -6.976574897766113,
      "loss": 12.2009,
      "nll_loss": 1.235642671585083,
      "po_loss": -0.8543656468391418,
      "rewards/accuracies": 0.9156249761581421,
      "rewards/chosen": -0.5501980185508728,
      "rewards/margins": 2.938089370727539,
      "rewards/rejected": -3.4882874488830566,
      "step": 345
    },
    {
      "epoch": 2.9728,
      "grad_norm": 1250.0882568359375,
      "learning_rate": 1.2433261014244136e-09,
      "log_odds_chosen": 7.907639980316162,
      "log_odds_ratio": -0.22040143609046936,
      "logps/chosen": -1.2740892171859741,
      "logps/rejected": -8.710322380065918,
      "loss": 9.5303,
      "nll_loss": 1.3264214992523193,
      "po_loss": -1.028598666191101,
      "rewards/accuracies": 0.9281250238418579,
      "rewards/chosen": -0.6370446085929871,
      "rewards/margins": 3.7181167602539062,
      "rewards/rejected": -4.355161190032959,
      "step": 350
    },
    {
      "epoch": 2.981333333333333,
      "step": 351,
      "total_flos": 0.0,
      "train_loss": 54.80183921716152,
      "train_runtime": 10133.5503,
      "train_samples_per_second": 2.22,
      "train_steps_per_second": 0.035
    }
  ],
  "logging_steps": 5,
  "max_steps": 351,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}