{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.986666666666667,
  "eval_steps": 500,
  "global_step": 315,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.047407407407407405,
      "grad_norm": 1152.0,
      "learning_rate": 7.8125e-06,
      "log_odds_chosen": -1.0631844997406006,
      "log_odds_ratio": -11.005602836608887,
      "logps/chosen": -22.707393646240234,
      "logps/rejected": -21.644441604614258,
      "loss": 467.0638,
      "nll_loss": 9.838674545288086,
      "rewards/accuracies": 0.4781250059604645,
      "rewards/chosen": -11.353696823120117,
      "rewards/margins": -0.5314784646034241,
      "rewards/rejected": -10.822220802307129,
      "step": 5
    },
    {
      "epoch": 0.09481481481481481,
      "grad_norm": 600.0,
      "learning_rate": 1.5625e-05,
      "log_odds_chosen": -3.1253325939178467,
      "log_odds_ratio": -11.045626640319824,
      "logps/chosen": -21.99468231201172,
      "logps/rejected": -18.869693756103516,
      "loss": 435.4043,
      "nll_loss": 8.75256633758545,
      "rewards/accuracies": 0.44062501192092896,
      "rewards/chosen": -10.99734115600586,
      "rewards/margins": -1.5624935626983643,
      "rewards/rejected": -9.434846878051758,
      "step": 10
    },
    {
      "epoch": 0.14222222222222222,
      "grad_norm": 464.0,
      "learning_rate": 2.34375e-05,
      "log_odds_chosen": -1.283685564994812,
      "log_odds_ratio": -11.318685531616211,
      "logps/chosen": -21.985923767089844,
      "logps/rejected": -20.700777053833008,
      "loss": 412.295,
      "nll_loss": 8.000757217407227,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": -10.992961883544922,
      "rewards/margins": -0.6425749063491821,
      "rewards/rejected": -10.350388526916504,
      "step": 15
    },
    {
      "epoch": 0.18962962962962962,
      "grad_norm": 3360.0,
      "learning_rate": 3.125e-05,
      "log_odds_chosen": -4.3174591064453125,
      "log_odds_ratio": -9.530021667480469,
      "logps/chosen": -17.268482208251953,
      "logps/rejected": -12.952882766723633,
      "loss": 332.5653,
      "nll_loss": 6.284921646118164,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -8.634241104125977,
      "rewards/margins": -2.157799482345581,
      "rewards/rejected": -6.476441383361816,
      "step": 20
    },
    {
      "epoch": 0.23703703703703705,
      "grad_norm": 241.0,
      "learning_rate": 3.90625e-05,
      "log_odds_chosen": -0.13406293094158173,
      "log_odds_ratio": -1.9662774801254272,
      "logps/chosen": -4.281245708465576,
      "logps/rejected": -4.128834247589111,
      "loss": 101.8704,
      "nll_loss": 2.7933926582336426,
      "rewards/accuracies": 0.5531250238418579,
      "rewards/chosen": -2.140622854232788,
      "rewards/margins": -0.07620568573474884,
      "rewards/rejected": -2.0644171237945557,
      "step": 25
    },
    {
      "epoch": 0.28444444444444444,
      "grad_norm": 242.0,
      "learning_rate": 4.6875e-05,
      "log_odds_chosen": 0.10939224064350128,
      "log_odds_ratio": -0.8504419326782227,
      "logps/chosen": -1.855107307434082,
      "logps/rejected": -1.9498792886734009,
      "loss": 64.8284,
      "nll_loss": 2.0649564266204834,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": -0.927553653717041,
      "rewards/margins": 0.04738599434494972,
      "rewards/rejected": -0.9749396443367004,
      "step": 30
    },
    {
      "epoch": 0.33185185185185184,
      "grad_norm": 282.0,
      "learning_rate": 4.998613757348784e-05,
      "log_odds_chosen": 0.3194878101348877,
      "log_odds_ratio": -0.7833901643753052,
      "logps/chosen": -1.7884143590927124,
      "logps/rejected": -2.071709156036377,
      "loss": 58.4951,
      "nll_loss": 1.9425550699234009,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.8942071795463562,
      "rewards/margins": 0.14164744317531586,
      "rewards/rejected": -1.0358545780181885,
      "step": 35
    },
    {
      "epoch": 0.37925925925925924,
      "grad_norm": 214.0,
      "learning_rate": 4.990147841143462e-05,
      "log_odds_chosen": 0.5328894853591919,
      "log_odds_ratio": -0.6567676663398743,
      "logps/chosen": -1.6281368732452393,
      "logps/rejected": -2.09126877784729,
      "loss": 53.2976,
      "nll_loss": 1.8886241912841797,
      "rewards/accuracies": 0.6656249761581421,
      "rewards/chosen": -0.8140684366226196,
      "rewards/margins": 0.2315659075975418,
      "rewards/rejected": -1.045634388923645,
      "step": 40
    },
    {
      "epoch": 0.4266666666666667,
      "grad_norm": 354.0,
      "learning_rate": 4.97401218720448e-05,
      "log_odds_chosen": 1.3014285564422607,
      "log_odds_ratio": -0.5872281789779663,
      "logps/chosen": -1.680598497390747,
      "logps/rejected": -2.899627208709717,
      "loss": 46.0017,
      "nll_loss": 1.8771553039550781,
      "rewards/accuracies": 0.6968749761581421,
      "rewards/chosen": -0.8402992486953735,
      "rewards/margins": 0.6095143556594849,
      "rewards/rejected": -1.4498136043548584,
      "step": 45
    },
    {
      "epoch": 0.4740740740740741,
      "grad_norm": 226.0,
      "learning_rate": 4.9502564938797946e-05,
      "log_odds_chosen": 0.6567279696464539,
      "log_odds_ratio": -0.6511496305465698,
      "logps/chosen": -1.4998167753219604,
      "logps/rejected": -2.0936269760131836,
      "loss": 49.0928,
      "nll_loss": 1.7653518915176392,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7499083876609802,
      "rewards/margins": 0.29690515995025635,
      "rewards/rejected": -1.0468134880065918,
      "step": 50
    },
    {
      "epoch": 0.5214814814814814,
      "grad_norm": 202.0,
      "learning_rate": 4.918953929490768e-05,
      "log_odds_chosen": 1.189807653427124,
      "log_odds_ratio": -0.617165744304657,
      "logps/chosen": -1.565733551979065,
      "logps/rejected": -2.669482707977295,
      "loss": 44.8712,
      "nll_loss": 1.786768913269043,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.7828667759895325,
      "rewards/margins": 0.5518745183944702,
      "rewards/rejected": -1.3347413539886475,
      "step": 55
    },
    {
      "epoch": 0.5688888888888889,
      "grad_norm": 278.0,
      "learning_rate": 4.88020090697132e-05,
      "log_odds_chosen": 1.5694297552108765,
      "log_odds_ratio": -0.5945903062820435,
      "logps/chosen": -1.5673797130584717,
      "logps/rejected": -3.0414257049560547,
      "loss": 44.6048,
      "nll_loss": 1.8120968341827393,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.7836898565292358,
      "rewards/margins": 0.7370229959487915,
      "rewards/rejected": -1.5207128524780273,
      "step": 60
    },
    {
      "epoch": 0.6162962962962963,
      "grad_norm": 160.0,
      "learning_rate": 4.834116786912897e-05,
      "log_odds_chosen": 1.5031869411468506,
      "log_odds_ratio": -0.663598895072937,
      "logps/chosen": -1.6163688898086548,
      "logps/rejected": -3.024693250656128,
      "loss": 46.3428,
      "nll_loss": 1.8134605884552002,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8081844449043274,
      "rewards/margins": 0.7041621208190918,
      "rewards/rejected": -1.512346625328064,
      "step": 65
    },
    {
      "epoch": 0.6637037037037037,
      "grad_norm": 298.0,
      "learning_rate": 4.7808435099299045e-05,
      "log_odds_chosen": 2.1792948246002197,
      "log_odds_ratio": -0.5366494059562683,
      "logps/chosen": -1.6390451192855835,
      "logps/rejected": -3.6916420459747314,
      "loss": 38.6914,
      "nll_loss": 1.7929494380950928,
      "rewards/accuracies": 0.753125011920929,
      "rewards/chosen": -0.8195225596427917,
      "rewards/margins": 1.0262985229492188,
      "rewards/rejected": -1.8458210229873657,
      "step": 70
    },
    {
      "epoch": 0.7111111111111111,
      "grad_norm": 300.0,
      "learning_rate": 4.720545159477922e-05,
      "log_odds_chosen": 1.6375625133514404,
      "log_odds_ratio": -0.6377304196357727,
      "logps/chosen": -1.5810494422912598,
      "logps/rejected": -3.109715223312378,
      "loss": 42.8578,
      "nll_loss": 1.7745224237442017,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.7905247211456299,
      "rewards/margins": 0.7643328905105591,
      "rewards/rejected": -1.554857611656189,
      "step": 75
    },
    {
      "epoch": 0.7585185185185185,
      "grad_norm": 208.0,
      "learning_rate": 4.653407456471222e-05,
      "log_odds_chosen": 1.206108808517456,
      "log_odds_ratio": -0.6781158447265625,
      "logps/chosen": -1.5513617992401123,
      "logps/rejected": -2.6515350341796875,
      "loss": 44.2435,
      "nll_loss": 1.7461738586425781,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.7756808996200562,
      "rewards/margins": 0.5500864386558533,
      "rewards/rejected": -1.3257675170898438,
      "step": 80
    },
    {
      "epoch": 0.8059259259259259,
      "grad_norm": 282.0,
      "learning_rate": 4.579637187256222e-05,
      "log_odds_chosen": 0.8860191106796265,
      "log_odds_ratio": -0.5883544683456421,
      "logps/chosen": -1.4292150735855103,
      "logps/rejected": -2.2102513313293457,
      "loss": 42.4359,
      "nll_loss": 1.656638741493225,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.7146075367927551,
      "rewards/margins": 0.3905182480812073,
      "rewards/rejected": -1.1051256656646729,
      "step": 85
    },
    {
      "epoch": 0.8533333333333334,
      "grad_norm": 1400.0,
      "learning_rate": 4.499461566702685e-05,
      "log_odds_chosen": 2.8823413848876953,
      "log_odds_ratio": -0.8564308285713196,
      "logps/chosen": -2.120701789855957,
      "logps/rejected": -4.8557209968566895,
      "loss": 47.9855,
      "nll_loss": 1.9995596408843994,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.0603508949279785,
      "rewards/margins": 1.3675094842910767,
      "rewards/rejected": -2.4278604984283447,
      "step": 90
    },
    {
      "epoch": 0.9007407407407407,
      "grad_norm": 366.0,
      "learning_rate": 4.413127538374411e-05,
      "log_odds_chosen": 1.6698837280273438,
      "log_odds_ratio": -0.7055229544639587,
      "logps/chosen": -1.6337862014770508,
      "logps/rejected": -3.1812427043914795,
      "loss": 41.8057,
      "nll_loss": 1.781404733657837,
      "rewards/accuracies": 0.684374988079071,
      "rewards/chosen": -0.8168931007385254,
      "rewards/margins": 0.7737283110618591,
      "rewards/rejected": -1.5906213521957397,
      "step": 95
    },
    {
      "epoch": 0.9481481481481482,
      "grad_norm": 364.0,
      "learning_rate": 4.320901013934887e-05,
      "log_odds_chosen": 1.8235197067260742,
      "log_odds_ratio": -0.5255730748176575,
      "logps/chosen": -1.462209939956665,
      "logps/rejected": -3.1428921222686768,
      "loss": 34.1059,
      "nll_loss": 1.6777896881103516,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.7311049699783325,
      "rewards/margins": 0.840340793132782,
      "rewards/rejected": -1.5714460611343384,
      "step": 100
    },
    {
      "epoch": 0.9955555555555555,
      "grad_norm": 268.0,
      "learning_rate": 4.223066054130568e-05,
      "log_odds_chosen": 1.5804908275604248,
      "log_odds_ratio": -0.580092191696167,
      "logps/chosen": -1.4532772302627563,
      "logps/rejected": -2.918117046356201,
      "loss": 37.4817,
      "nll_loss": 1.6894712448120117,
      "rewards/accuracies": 0.690625011920929,
      "rewards/chosen": -0.7266386151313782,
      "rewards/margins": 0.7324199080467224,
      "rewards/rejected": -1.4590585231781006,
      "step": 105
    },
    {
      "epoch": 1.0429629629629629,
      "grad_norm": 286.0,
      "learning_rate": 4.1199239938743797e-05,
      "log_odds_chosen": 2.7626733779907227,
      "log_odds_ratio": -0.424811989068985,
      "logps/chosen": -1.4315791130065918,
      "logps/rejected": -3.964046001434326,
      "loss": 24.2139,
      "nll_loss": 1.5723967552185059,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.7157895565032959,
      "rewards/margins": 1.2662335634231567,
      "rewards/rejected": -1.982023000717163,
      "step": 110
    },
    {
      "epoch": 1.0903703703703704,
      "grad_norm": 254.0,
      "learning_rate": 4.0117925141242174e-05,
      "log_odds_chosen": 3.299778461456299,
      "log_odds_ratio": -0.5258711576461792,
      "logps/chosen": -1.3833136558532715,
      "logps/rejected": -4.405320644378662,
      "loss": 25.4402,
      "nll_loss": 1.6238991022109985,
      "rewards/accuracies": 0.784375011920929,
      "rewards/chosen": -0.6916568279266357,
      "rewards/margins": 1.5110036134719849,
      "rewards/rejected": -2.202660322189331,
      "step": 115
    },
    {
      "epoch": 1.1377777777777778,
      "grad_norm": 528.0,
      "learning_rate": 3.899004663415084e-05,
      "log_odds_chosen": 2.667694568634033,
      "log_odds_ratio": -0.40380731225013733,
      "logps/chosen": -1.2623659372329712,
      "logps/rejected": -3.6577308177948,
      "loss": 23.1023,
      "nll_loss": 1.5065176486968994,
      "rewards/accuracies": 0.815625011920929,
      "rewards/chosen": -0.6311829686164856,
      "rewards/margins": 1.1976823806762695,
      "rewards/rejected": -1.8288654088974,
      "step": 120
    },
    {
      "epoch": 1.1851851851851851,
      "grad_norm": 342.0,
      "learning_rate": 3.781907832058587e-05,
      "log_odds_chosen": 2.9293477535247803,
      "log_odds_ratio": -0.45116695761680603,
      "logps/chosen": -1.4864990711212158,
      "logps/rejected": -4.153409957885742,
      "loss": 25.0104,
      "nll_loss": 1.6183675527572632,
      "rewards/accuracies": 0.8031250238418579,
      "rewards/chosen": -0.7432495355606079,
      "rewards/margins": 1.3334555625915527,
      "rewards/rejected": -2.076704978942871,
      "step": 125
    },
    {
      "epoch": 1.2325925925925927,
      "grad_norm": 268.0,
      "learning_rate": 3.660862682169282e-05,
      "log_odds_chosen": 3.8303322792053223,
      "log_odds_ratio": -0.37618154287338257,
      "logps/chosen": -1.4800456762313843,
      "logps/rejected": -5.02761697769165,
      "loss": 18.1555,
      "nll_loss": 1.5957032442092896,
      "rewards/accuracies": 0.8343750238418579,
      "rewards/chosen": -0.7400228381156921,
      "rewards/margins": 1.7737855911254883,
      "rewards/rejected": -2.513808488845825,
      "step": 130
    },
    {
      "epoch": 1.28,
      "grad_norm": 458.0,
      "learning_rate": 3.5362420368134356e-05,
      "log_odds_chosen": 2.9902822971343994,
      "log_odds_ratio": -0.43725720047950745,
      "logps/chosen": -1.473851203918457,
      "logps/rejected": -4.198307991027832,
      "loss": 22.4211,
      "nll_loss": 1.5640933513641357,
      "rewards/accuracies": 0.8218749761581421,
      "rewards/chosen": -0.7369256019592285,
      "rewards/margins": 1.362228274345398,
      "rewards/rejected": -2.099153995513916,
      "step": 135
    },
    {
      "epoch": 1.3274074074074074,
      "grad_norm": 201.0,
      "learning_rate": 3.408429731701635e-05,
      "log_odds_chosen": 2.826463460922241,
      "log_odds_ratio": -0.39234691858291626,
      "logps/chosen": -1.3628809452056885,
      "logps/rejected": -3.936453342437744,
      "loss": 21.0249,
      "nll_loss": 1.5445901155471802,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.6814404726028442,
      "rewards/margins": 1.2867863178253174,
      "rewards/rejected": -1.968226671218872,
      "step": 140
    },
    {
      "epoch": 1.374814814814815,
      "grad_norm": 784.0,
      "learning_rate": 3.2778194329621104e-05,
      "log_odds_chosen": 3.861825942993164,
      "log_odds_ratio": -0.3445121943950653,
      "logps/chosen": -1.6509920358657837,
      "logps/rejected": -5.243237495422363,
      "loss": 18.864,
      "nll_loss": 1.7057406902313232,
      "rewards/accuracies": 0.8843749761581421,
      "rewards/chosen": -0.8254960179328918,
      "rewards/margins": 1.7961229085922241,
      "rewards/rejected": -2.6216187477111816,
      "step": 145
    },
    {
      "epoch": 1.4222222222222223,
      "grad_norm": 316.0,
      "learning_rate": 3.144813424636031e-05,
      "log_odds_chosen": 4.123653888702393,
      "log_odds_ratio": -0.4155045449733734,
      "logps/chosen": -1.6837345361709595,
      "logps/rejected": -5.53621768951416,
      "loss": 18.678,
      "nll_loss": 1.6780169010162354,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.8418672680854797,
      "rewards/margins": 1.9262415170669556,
      "rewards/rejected": -2.76810884475708,
      "step": 150
    },
    {
      "epoch": 1.4696296296296296,
      "grad_norm": 466.0,
      "learning_rate": 3.0098213696293542e-05,
      "log_odds_chosen": 3.181971311569214,
      "log_odds_ratio": -0.4872185289859772,
      "logps/chosen": -1.4469890594482422,
      "logps/rejected": -4.361490726470947,
      "loss": 21.8554,
      "nll_loss": 1.5998506546020508,
      "rewards/accuracies": 0.815625011920929,
      "rewards/chosen": -0.7234945297241211,
      "rewards/margins": 1.457250952720642,
      "rewards/rejected": -2.1807453632354736,
      "step": 155
    },
    {
      "epoch": 1.5170370370370372,
      "grad_norm": 412.0,
      "learning_rate": 2.8732590479375165e-05,
      "log_odds_chosen": 4.313605308532715,
      "log_odds_ratio": -0.4044560492038727,
      "logps/chosen": -1.7111812829971313,
      "logps/rejected": -5.7456865310668945,
      "loss": 14.9335,
      "nll_loss": 1.6823270320892334,
      "rewards/accuracies": 0.8343750238418579,
      "rewards/chosen": -0.8555906414985657,
      "rewards/margins": 2.0172524452209473,
      "rewards/rejected": -2.8728432655334473,
      "step": 160
    },
    {
      "epoch": 1.5644444444444443,
      "grad_norm": 270.0,
      "learning_rate": 2.7355470760292956e-05,
      "log_odds_chosen": 3.449075698852539,
      "log_odds_ratio": -0.4021359980106354,
      "logps/chosen": -1.360383152961731,
      "logps/rejected": -4.50086784362793,
      "loss": 17.5439,
      "nll_loss": 1.5542360544204712,
      "rewards/accuracies": 0.8531249761581421,
      "rewards/chosen": -0.6801915764808655,
      "rewards/margins": 1.5702422857284546,
      "rewards/rejected": -2.250433921813965,
      "step": 165
    },
    {
      "epoch": 1.6118518518518519,
      "grad_norm": 334.0,
      "learning_rate": 2.597109611334169e-05,
      "log_odds_chosen": 4.0982818603515625,
      "log_odds_ratio": -0.4611207842826843,
      "logps/chosen": -1.5969505310058594,
      "logps/rejected": -5.419460773468018,
      "loss": 18.2855,
      "nll_loss": 1.6615581512451172,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.7984752655029297,
      "rewards/margins": 1.911254644393921,
      "rewards/rejected": -2.709730386734009,
      "step": 170
    },
    {
      "epoch": 1.6592592592592592,
      "grad_norm": 424.0,
      "learning_rate": 2.458373045823404e-05,
      "log_odds_chosen": 3.9468979835510254,
      "log_odds_ratio": -0.36181649565696716,
      "logps/chosen": -1.4357500076293945,
      "logps/rejected": -5.089698791503906,
      "loss": 14.1876,
      "nll_loss": 1.5684595108032227,
      "rewards/accuracies": 0.8218749761581421,
      "rewards/chosen": -0.7178750038146973,
      "rewards/margins": 1.8269745111465454,
      "rewards/rejected": -2.544849395751953,
      "step": 175
    },
    {
      "epoch": 1.7066666666666666,
      "grad_norm": 692.0,
      "learning_rate": 2.3197646927086697e-05,
      "log_odds_chosen": 3.9691085815429688,
      "log_odds_ratio": -0.35701385140419006,
      "logps/chosen": -1.550195336341858,
      "logps/rejected": -5.20673942565918,
      "loss": 14.9335,
      "nll_loss": 1.635414719581604,
      "rewards/accuracies": 0.8843749761581421,
      "rewards/chosen": -0.775097668170929,
      "rewards/margins": 1.8282718658447266,
      "rewards/rejected": -2.60336971282959,
      "step": 180
    },
    {
      "epoch": 1.7540740740740741,
      "grad_norm": 304.0,
      "learning_rate": 2.1817114703032176e-05,
      "log_odds_chosen": 4.004181861877441,
      "log_odds_ratio": -0.4124390184879303,
      "logps/chosen": -1.5889790058135986,
      "logps/rejected": -5.336648464202881,
      "loss": 13.5162,
      "nll_loss": 1.580910563468933,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.7944895029067993,
      "rewards/margins": 1.8738348484039307,
      "rewards/rejected": -2.6683242321014404,
      "step": 185
    },
    {
      "epoch": 1.8014814814814815,
      "grad_norm": 344.0,
      "learning_rate": 2.0446385870993467e-05,
      "log_odds_chosen": 3.7639663219451904,
      "log_odds_ratio": -0.3424893021583557,
      "logps/chosen": -1.3625359535217285,
      "logps/rejected": -4.8325395584106445,
      "loss": 10.6844,
      "nll_loss": 1.4781030416488647,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.6812679767608643,
      "rewards/margins": 1.735001564025879,
      "rewards/rejected": -2.4162697792053223,
      "step": 190
    },
    {
      "epoch": 1.8488888888888888,
      "grad_norm": 632.0,
      "learning_rate": 1.9089682321121834e-05,
      "log_odds_chosen": 4.872841835021973,
      "log_odds_ratio": -0.3931121528148651,
      "logps/chosen": -1.8755972385406494,
      "logps/rejected": -6.473967552185059,
      "loss": 14.5199,
      "nll_loss": 1.8242985010147095,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.9377986192703247,
      "rewards/margins": 2.299185037612915,
      "rewards/rejected": -3.2369837760925293,
      "step": 195
    },
    {
      "epoch": 1.8962962962962964,
      "grad_norm": 438.0,
      "learning_rate": 1.775118274523545e-05,
      "log_odds_chosen": 3.7074005603790283,
      "log_odds_ratio": -0.5268596410751343,
      "logps/chosen": -1.7662988901138306,
      "logps/rejected": -5.216193199157715,
      "loss": 20.4733,
      "nll_loss": 1.7086811065673828,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.8831494450569153,
      "rewards/margins": 1.724947214126587,
      "rewards/rejected": -2.6080965995788574,
      "step": 200
    },
    {
      "epoch": 1.9437037037037037,
      "grad_norm": 576.0,
      "learning_rate": 1.643500976631037e-05,
      "log_odds_chosen": 3.051617383956909,
      "log_odds_ratio": -0.38788479566574097,
      "logps/chosen": -1.2543895244598389,
      "logps/rejected": -4.006674289703369,
      "loss": 16.8108,
      "nll_loss": 1.5027841329574585,
      "rewards/accuracies": 0.8218749761581421,
      "rewards/chosen": -0.6271947622299194,
      "rewards/margins": 1.3761423826217651,
      "rewards/rejected": -2.0033371448516846,
      "step": 205
    },
    {
      "epoch": 1.991111111111111,
      "grad_norm": 386.0,
      "learning_rate": 1.514521724066537e-05,
      "log_odds_chosen": 3.231032609939575,
      "log_odds_ratio": -0.43392324447631836,
      "logps/chosen": -1.4704937934875488,
      "logps/rejected": -4.4187726974487305,
      "loss": 17.4563,
      "nll_loss": 1.565903902053833,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.7352468967437744,
      "rewards/margins": 1.4741394519805908,
      "rewards/rejected": -2.2093863487243652,
      "step": 210
    },
    {
      "epoch": 2.0385185185185186,
      "grad_norm": 278.0,
      "learning_rate": 1.3885777771950348e-05,
      "log_odds_chosen": 5.03135871887207,
      "log_odds_ratio": -0.3135664165019989,
      "logps/chosen": -1.407021164894104,
      "logps/rejected": -6.015364646911621,
      "loss": 0.5226,
      "nll_loss": 1.4759743213653564,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.703510582447052,
      "rewards/margins": 2.3041718006134033,
      "rewards/rejected": -3.0076823234558105,
      "step": 215
    },
    {
      "epoch": 2.0859259259259257,
      "grad_norm": 378.0,
      "learning_rate": 1.2660570475395683e-05,
      "log_odds_chosen": 5.138548374176025,
      "log_odds_ratio": -0.25997596979141235,
      "logps/chosen": -1.37165367603302,
      "logps/rejected": -6.125999450683594,
      "loss": -0.8241,
      "nll_loss": 1.4749200344085693,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.68582683801651,
      "rewards/margins": 2.3771729469299316,
      "rewards/rejected": -3.062999725341797,
      "step": 220
    },
    {
      "epoch": 2.1333333333333333,
      "grad_norm": 396.0,
      "learning_rate": 1.1473369030008974e-05,
      "log_odds_chosen": 6.11403751373291,
      "log_odds_ratio": -0.1846025437116623,
      "logps/chosen": -1.402966856956482,
      "logps/rejected": -7.111693382263184,
      "loss": -8.5304,
      "nll_loss": 1.5103604793548584,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.701483428478241,
      "rewards/margins": 2.854363441467285,
      "rewards/rejected": -3.555846691131592,
      "step": 225
    },
    {
      "epoch": 2.180740740740741,
      "grad_norm": 368.0,
      "learning_rate": 1.0327830055518842e-05,
      "log_odds_chosen": 6.256018161773682,
      "log_odds_ratio": -0.18493634462356567,
      "logps/chosen": -1.3649917840957642,
      "logps/rejected": -7.209232330322266,
      "loss": -7.4943,
      "nll_loss": 1.4895808696746826,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.6824958920478821,
      "rewards/margins": 2.9221205711364746,
      "rewards/rejected": -3.604616165161133,
      "step": 230
    },
    {
      "epoch": 2.228148148148148,
      "grad_norm": 328.0,
      "learning_rate": 9.227481849865235e-06,
      "log_odds_chosen": 6.152120113372803,
      "log_odds_ratio": -0.22947028279304504,
      "logps/chosen": -1.2814245223999023,
      "logps/rejected": -7.014758110046387,
      "loss": -6.2174,
      "nll_loss": 1.4657459259033203,
      "rewards/accuracies": 0.9156249761581421,
      "rewards/chosen": -0.6407122611999512,
      "rewards/margins": 2.866666793823242,
      "rewards/rejected": -3.5073790550231934,
      "step": 235
    },
    {
      "epoch": 2.2755555555555556,
      "grad_norm": 440.0,
      "learning_rate": 8.175713521924978e-06,
      "log_odds_chosen": 6.160048484802246,
      "log_odds_ratio": -0.1592252552509308,
      "logps/chosen": -1.235910177230835,
      "logps/rejected": -6.972237586975098,
      "loss": -8.9307,
      "nll_loss": 1.388727068901062,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": -0.6179550886154175,
      "rewards/margins": 2.8681633472442627,
      "rewards/rejected": -3.486118793487549,
      "step": 240
    },
    {
      "epoch": 2.322962962962963,
      "grad_norm": 412.0,
      "learning_rate": 7.1757645529443665e-06,
      "log_odds_chosen": 6.512506008148193,
      "log_odds_ratio": -0.16299432516098022,
      "logps/chosen": -1.2822771072387695,
      "logps/rejected": -7.385122776031494,
      "loss": -10.2451,
      "nll_loss": 1.4544556140899658,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.6411385536193848,
      "rewards/margins": 3.051422595977783,
      "rewards/rejected": -3.692561388015747,
      "step": 245
    },
    {
      "epoch": 2.3703703703703702,
      "grad_norm": 366.0,
      "learning_rate": 6.230714818829733e-06,
      "log_odds_chosen": 6.763820648193359,
      "log_odds_ratio": -0.15657763183116913,
      "logps/chosen": -1.2465496063232422,
      "logps/rejected": -7.550148010253906,
      "loss": -8.9088,
      "nll_loss": 1.4477818012237549,
      "rewards/accuracies": 0.9281250238418579,
      "rewards/chosen": -0.6232748031616211,
      "rewards/margins": 3.151798963546753,
      "rewards/rejected": -3.775074005126953,
      "step": 250
    },
    {
      "epoch": 2.417777777777778,
      "grad_norm": 348.0,
      "learning_rate": 5.343475104027743e-06,
      "log_odds_chosen": 6.812985897064209,
      "log_odds_ratio": -0.13997027277946472,
      "logps/chosen": -1.285111665725708,
      "logps/rejected": -7.66811990737915,
      "loss": -11.2052,
      "nll_loss": 1.4328773021697998,
      "rewards/accuracies": 0.9468749761581421,
      "rewards/chosen": -0.642555832862854,
      "rewards/margins": 3.1915042400360107,
      "rewards/rejected": -3.834059953689575,
      "step": 255
    },
    {
      "epoch": 2.4651851851851854,
      "grad_norm": 632.0,
      "learning_rate": 4.516778136213037e-06,
      "log_odds_chosen": 6.276130676269531,
      "log_odds_ratio": -0.18893949687480927,
      "logps/chosen": -1.3154428005218506,
      "logps/rejected": -7.166337013244629,
      "loss": -5.5648,
      "nll_loss": 1.4617598056793213,
      "rewards/accuracies": 0.9156249761581421,
      "rewards/chosen": -0.6577214002609253,
      "rewards/margins": 2.9254469871520996,
      "rewards/rejected": -3.5831685066223145,
      "step": 260
    },
    {
      "epoch": 2.5125925925925925,
      "grad_norm": 644.0,
      "learning_rate": 3.7531701693965554e-06,
      "log_odds_chosen": 6.7295122146606445,
      "log_odds_ratio": -0.2116066962480545,
      "logps/chosen": -1.275740623474121,
      "logps/rejected": -7.586668968200684,
      "loss": -9.4219,
      "nll_loss": 1.4439576864242554,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.6378703117370605,
      "rewards/margins": 3.155463933944702,
      "rewards/rejected": -3.793334484100342,
      "step": 265
    },
    {
      "epoch": 2.56,
      "grad_norm": 314.0,
      "learning_rate": 3.055003141378948e-06,
      "log_odds_chosen": 6.7960004806518555,
      "log_odds_ratio": -0.18422751128673553,
      "logps/chosen": -1.1799513101577759,
      "logps/rejected": -7.515390872955322,
      "loss": -11.538,
      "nll_loss": 1.3993252515792847,
      "rewards/accuracies": 0.934374988079071,
      "rewards/chosen": -0.5899756550788879,
      "rewards/margins": 3.167720079421997,
      "rewards/rejected": -3.757695436477661,
      "step": 270
    },
    {
      "epoch": 2.6074074074074076,
      "grad_norm": 398.0,
      "learning_rate": 2.424427429704365e-06,
      "log_odds_chosen": 6.602663516998291,
      "log_odds_ratio": -0.18241380155086517,
      "logps/chosen": -1.3822739124298096,
      "logps/rejected": -7.560347557067871,
      "loss": -8.7863,
      "nll_loss": 1.474333643913269,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6911369562149048,
      "rewards/margins": 3.0890371799468994,
      "rewards/rejected": -3.7801737785339355,
      "step": 275
    },
    {
      "epoch": 2.6548148148148147,
      "grad_norm": 700.0,
      "learning_rate": 1.8633852284264508e-06,
      "log_odds_chosen": 6.423117637634277,
      "log_odds_ratio": -0.18244215846061707,
      "logps/chosen": -1.2519899606704712,
      "logps/rejected": -7.237380027770996,
      "loss": -7.6169,
      "nll_loss": 1.4630454778671265,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.6259949803352356,
      "rewards/margins": 2.9926953315734863,
      "rewards/rejected": -3.618690013885498,
      "step": 280
    },
    {
      "epoch": 2.7022222222222223,
      "grad_norm": 376.0,
      "learning_rate": 1.3736045660864034e-06,
      "log_odds_chosen": 6.383296966552734,
      "log_odds_ratio": -0.22375822067260742,
      "logps/chosen": -1.2126858234405518,
      "logps/rejected": -7.17873477935791,
      "loss": -6.2815,
      "nll_loss": 1.423060417175293,
      "rewards/accuracies": 0.8968750238418579,
      "rewards/chosen": -0.6063429117202759,
      "rewards/margins": 2.9830245971679688,
      "rewards/rejected": -3.589367389678955,
      "step": 285
    },
    {
      "epoch": 2.74962962962963,
      "grad_norm": 288.0,
      "learning_rate": 9.565939833279192e-07,
      "log_odds_chosen": 5.9918012619018555,
      "log_odds_ratio": -0.24828758835792542,
      "logps/chosen": -1.4515832662582397,
      "logps/rejected": -7.042089939117432,
      "loss": -2.5592,
      "nll_loss": 1.5330219268798828,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.7257916331291199,
      "rewards/margins": 2.7952535152435303,
      "rewards/rejected": -3.521044969558716,
      "step": 290
    },
    {
      "epoch": 2.797037037037037,
      "grad_norm": 452.0,
      "learning_rate": 6.136378865420872e-07,
      "log_odds_chosen": 6.23110294342041,
      "log_odds_ratio": -0.2531821131706238,
      "logps/chosen": -1.3049705028533936,
      "logps/rejected": -7.137003421783447,
      "loss": -4.6802,
      "nll_loss": 1.456378698348999,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.6524852514266968,
      "rewards/margins": 2.9160165786743164,
      "rewards/rejected": -3.5685017108917236,
      "step": 295
    },
    {
      "epoch": 2.8444444444444446,
      "grad_norm": 354.0,
      "learning_rate": 3.45792591853214e-07,
      "log_odds_chosen": 6.851447105407715,
      "log_odds_ratio": -0.21110931038856506,
      "logps/chosen": -1.1985723972320557,
      "logps/rejected": -7.616302490234375,
      "loss": -9.172,
      "nll_loss": 1.4328322410583496,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.5992861986160278,
      "rewards/margins": 3.20886492729187,
      "rewards/rejected": -3.8081512451171875,
      "step": 300
    },
    {
      "epoch": 2.891851851851852,
      "grad_norm": 468.0,
      "learning_rate": 1.538830716302092e-07,
      "log_odds_chosen": 6.681235313415527,
      "log_odds_ratio": -0.16963359713554382,
      "logps/chosen": -1.2472527027130127,
      "logps/rejected": -7.471646308898926,
      "loss": -10.8492,
      "nll_loss": 1.403349757194519,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.6236263513565063,
      "rewards/margins": 3.112197160720825,
      "rewards/rejected": -3.735823154449463,
      "step": 305
    },
    {
      "epoch": 2.9392592592592592,
      "grad_norm": 384.0,
      "learning_rate": 3.8500413544415025e-08,
      "log_odds_chosen": 6.875144958496094,
      "log_odds_ratio": -0.12083999067544937,
      "logps/chosen": -1.2005863189697266,
      "logps/rejected": -7.624534606933594,
      "loss": -11.3909,
      "nll_loss": 1.4330816268920898,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": -0.6002931594848633,
      "rewards/margins": 3.2119739055633545,
      "rewards/rejected": -3.812267303466797,
      "step": 310
    },
    {
      "epoch": 2.986666666666667,
      "grad_norm": 354.0,
      "learning_rate": 0.0,
      "log_odds_chosen": 6.552358150482178,
      "log_odds_ratio": -0.21020980179309845,
      "logps/chosen": -1.2049369812011719,
      "logps/rejected": -7.295576572418213,
      "loss": -9.7319,
      "nll_loss": 1.3733375072479248,
      "rewards/accuracies": 0.903124988079071,
      "rewards/chosen": -0.6024684906005859,
      "rewards/margins": 3.0453197956085205,
      "rewards/rejected": -3.6477882862091064,
      "step": 315
    },
    {
      "epoch": 2.986666666666667,
      "step": 315,
      "total_flos": 0.0,
      "train_loss": 43.15912766380916,
      "train_runtime": 7320.6498,
      "train_samples_per_second": 2.766,
      "train_steps_per_second": 0.043
    }
  ],
  "logging_steps": 5,
  "max_steps": 315,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}