{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.286647611269906,
  "eval_steps": 500,
  "global_step": 700,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.016333197223356473,
      "grad_norm": 1.1593419313430786,
      "learning_rate": 4.9996340229869763e-05,
      "logits/chosen": -1.0346699953079224,
      "logits/rejected": -2.6695446968078613,
      "logps/chosen": -1.101169466972351,
      "logps/rejected": -1.0235556364059448,
      "loss": 1.178,
      "odds_ratio_loss": 0.768240213394165,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.11011694371700287,
      "rewards/margins": -0.007761377841234207,
      "rewards/rejected": -0.10235557705163956,
      "sft_loss": 1.101169466972351,
      "step": 5
    },
    {
      "epoch": 0.032666394446712946,
      "grad_norm": 1.1402696371078491,
      "learning_rate": 4.9985361990992455e-05,
      "logits/chosen": -0.8193758726119995,
      "logits/rejected": -2.842833995819092,
      "logps/chosen": -1.0217273235321045,
      "logps/rejected": -1.0100481510162354,
      "loss": 1.0927,
      "odds_ratio_loss": 0.7092453837394714,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.10217274725437164,
      "rewards/margins": -0.0011679243762046099,
      "rewards/rejected": -0.10100481659173965,
      "sft_loss": 1.0217273235321045,
      "step": 10
    },
    {
      "epoch": 0.04899959167006942,
      "grad_norm": 1.3554140329360962,
      "learning_rate": 4.996706849759453e-05,
      "logits/chosen": -0.7696830034255981,
      "logits/rejected": -2.703421115875244,
      "logps/chosen": -1.0294106006622314,
      "logps/rejected": -1.0599125623703003,
      "loss": 1.0993,
      "odds_ratio_loss": 0.6984036564826965,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.10294107347726822,
      "rewards/margins": 0.003050185739994049,
      "rewards/rejected": -0.10599125921726227,
      "sft_loss": 1.0294106006622314,
      "step": 15
    },
    {
      "epoch": 0.06533278889342589,
      "grad_norm": 1.406853199005127,
      "learning_rate": 4.9941465105674435e-05,
      "logits/chosen": -0.698488712310791,
      "logits/rejected": -2.808516263961792,
      "logps/chosen": -1.0651928186416626,
      "logps/rejected": -0.996777355670929,
      "loss": 1.1402,
      "odds_ratio_loss": 0.7504132390022278,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.10651928186416626,
      "rewards/margins": -0.006841552909463644,
      "rewards/rejected": -0.09967772662639618,
      "sft_loss": 1.0651928186416626,
      "step": 20
    },
    {
      "epoch": 0.08166598611678236,
      "grad_norm": 1.238513469696045,
      "learning_rate": 4.99085593114345e-05,
      "logits/chosen": -0.6306881904602051,
      "logits/rejected": -2.698362350463867,
      "logps/chosen": -1.0242538452148438,
      "logps/rejected": -1.0320734977722168,
      "loss": 1.0937,
      "odds_ratio_loss": 0.6945221424102783,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.10242538154125214,
      "rewards/margins": 0.0007819652673788369,
      "rewards/rejected": -0.10320734977722168,
      "sft_loss": 1.0242538452148438,
      "step": 25
    },
    {
      "epoch": 0.09799918334013884,
      "grad_norm": 1.9588699340820312,
      "learning_rate": 4.986836074908616e-05,
      "logits/chosen": -0.5534936189651489,
      "logits/rejected": -2.678889036178589,
      "logps/chosen": -1.1003000736236572,
      "logps/rejected": -1.0425399541854858,
      "loss": 1.1745,
      "odds_ratio_loss": 0.7416313290596008,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11003003269433975,
      "rewards/margins": -0.005776026751846075,
      "rewards/rejected": -0.10425399243831635,
      "sft_loss": 1.1003000736236572,
      "step": 30
    },
    {
      "epoch": 0.1143323805634953,
      "grad_norm": 1.1857821941375732,
      "learning_rate": 4.982088118802922e-05,
      "logits/chosen": -0.6129628419876099,
      "logits/rejected": -2.876364231109619,
      "logps/chosen": -0.9628928303718567,
      "logps/rejected": -0.952060341835022,
      "loss": 1.0336,
      "odds_ratio_loss": 0.7068274617195129,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.09628929197788239,
      "rewards/margins": -0.001083258306607604,
      "rewards/rejected": -0.09520602226257324,
      "sft_loss": 0.9628928303718567,
      "step": 35
    },
    {
      "epoch": 0.13066557778685178,
      "grad_norm": 1.33924400806427,
      "learning_rate": 4.976613452940604e-05,
      "logits/chosen": -0.7091814279556274,
      "logits/rejected": -2.6595664024353027,
      "logps/chosen": -0.89775550365448,
      "logps/rejected": -0.935952365398407,
      "loss": 0.9642,
      "odds_ratio_loss": 0.6644185781478882,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08977555483579636,
      "rewards/margins": 0.003819683101028204,
      "rewards/rejected": -0.0935952290892601,
      "sft_loss": 0.89775550365448,
      "step": 40
    },
    {
      "epoch": 0.14699877501020825,
      "grad_norm": 1.136667013168335,
      "learning_rate": 4.9704136802031485e-05,
      "logits/chosen": -0.7233263850212097,
      "logits/rejected": -2.6673388481140137,
      "logps/chosen": -0.9036429524421692,
      "logps/rejected": -0.9419530630111694,
      "loss": 0.9699,
      "odds_ratio_loss": 0.662254273891449,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09036429226398468,
      "rewards/margins": 0.0038310133386403322,
      "rewards/rejected": -0.09419531375169754,
      "sft_loss": 0.9036429524421692,
      "step": 45
    },
    {
      "epoch": 0.16333197223356472,
      "grad_norm": 1.4290399551391602,
      "learning_rate": 4.9634906157700036e-05,
      "logits/chosen": -0.6113058924674988,
      "logits/rejected": -2.705373764038086,
      "logps/chosen": -0.9550830125808716,
      "logps/rejected": -0.9817631840705872,
      "loss": 1.0223,
      "odds_ratio_loss": 0.6725824475288391,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.09550830721855164,
      "rewards/margins": 0.002668013097718358,
      "rewards/rejected": -0.09817632287740707,
      "sft_loss": 0.9550830125808716,
      "step": 50
    },
    {
      "epoch": 0.17966516945692118,
      "grad_norm": 1.4945589303970337,
      "learning_rate": 4.955846286587122e-05,
      "logits/chosen": -0.647710919380188,
      "logits/rejected": -2.7208316326141357,
      "logps/chosen": -0.9462896585464478,
      "logps/rejected": -0.9507497549057007,
      "loss": 1.0148,
      "odds_ratio_loss": 0.6852728128433228,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0946289673447609,
      "rewards/margins": 0.0004460157942958176,
      "rewards/rejected": -0.09507497400045395,
      "sft_loss": 0.9462896585464478,
      "step": 55
    },
    {
      "epoch": 0.19599836668027767,
      "grad_norm": 1.3553156852722168,
      "learning_rate": 4.9474829307735115e-05,
      "logits/chosen": -0.6279743909835815,
      "logits/rejected": -2.6854496002197266,
      "logps/chosen": -0.8896121978759766,
      "logps/rejected": -0.9993942975997925,
      "loss": 0.9509,
      "odds_ratio_loss": 0.6129180192947388,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.08896122127771378,
      "rewards/margins": 0.010978218168020248,
      "rewards/rejected": -0.09993942826986313,
      "sft_loss": 0.8896121978759766,
      "step": 60
    },
    {
      "epoch": 0.21233156390363414,
      "grad_norm": 1.4247411489486694,
      "learning_rate": 4.9384029969659573e-05,
      "logits/chosen": -0.698381781578064,
      "logits/rejected": -2.7967753410339355,
      "logps/chosen": -0.9455701112747192,
      "logps/rejected": -0.9564310908317566,
      "loss": 1.0147,
      "odds_ratio_loss": 0.6911824941635132,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0945570170879364,
      "rewards/margins": 0.0010860899928957224,
      "rewards/rejected": -0.09564311057329178,
      "sft_loss": 0.9455701112747192,
      "step": 65
    },
    {
      "epoch": 0.2286647611269906,
      "grad_norm": 1.6154508590698242,
      "learning_rate": 4.9286091436021015e-05,
      "logits/chosen": -0.595032274723053,
      "logits/rejected": -2.740784168243408,
      "logps/chosen": -0.9214372634887695,
      "logps/rejected": -1.0210167169570923,
      "loss": 0.9836,
      "odds_ratio_loss": 0.6212525963783264,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09214373677968979,
      "rewards/margins": 0.00995794590562582,
      "rewards/rejected": -0.10210166871547699,
      "sft_loss": 0.9214372634887695,
      "step": 70
    },
    {
      "epoch": 0.24499795835034707,
      "grad_norm": 1.8231102228164673,
      "learning_rate": 4.918104238142104e-05,
      "logits/chosen": -0.5186889171600342,
      "logits/rejected": -2.7576358318328857,
      "logps/chosen": -0.9789311289787292,
      "logps/rejected": -1.0169849395751953,
      "loss": 1.0454,
      "odds_ratio_loss": 0.6644267439842224,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09789310395717621,
      "rewards/margins": 0.0038053765892982483,
      "rewards/rejected": -0.10169849544763565,
      "sft_loss": 0.9789311289787292,
      "step": 75
    },
    {
      "epoch": 0.26133115557370357,
      "grad_norm": 2.05936336517334,
      "learning_rate": 4.906891356229103e-05,
      "logits/chosen": -0.47254711389541626,
      "logits/rejected": -2.68280029296875,
      "logps/chosen": -0.9620391726493835,
      "logps/rejected": -1.0026928186416626,
      "loss": 1.0294,
      "odds_ratio_loss": 0.6731926202774048,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.09620390832424164,
      "rewards/margins": 0.004065364599227905,
      "rewards/rejected": -0.10026928037405014,
      "sft_loss": 0.9620391726493835,
      "step": 80
    },
    {
      "epoch": 0.27766435279706003,
      "grad_norm": 1.6302626132965088,
      "learning_rate": 4.894973780788722e-05,
      "logits/chosen": -0.5315309762954712,
      "logits/rejected": -2.7227425575256348,
      "logps/chosen": -0.9423310160636902,
      "logps/rejected": -1.071781039237976,
      "loss": 1.0032,
      "odds_ratio_loss": 0.6090766191482544,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.09423311054706573,
      "rewards/margins": 0.012944999150931835,
      "rewards/rejected": -0.10717810690402985,
      "sft_loss": 0.9423310160636902,
      "step": 85
    },
    {
      "epoch": 0.2939975500204165,
      "grad_norm": 1.7891793251037598,
      "learning_rate": 4.882355001067892e-05,
      "logits/chosen": -0.5275685787200928,
      "logits/rejected": -2.6127567291259766,
      "logps/chosen": -0.9351356625556946,
      "logps/rejected": -1.0083874464035034,
      "loss": 0.9995,
      "odds_ratio_loss": 0.6435801386833191,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09351354837417603,
      "rewards/margins": 0.0073251971043646336,
      "rewards/rejected": -0.10083875805139542,
      "sft_loss": 0.9351356625556946,
      "step": 90
    },
    {
      "epoch": 0.31033074724377296,
      "grad_norm": 1.852216124534607,
      "learning_rate": 4.869038711613259e-05,
      "logits/chosen": -0.5760647058486938,
      "logits/rejected": -2.722364664077759,
      "logps/chosen": -0.9301847219467163,
      "logps/rejected": -1.0749822854995728,
      "loss": 0.9903,
      "odds_ratio_loss": 0.601297914981842,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09301847964525223,
      "rewards/margins": 0.014479748904705048,
      "rewards/rejected": -0.10749821364879608,
      "sft_loss": 0.9301847219467163,
      "step": 95
    },
    {
      "epoch": 0.32666394446712943,
      "grad_norm": 1.8597307205200195,
      "learning_rate": 4.855028811189496e-05,
      "logits/chosen": -0.5871164202690125,
      "logits/rejected": -2.6969285011291504,
      "logps/chosen": -1.0701930522918701,
      "logps/rejected": -1.0556358098983765,
      "loss": 1.1427,
      "odds_ratio_loss": 0.7247555255889893,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.10701930522918701,
      "rewards/margins": -0.0014557272661477327,
      "rewards/rejected": -0.10556356608867645,
      "sft_loss": 1.0701930522918701,
      "step": 100
    },
    {
      "epoch": 0.3429971416904859,
      "grad_norm": 1.6869758367538452,
      "learning_rate": 4.8403294016378095e-05,
      "logits/chosen": -0.4466473460197449,
      "logits/rejected": -2.501394510269165,
      "logps/chosen": -0.880301833152771,
      "logps/rejected": -1.0003767013549805,
      "loss": 0.941,
      "odds_ratio_loss": 0.60676509141922,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.08803018927574158,
      "rewards/margins": 0.012007484212517738,
      "rewards/rejected": -0.10003767162561417,
      "sft_loss": 0.880301833152771,
      "step": 105
    },
    {
      "epoch": 0.35933033891384236,
      "grad_norm": 1.8963792324066162,
      "learning_rate": 4.8249447866750025e-05,
      "logits/chosen": -0.41199809312820435,
      "logits/rejected": -2.594975471496582,
      "logps/chosen": -0.8528003692626953,
      "logps/rejected": -0.9772283434867859,
      "loss": 0.9128,
      "odds_ratio_loss": 0.6004671454429626,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.08528003096580505,
      "rewards/margins": 0.012442806735634804,
      "rewards/rejected": -0.0977228432893753,
      "sft_loss": 0.8528003692626953,
      "step": 110
    },
    {
      "epoch": 0.3756635361371989,
      "grad_norm": 2.6958725452423096,
      "learning_rate": 4.808879470633417e-05,
      "logits/chosen": -0.43985629081726074,
      "logits/rejected": -2.6999452114105225,
      "logps/chosen": -0.9486901164054871,
      "logps/rejected": -1.0987169742584229,
      "loss": 1.0076,
      "odds_ratio_loss": 0.5889819264411926,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.09486902505159378,
      "rewards/margins": 0.015002680011093616,
      "rewards/rejected": -0.10987170040607452,
      "sft_loss": 0.9486901164054871,
      "step": 115
    },
    {
      "epoch": 0.39199673336055535,
      "grad_norm": 1.6014004945755005,
      "learning_rate": 4.792138157142158e-05,
      "logits/chosen": -0.43066906929016113,
      "logits/rejected": -2.634669065475464,
      "logps/chosen": -0.8806023597717285,
      "logps/rejected": -1.0933107137680054,
      "loss": 0.9352,
      "odds_ratio_loss": 0.5455727577209473,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.08806024491786957,
      "rewards/margins": 0.021270830184221268,
      "rewards/rejected": -0.10933107137680054,
      "sft_loss": 0.8806023597717285,
      "step": 120
    },
    {
      "epoch": 0.4083299305839118,
      "grad_norm": 3.4042723178863525,
      "learning_rate": 4.7747257477499474e-05,
      "logits/chosen": -0.4240710735321045,
      "logits/rejected": -2.664668560028076,
      "logps/chosen": -0.8993316888809204,
      "logps/rejected": -1.0890860557556152,
      "loss": 0.9553,
      "odds_ratio_loss": 0.5601183772087097,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.08993317186832428,
      "rewards/margins": 0.018975432962179184,
      "rewards/rejected": -0.10890860855579376,
      "sft_loss": 0.8993316888809204,
      "step": 125
    },
    {
      "epoch": 0.4246631278072683,
      "grad_norm": 1.724076509475708,
      "learning_rate": 4.75664734049005e-05,
      "logits/chosen": -0.3910972476005554,
      "logits/rejected": -2.571296215057373,
      "logps/chosen": -0.9113162755966187,
      "logps/rejected": -1.094262719154358,
      "loss": 0.9682,
      "odds_ratio_loss": 0.5683911442756653,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09113162755966187,
      "rewards/margins": 0.018294643610715866,
      "rewards/rejected": -0.10942627489566803,
      "sft_loss": 0.9113162755966187,
      "step": 130
    },
    {
      "epoch": 0.44099632503062475,
      "grad_norm": 1.6363219022750854,
      "learning_rate": 4.7379082283876566e-05,
      "logits/chosen": -0.5021435022354126,
      "logits/rejected": -2.6572070121765137,
      "logps/chosen": -0.8906953930854797,
      "logps/rejected": -1.0344655513763428,
      "loss": 0.9498,
      "odds_ratio_loss": 0.5914202928543091,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08906953036785126,
      "rewards/margins": 0.014377007260918617,
      "rewards/rejected": -0.10344654321670532,
      "sft_loss": 0.8906953930854797,
      "step": 135
    },
    {
      "epoch": 0.4573295222539812,
      "grad_norm": 1.7812143564224243,
      "learning_rate": 4.7185138979101864e-05,
      "logits/chosen": -0.507682740688324,
      "logits/rejected": -2.593475580215454,
      "logps/chosen": -0.9167561531066895,
      "logps/rejected": -1.1608246564865112,
      "loss": 0.9698,
      "odds_ratio_loss": 0.5303933024406433,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.09167560935020447,
      "rewards/margins": 0.024406855925917625,
      "rewards/rejected": -0.11608246713876724,
      "sft_loss": 0.9167561531066895,
      "step": 140
    },
    {
      "epoch": 0.4736627194773377,
      "grad_norm": 1.823789119720459,
      "learning_rate": 4.6984700273609495e-05,
      "logits/chosen": -0.3446227014064789,
      "logits/rejected": -2.5670173168182373,
      "logps/chosen": -0.8484543561935425,
      "logps/rejected": -1.0493099689483643,
      "loss": 0.903,
      "odds_ratio_loss": 0.5457809567451477,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.08484544605016708,
      "rewards/margins": 0.020085560157895088,
      "rewards/rejected": -0.10493101179599762,
      "sft_loss": 0.8484543561935425,
      "step": 145
    },
    {
      "epoch": 0.48999591670069415,
      "grad_norm": 1.889367699623108,
      "learning_rate": 4.677782485216644e-05,
      "logits/chosen": -0.44083428382873535,
      "logits/rejected": -2.573638439178467,
      "logps/chosen": -0.833287239074707,
      "logps/rejected": -1.1050159931182861,
      "loss": 0.8838,
      "odds_ratio_loss": 0.5049694180488586,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0833287239074707,
      "rewards/margins": 0.02717287465929985,
      "rewards/rejected": -0.11050160229206085,
      "sft_loss": 0.833287239074707,
      "step": 150
    },
    {
      "epoch": 0.5063291139240507,
      "grad_norm": 1.9978969097137451,
      "learning_rate": 4.6564573284091765e-05,
      "logits/chosen": -0.37475016713142395,
      "logits/rejected": -2.530078887939453,
      "logps/chosen": -0.9055852890014648,
      "logps/rejected": -1.3165007829666138,
      "loss": 0.9498,
      "odds_ratio_loss": 0.4424561858177185,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.09055852890014648,
      "rewards/margins": 0.04109155014157295,
      "rewards/rejected": -0.13165007531642914,
      "sft_loss": 0.9055852890014648,
      "step": 155
    },
    {
      "epoch": 0.5226623111474071,
      "grad_norm": 2.2659225463867188,
      "learning_rate": 4.6345008005522966e-05,
      "logits/chosen": -0.27824097871780396,
      "logits/rejected": -2.1937553882598877,
      "logps/chosen": -0.7978348731994629,
      "logps/rejected": -1.7242333889007568,
      "loss": 0.8243,
      "odds_ratio_loss": 0.26432332396507263,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07978348433971405,
      "rewards/margins": 0.09263985604047775,
      "rewards/rejected": -0.1724233478307724,
      "sft_loss": 0.7978348731994629,
      "step": 160
    },
    {
      "epoch": 0.5389955083707636,
      "grad_norm": 2.497178792953491,
      "learning_rate": 4.611919330113592e-05,
      "logits/chosen": -0.1297352910041809,
      "logits/rejected": -1.7525418996810913,
      "logps/chosen": -0.9057181477546692,
      "logps/rejected": -2.926887035369873,
      "loss": 0.9163,
      "odds_ratio_loss": 0.10600633919239044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0905718058347702,
      "rewards/margins": 0.20211689174175262,
      "rewards/rejected": -0.2926887273788452,
      "sft_loss": 0.9057181477546692,
      "step": 165
    },
    {
      "epoch": 0.5553287055941201,
      "grad_norm": 1.9597887992858887,
      "learning_rate": 4.588719528532342e-05,
      "logits/chosen": -0.20320752263069153,
      "logits/rejected": -1.4767993688583374,
      "logps/chosen": -0.9343756437301636,
      "logps/rejected": -3.7731080055236816,
      "loss": 0.9406,
      "odds_ratio_loss": 0.062325619161129,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0934375673532486,
      "rewards/margins": 0.28387323021888733,
      "rewards/rejected": -0.3773108124732971,
      "sft_loss": 0.9343756437301636,
      "step": 170
    },
    {
      "epoch": 0.5716619028174765,
      "grad_norm": 1.9177374839782715,
      "learning_rate": 4.564908188283814e-05,
      "logits/chosen": -0.16985416412353516,
      "logits/rejected": -1.3144887685775757,
      "logps/chosen": -0.8834988474845886,
      "logps/rejected": -4.147261619567871,
      "loss": 0.8871,
      "odds_ratio_loss": 0.03579668328166008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08834987133741379,
      "rewards/margins": 0.3263762891292572,
      "rewards/rejected": -0.414726197719574,
      "sft_loss": 0.8834988474845886,
      "step": 175
    },
    {
      "epoch": 0.587995100040833,
      "grad_norm": 1.9115794897079468,
      "learning_rate": 4.540492280890555e-05,
      "logits/chosen": -0.34701991081237793,
      "logits/rejected": -1.1767631769180298,
      "logps/chosen": -0.8725343942642212,
      "logps/rejected": -4.597090721130371,
      "loss": 0.8753,
      "odds_ratio_loss": 0.027656957507133484,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08725343644618988,
      "rewards/margins": 0.3724556267261505,
      "rewards/rejected": -0.4597090780735016,
      "sft_loss": 0.8725343942642212,
      "step": 180
    },
    {
      "epoch": 0.6043282972641895,
      "grad_norm": 1.7303317785263062,
      "learning_rate": 4.515478954881248e-05,
      "logits/chosen": -0.37866610288619995,
      "logits/rejected": -1.20633065700531,
      "logps/chosen": -0.8602287173271179,
      "logps/rejected": -4.4016923904418945,
      "loss": 0.8625,
      "odds_ratio_loss": 0.022269519045948982,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08602287620306015,
      "rewards/margins": 0.35414639115333557,
      "rewards/rejected": -0.4401692748069763,
      "sft_loss": 0.8602287173271179,
      "step": 185
    },
    {
      "epoch": 0.6206614944875459,
      "grad_norm": 5.377653121948242,
      "learning_rate": 4.4898755336977673e-05,
      "logits/chosen": -0.32614630460739136,
      "logits/rejected": -1.1425362825393677,
      "logps/chosen": -0.9115999937057495,
      "logps/rejected": -4.731467247009277,
      "loss": 0.9149,
      "odds_ratio_loss": 0.0329320952296257,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09115998446941376,
      "rewards/margins": 0.38198670744895935,
      "rewards/rejected": -0.4731467366218567,
      "sft_loss": 0.9115999937057495,
      "step": 190
    },
    {
      "epoch": 0.6369946917109024,
      "grad_norm": 1.6315754652023315,
      "learning_rate": 4.463689513550997e-05,
      "logits/chosen": -0.3550247550010681,
      "logits/rejected": -1.241008996963501,
      "logps/chosen": -0.8556908369064331,
      "logps/rejected": -4.392051696777344,
      "loss": 0.8582,
      "odds_ratio_loss": 0.024758270010352135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08556908369064331,
      "rewards/margins": 0.35363608598709106,
      "rewards/rejected": -0.4392051696777344,
      "sft_loss": 0.8556908369064331,
      "step": 195
    },
    {
      "epoch": 0.6533278889342589,
      "grad_norm": 1.8591830730438232,
      "learning_rate": 4.436928561226087e-05,
      "logits/chosen": -0.3138996660709381,
      "logits/rejected": -1.2280880212783813,
      "logps/chosen": -0.901713490486145,
      "logps/rejected": -4.212186336517334,
      "loss": 0.9045,
      "odds_ratio_loss": 0.027884572744369507,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09017134457826614,
      "rewards/margins": 0.3310473561286926,
      "rewards/rejected": -0.4212186932563782,
      "sft_loss": 0.901713490486145,
      "step": 200
    },
    {
      "epoch": 0.6696610861576153,
      "grad_norm": 1.8346513509750366,
      "learning_rate": 4.409600511837756e-05,
      "logits/chosen": -0.23156340420246124,
      "logits/rejected": -1.138501524925232,
      "logps/chosen": -0.9389227032661438,
      "logps/rejected": -4.50938606262207,
      "loss": 0.9416,
      "odds_ratio_loss": 0.02684590592980385,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09389226883649826,
      "rewards/margins": 0.35704633593559265,
      "rewards/rejected": -0.4509386122226715,
      "sft_loss": 0.9389227032661438,
      "step": 205
    },
    {
      "epoch": 0.6859942833809718,
      "grad_norm": 1.5812890529632568,
      "learning_rate": 4.381713366536311e-05,
      "logits/chosen": -0.24586188793182373,
      "logits/rejected": -1.1232062578201294,
      "logps/chosen": -0.859118640422821,
      "logps/rejected": -4.883942127227783,
      "loss": 0.8609,
      "odds_ratio_loss": 0.018183354288339615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08591187745332718,
      "rewards/margins": 0.4024823307991028,
      "rewards/rejected": -0.48839420080184937,
      "sft_loss": 0.859118640422821,
      "step": 210
    },
    {
      "epoch": 0.7023274806043283,
      "grad_norm": 1.8375471830368042,
      "learning_rate": 4.353275290165066e-05,
      "logits/chosen": -0.15520325303077698,
      "logits/rejected": -0.985150933265686,
      "logps/chosen": -0.8214719891548157,
      "logps/rejected": -4.9630866050720215,
      "loss": 0.823,
      "odds_ratio_loss": 0.014944888651371002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08214719593524933,
      "rewards/margins": 0.4141615033149719,
      "rewards/rejected": -0.49630871415138245,
      "sft_loss": 0.8214719891548157,
      "step": 215
    },
    {
      "epoch": 0.7186606778276847,
      "grad_norm": 2.556330919265747,
      "learning_rate": 4.324294608869817e-05,
      "logits/chosen": -0.3221624791622162,
      "logits/rejected": -0.9542757272720337,
      "logps/chosen": -0.9679568409919739,
      "logps/rejected": -5.273629188537598,
      "loss": 0.9693,
      "odds_ratio_loss": 0.013113953173160553,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0967956930398941,
      "rewards/margins": 0.4305671751499176,
      "rewards/rejected": -0.5273628830909729,
      "sft_loss": 0.9679568409919739,
      "step": 220
    },
    {
      "epoch": 0.7349938750510412,
      "grad_norm": 2.346754312515259,
      "learning_rate": 4.294779807661105e-05,
      "logits/chosen": -0.2690557539463043,
      "logits/rejected": -1.0382682085037231,
      "logps/chosen": -0.8765946626663208,
      "logps/rejected": -5.27601432800293,
      "loss": 0.8783,
      "odds_ratio_loss": 0.016597110778093338,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08765946328639984,
      "rewards/margins": 0.43994197249412537,
      "rewards/rejected": -0.527601420879364,
      "sft_loss": 0.8765946626663208,
      "step": 225
    },
    {
      "epoch": 0.7513270722743978,
      "grad_norm": 1.543803095817566,
      "learning_rate": 4.270789202181204e-05,
      "logits/chosen": -0.3406456410884857,
      "logits/rejected": -1.0243726968765259,
      "logps/chosen": -0.938999354839325,
      "logps/rejected": -5.179265022277832,
      "loss": 0.9411,
      "odds_ratio_loss": 0.021242566406726837,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09389995038509369,
      "rewards/margins": 0.4240265488624573,
      "rewards/rejected": -0.5179265141487122,
      "sft_loss": 0.938999354839325,
      "step": 230
    },
    {
      "epoch": 0.7676602694977542,
      "grad_norm": 1.8233140707015991,
      "learning_rate": 4.2403348649073174e-05,
      "logits/chosen": -0.2546311020851135,
      "logits/rejected": -0.9541198015213013,
      "logps/chosen": -0.7934538722038269,
      "logps/rejected": -5.206686973571777,
      "loss": 0.7944,
      "odds_ratio_loss": 0.00939333625137806,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07934539020061493,
      "rewards/margins": 0.4413233697414398,
      "rewards/rejected": -0.5206687450408936,
      "sft_loss": 0.7934538722038269,
      "step": 235
    },
    {
      "epoch": 0.7839934667211107,
      "grad_norm": 1.5795395374298096,
      "learning_rate": 4.209370989589013e-05,
      "logits/chosen": -0.3499365448951721,
      "logits/rejected": -1.0148460865020752,
      "logps/chosen": -0.7999550104141235,
      "logps/rejected": -5.092876434326172,
      "loss": 0.8013,
      "odds_ratio_loss": 0.013519616797566414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07999551296234131,
      "rewards/margins": 0.4292922019958496,
      "rewards/rejected": -0.5092877149581909,
      "sft_loss": 0.7999550104141235,
      "step": 240
    },
    {
      "epoch": 0.8003266639444672,
      "grad_norm": 1.8533421754837036,
      "learning_rate": 4.177906641879575e-05,
      "logits/chosen": -0.3404558598995209,
      "logits/rejected": -1.0349606275558472,
      "logps/chosen": -0.9274562001228333,
      "logps/rejected": -4.853649616241455,
      "loss": 0.9298,
      "odds_ratio_loss": 0.023337265476584435,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09274561703205109,
      "rewards/margins": 0.3926192820072174,
      "rewards/rejected": -0.4853649139404297,
      "sft_loss": 0.9274562001228333,
      "step": 245
    },
    {
      "epoch": 0.8166598611678236,
      "grad_norm": 1.792864441871643,
      "learning_rate": 4.145951033961395e-05,
      "logits/chosen": -0.39237070083618164,
      "logits/rejected": -1.20218825340271,
      "logps/chosen": -0.8956495523452759,
      "logps/rejected": -4.416346549987793,
      "loss": 0.8988,
      "odds_ratio_loss": 0.03134030103683472,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08956494927406311,
      "rewards/margins": 0.3520697355270386,
      "rewards/rejected": -0.4416346549987793,
      "sft_loss": 0.8956495523452759,
      "step": 250
    },
    {
      "epoch": 0.8329930583911801,
      "grad_norm": 1.388491153717041,
      "learning_rate": 4.1135135218488206e-05,
      "logits/chosen": -0.4966661036014557,
      "logits/rejected": -1.301102638244629,
      "logps/chosen": -0.822670578956604,
      "logps/rejected": -4.420852184295654,
      "loss": 0.8251,
      "odds_ratio_loss": 0.02416636422276497,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08226706087589264,
      "rewards/margins": 0.3598182201385498,
      "rewards/rejected": -0.4420852065086365,
      "sft_loss": 0.822670578956604,
      "step": 255
    },
    {
      "epoch": 0.8493262556145366,
      "grad_norm": 1.7927485704421997,
      "learning_rate": 4.080603602648887e-05,
      "logits/chosen": -0.38711199164390564,
      "logits/rejected": -1.1762360334396362,
      "logps/chosen": -0.8747900128364563,
      "logps/rejected": -4.844551086425781,
      "loss": 0.8772,
      "odds_ratio_loss": 0.024142403155565262,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08747899532318115,
      "rewards/margins": 0.3969760835170746,
      "rewards/rejected": -0.4844551086425781,
      "sft_loss": 0.8747900128364563,
      "step": 260
    },
    {
      "epoch": 0.865659452837893,
      "grad_norm": 1.763637661933899,
      "learning_rate": 4.047230911780737e-05,
      "logits/chosen": -0.4611227512359619,
      "logits/rejected": -1.092810034751892,
      "logps/chosen": -0.8380341529846191,
      "logps/rejected": -4.902915000915527,
      "loss": 0.8414,
      "odds_ratio_loss": 0.03330729901790619,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08380343019962311,
      "rewards/margins": 0.4064881205558777,
      "rewards/rejected": -0.4902915358543396,
      "sft_loss": 0.8380341529846191,
      "step": 265
    },
    {
      "epoch": 0.8819926500612495,
      "grad_norm": 1.7203322649002075,
      "learning_rate": 4.0134052201545447e-05,
      "logits/chosen": -0.24241454899311066,
      "logits/rejected": -0.9308720827102661,
      "logps/chosen": -0.8675628900527954,
      "logps/rejected": -5.390278339385986,
      "loss": 0.8685,
      "odds_ratio_loss": 0.009246011264622211,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08675628900527954,
      "rewards/margins": 0.4522715210914612,
      "rewards/rejected": -0.5390278100967407,
      "sft_loss": 0.8675628900527954,
      "step": 270
    },
    {
      "epoch": 0.898325847284606,
      "grad_norm": 1.9116899967193604,
      "learning_rate": 3.9791364313107804e-05,
      "logits/chosen": -0.401073157787323,
      "logits/rejected": -0.9959608912467957,
      "logps/chosen": -0.8706862330436707,
      "logps/rejected": -5.204241752624512,
      "loss": 0.872,
      "odds_ratio_loss": 0.0133589506149292,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08706863224506378,
      "rewards/margins": 0.4333556294441223,
      "rewards/rejected": -0.5204241871833801,
      "sft_loss": 0.8706862330436707,
      "step": 275
    },
    {
      "epoch": 0.9146590445079624,
      "grad_norm": 1.9296520948410034,
      "learning_rate": 3.9444345785206285e-05,
      "logits/chosen": -0.40887147188186646,
      "logits/rejected": -0.9652082324028015,
      "logps/chosen": -0.8309628367424011,
      "logps/rejected": -5.662296295166016,
      "loss": 0.8318,
      "odds_ratio_loss": 0.008708246983587742,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08309628069400787,
      "rewards/margins": 0.48313331604003906,
      "rewards/rejected": -0.5662296414375305,
      "sft_loss": 0.8309628367424011,
      "step": 280
    },
    {
      "epoch": 0.9309922417313189,
      "grad_norm": 3.055554151535034,
      "learning_rate": 3.909309821848432e-05,
      "logits/chosen": -0.3440246284008026,
      "logits/rejected": -0.8914414644241333,
      "logps/chosen": -0.8914461135864258,
      "logps/rejected": -5.75334358215332,
      "loss": 0.8921,
      "odds_ratio_loss": 0.0068351454101502895,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08914461731910706,
      "rewards/margins": 0.48618975281715393,
      "rewards/rejected": -0.575334370136261,
      "sft_loss": 0.8914461135864258,
      "step": 285
    },
    {
      "epoch": 0.9473254389546754,
      "grad_norm": 1.5384926795959473,
      "learning_rate": 3.873772445177015e-05,
      "logits/chosen": -0.37930798530578613,
      "logits/rejected": -0.9451101422309875,
      "logps/chosen": -0.8741447329521179,
      "logps/rejected": -5.84790563583374,
      "loss": 0.8751,
      "odds_ratio_loss": 0.009465084411203861,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08741448074579239,
      "rewards/margins": 0.49737611413002014,
      "rewards/rejected": -0.5847905874252319,
      "sft_loss": 0.8741447329521179,
      "step": 290
    },
    {
      "epoch": 0.9636586361780318,
      "grad_norm": 2.4715216159820557,
      "learning_rate": 3.837832853196751e-05,
      "logits/chosen": -0.3153218626976013,
      "logits/rejected": -0.9612942934036255,
      "logps/chosen": -0.8935874700546265,
      "logps/rejected": -5.140650749206543,
      "loss": 0.8956,
      "odds_ratio_loss": 0.01970650628209114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08935876190662384,
      "rewards/margins": 0.4247063100337982,
      "rewards/rejected": -0.514065146446228,
      "sft_loss": 0.8935874700546265,
      "step": 295
    },
    {
      "epoch": 0.9799918334013883,
      "grad_norm": 1.9313322305679321,
      "learning_rate": 3.8015015683592575e-05,
      "logits/chosen": -0.30432718992233276,
      "logits/rejected": -1.099277377128601,
      "logps/chosen": -0.8711298108100891,
      "logps/rejected": -4.722751140594482,
      "loss": 0.873,
      "odds_ratio_loss": 0.018964512273669243,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08711298555135727,
      "rewards/margins": 0.3851621747016907,
      "rewards/rejected": -0.47227516770362854,
      "sft_loss": 0.8711298108100891,
      "step": 300
    },
    {
      "epoch": 0.9963250306247448,
      "grad_norm": 2.2331244945526123,
      "learning_rate": 3.764789227796616e-05,
      "logits/chosen": -0.3758210837841034,
      "logits/rejected": -1.1826483011245728,
      "logps/chosen": -0.8142495155334473,
      "logps/rejected": -4.7738471031188965,
      "loss": 0.816,
      "odds_ratio_loss": 0.017485955730080605,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08142495155334473,
      "rewards/margins": 0.3959597945213318,
      "rewards/rejected": -0.4773847460746765,
      "sft_loss": 0.8142495155334473,
      "step": 305
    },
    {
      "epoch": 1.0126582278481013,
      "grad_norm": 1.7077336311340332,
      "learning_rate": 3.727706580207021e-05,
      "logits/chosen": -0.38093143701553345,
      "logits/rejected": -1.096470594406128,
      "logps/chosen": -0.8485842943191528,
      "logps/rejected": -4.818546295166016,
      "loss": 0.8503,
      "odds_ratio_loss": 0.01746557094156742,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08485842496156693,
      "rewards/margins": 0.3969961702823639,
      "rewards/rejected": -0.4818546175956726,
      "sft_loss": 0.8485842943191528,
      "step": 310
    },
    {
      "epoch": 1.0289914250714578,
      "grad_norm": 1.8160737752914429,
      "learning_rate": 3.69026448270775e-05,
      "logits/chosen": -0.37943610548973083,
      "logits/rejected": -1.0238980054855347,
      "logps/chosen": -0.8509234189987183,
      "logps/rejected": -5.104341506958008,
      "loss": 0.8521,
      "odds_ratio_loss": 0.011288631707429886,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08509235084056854,
      "rewards/margins": 0.4253418445587158,
      "rewards/rejected": -0.5104341506958008,
      "sft_loss": 0.8509234189987183,
      "step": 315
    },
    {
      "epoch": 1.0453246222948143,
      "grad_norm": 2.9576852321624756,
      "learning_rate": 3.652473897656409e-05,
      "logits/chosen": -0.30405062437057495,
      "logits/rejected": -0.9463305473327637,
      "logps/chosen": -0.9074727296829224,
      "logps/rejected": -5.486522674560547,
      "loss": 0.9089,
      "odds_ratio_loss": 0.013948513194918633,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09074728190898895,
      "rewards/margins": 0.45790499448776245,
      "rewards/rejected": -0.5486522912979126,
      "sft_loss": 0.9074727296829224,
      "step": 320
    },
    {
      "epoch": 1.0616578195181707,
      "grad_norm": 1.9609084129333496,
      "learning_rate": 3.6143458894413465e-05,
      "logits/chosen": -0.39042356610298157,
      "logits/rejected": -1.0480425357818604,
      "logps/chosen": -0.7837303876876831,
      "logps/rejected": -5.29976749420166,
      "loss": 0.7846,
      "odds_ratio_loss": 0.008555177599191666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0783730298280716,
      "rewards/margins": 0.45160380005836487,
      "rewards/rejected": -0.5299768447875977,
      "sft_loss": 0.7837303876876831,
      "step": 325
    },
    {
      "epoch": 1.0779910167415272,
      "grad_norm": 2.609851360321045,
      "learning_rate": 3.575891621242209e-05,
      "logits/chosen": -0.35802769660949707,
      "logits/rejected": -0.9912473559379578,
      "logps/chosen": -0.8010882139205933,
      "logps/rejected": -5.255009651184082,
      "loss": 0.8022,
      "odds_ratio_loss": 0.010739787481725216,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08010882139205933,
      "rewards/margins": 0.4453921914100647,
      "rewards/rejected": -0.525501012802124,
      "sft_loss": 0.8010882139205933,
      "step": 330
    },
    {
      "epoch": 1.0943242139648837,
      "grad_norm": 1.9164317846298218,
      "learning_rate": 3.537122351761568e-05,
      "logits/chosen": -0.29361772537231445,
      "logits/rejected": -0.9894407987594604,
      "logps/chosen": -0.8663752675056458,
      "logps/rejected": -5.1811041831970215,
      "loss": 0.8682,
      "odds_ratio_loss": 0.01779218763113022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08663753420114517,
      "rewards/margins": 0.43147286772727966,
      "rewards/rejected": -0.5181103944778442,
      "sft_loss": 0.8663752675056458,
      "step": 335
    },
    {
      "epoch": 1.1106574111882401,
      "grad_norm": 1.8551751375198364,
      "learning_rate": 3.4980494319285775e-05,
      "logits/chosen": -0.2934134900569916,
      "logits/rejected": -0.9298968315124512,
      "logps/chosen": -0.8729950189590454,
      "logps/rejected": -5.402509689331055,
      "loss": 0.8739,
      "odds_ratio_loss": 0.009193124249577522,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08729950338602066,
      "rewards/margins": 0.45295149087905884,
      "rewards/rejected": -0.5402510166168213,
      "sft_loss": 0.8729950189590454,
      "step": 340
    },
    {
      "epoch": 1.1269906084115966,
      "grad_norm": 1.8746849298477173,
      "learning_rate": 3.45868430157563e-05,
      "logits/chosen": -0.27233654260635376,
      "logits/rejected": -0.9365426301956177,
      "logps/chosen": -0.8479824066162109,
      "logps/rejected": -5.225064754486084,
      "loss": 0.8493,
      "odds_ratio_loss": 0.013331398367881775,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08479823917150497,
      "rewards/margins": 0.43770831823349,
      "rewards/rejected": -0.522506594657898,
      "sft_loss": 0.8479824066162109,
      "step": 345
    },
    {
      "epoch": 1.143323805634953,
      "grad_norm": 1.7111650705337524,
      "learning_rate": 3.4190384860889824e-05,
      "logits/chosen": -0.366852343082428,
      "logits/rejected": -1.0381531715393066,
      "logps/chosen": -0.8182533383369446,
      "logps/rejected": -5.278301239013672,
      "loss": 0.8204,
      "odds_ratio_loss": 0.02150268480181694,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08182532340288162,
      "rewards/margins": 0.4460047781467438,
      "rewards/rejected": -0.5278300642967224,
      "sft_loss": 0.8182533383369446,
      "step": 350
    },
    {
      "epoch": 1.1596570028583095,
      "grad_norm": 1.9346224069595337,
      "learning_rate": 3.379123593034342e-05,
      "logits/chosen": -0.2924169898033142,
      "logits/rejected": -1.028192400932312,
      "logps/chosen": -0.9392467737197876,
      "logps/rejected": -5.247817516326904,
      "loss": 0.9409,
      "odds_ratio_loss": 0.0163104310631752,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09392467886209488,
      "rewards/margins": 0.4308570921421051,
      "rewards/rejected": -0.5247817635536194,
      "sft_loss": 0.9392467737197876,
      "step": 355
    },
    {
      "epoch": 1.175990200081666,
      "grad_norm": 1.9083410501480103,
      "learning_rate": 3.3389513087583754e-05,
      "logits/chosen": -0.1519567221403122,
      "logits/rejected": -1.0742218494415283,
      "logps/chosen": -0.8435864448547363,
      "logps/rejected": -4.644381523132324,
      "loss": 0.8465,
      "odds_ratio_loss": 0.028982138261198997,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08435864001512527,
      "rewards/margins": 0.3800795078277588,
      "rewards/rejected": -0.4644381105899811,
      "sft_loss": 0.8435864448547363,
      "step": 360
    },
    {
      "epoch": 1.1923233973050225,
      "grad_norm": 1.8590240478515625,
      "learning_rate": 3.2985333949671685e-05,
      "logits/chosen": -0.43951839208602905,
      "logits/rejected": -1.1476421356201172,
      "logps/chosen": -0.8573881387710571,
      "logps/rejected": -4.641860485076904,
      "loss": 0.8605,
      "odds_ratio_loss": 0.030726104974746704,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08573882281780243,
      "rewards/margins": 0.3784472346305847,
      "rewards/rejected": -0.4641861021518707,
      "sft_loss": 0.8573881387710571,
      "step": 365
    },
    {
      "epoch": 1.208656594528379,
      "grad_norm": 2.064635753631592,
      "learning_rate": 3.257881685282609e-05,
      "logits/chosen": -0.2586503028869629,
      "logits/rejected": -1.1886835098266602,
      "logps/chosen": -0.7855366468429565,
      "logps/rejected": -4.374009132385254,
      "loss": 0.7886,
      "odds_ratio_loss": 0.03049846924841404,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07855366170406342,
      "rewards/margins": 0.3588472604751587,
      "rewards/rejected": -0.4374009668827057,
      "sft_loss": 0.7855366468429565,
      "step": 370
    },
    {
      "epoch": 1.2249897917517354,
      "grad_norm": 1.986666202545166,
      "learning_rate": 3.217008081777726e-05,
      "logits/chosen": -0.3716577887535095,
      "logits/rejected": -1.2067687511444092,
      "logps/chosen": -0.8298207521438599,
      "logps/rejected": -4.59980583190918,
      "loss": 0.8315,
      "odds_ratio_loss": 0.016955239698290825,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08298207074403763,
      "rewards/margins": 0.3769984841346741,
      "rewards/rejected": -0.4599805772304535,
      "sft_loss": 0.8298207521438599,
      "step": 375
    },
    {
      "epoch": 1.2413229889750919,
      "grad_norm": 1.8263311386108398,
      "learning_rate": 3.175924551491977e-05,
      "logits/chosen": -0.3963642716407776,
      "logits/rejected": -1.1271188259124756,
      "logps/chosen": -0.8212121725082397,
      "logps/rejected": -4.6982741355896,
      "loss": 0.8234,
      "odds_ratio_loss": 0.02219826728105545,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08212121576070786,
      "rewards/margins": 0.3877061605453491,
      "rewards/rejected": -0.4698273539543152,
      "sft_loss": 0.8212121725082397,
      "step": 380
    },
    {
      "epoch": 1.2576561861984483,
      "grad_norm": 1.946442723274231,
      "learning_rate": 3.13464312292752e-05,
      "logits/chosen": -0.343479186296463,
      "logits/rejected": -0.9710591435432434,
      "logps/chosen": -0.8622458577156067,
      "logps/rejected": -5.370631217956543,
      "loss": 0.8646,
      "odds_ratio_loss": 0.023645440116524696,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08622457832098007,
      "rewards/margins": 0.450838565826416,
      "rewards/rejected": -0.5370631217956543,
      "sft_loss": 0.8622458577156067,
      "step": 385
    },
    {
      "epoch": 1.2739893834218048,
      "grad_norm": 1.7212151288986206,
      "learning_rate": 3.093175882527491e-05,
      "logits/chosen": -0.3060234785079956,
      "logits/rejected": -1.022698163986206,
      "logps/chosen": -0.8830709457397461,
      "logps/rejected": -5.35630464553833,
      "loss": 0.8848,
      "odds_ratio_loss": 0.01716751977801323,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08830709010362625,
      "rewards/margins": 0.44732338190078735,
      "rewards/rejected": -0.535630464553833,
      "sft_loss": 0.8830709457397461,
      "step": 390
    },
    {
      "epoch": 1.2903225806451613,
      "grad_norm": 2.029407024383545,
      "learning_rate": 3.0515349711373146e-05,
      "logits/chosen": -0.13591887056827545,
      "logits/rejected": -0.9809386134147644,
      "logps/chosen": -0.880872368812561,
      "logps/rejected": -5.28397798538208,
      "loss": 0.8826,
      "odds_ratio_loss": 0.01720813475549221,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08808723837137222,
      "rewards/margins": 0.4403105676174164,
      "rewards/rejected": -0.5283978581428528,
      "sft_loss": 0.880872368812561,
      "step": 395
    },
    {
      "epoch": 1.3066557778685177,
      "grad_norm": 1.8320516347885132,
      "learning_rate": 3.0097325804500864e-05,
      "logits/chosen": -0.3814406991004944,
      "logits/rejected": -1.0283949375152588,
      "logps/chosen": -0.7927986979484558,
      "logps/rejected": -5.100050926208496,
      "loss": 0.7945,
      "odds_ratio_loss": 0.016741547733545303,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07927986979484558,
      "rewards/margins": 0.4307252764701843,
      "rewards/rejected": -0.5100051164627075,
      "sft_loss": 0.7927986979484558,
      "step": 400
    },
    {
      "epoch": 1.3229889750918742,
      "grad_norm": 10.229716300964355,
      "learning_rate": 2.9677809494370712e-05,
      "logits/chosen": -0.3555116057395935,
      "logits/rejected": -0.966505229473114,
      "logps/chosen": -0.9298495054244995,
      "logps/rejected": -5.674726963043213,
      "loss": 0.9316,
      "odds_ratio_loss": 0.017632385715842247,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09298495203256607,
      "rewards/margins": 0.4744877219200134,
      "rewards/rejected": -0.5674726963043213,
      "sft_loss": 0.9298495054244995,
      "step": 405
    },
    {
      "epoch": 1.3393221723152307,
      "grad_norm": 2.0804049968719482,
      "learning_rate": 2.925692360764356e-05,
      "logits/chosen": -0.2548617124557495,
      "logits/rejected": -1.0827587842941284,
      "logps/chosen": -0.8656631708145142,
      "logps/rejected": -4.993069648742676,
      "loss": 0.8675,
      "odds_ratio_loss": 0.018203863874077797,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08656631410121918,
      "rewards/margins": 0.41274064779281616,
      "rewards/rejected": -0.4993070662021637,
      "sft_loss": 0.8656631708145142,
      "step": 410
    },
    {
      "epoch": 1.3556553695385871,
      "grad_norm": 2.4393317699432373,
      "learning_rate": 2.8834791371967142e-05,
      "logits/chosen": -0.18482331931591034,
      "logits/rejected": -1.0267010927200317,
      "logps/chosen": -0.868569016456604,
      "logps/rejected": -5.216423511505127,
      "loss": 0.8702,
      "odds_ratio_loss": 0.01657196693122387,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0868569016456604,
      "rewards/margins": 0.4347854554653168,
      "rewards/rejected": -0.5216423869132996,
      "sft_loss": 0.868569016456604,
      "step": 415
    },
    {
      "epoch": 1.3719885667619436,
      "grad_norm": 1.7740427255630493,
      "learning_rate": 2.8411536379897203e-05,
      "logits/chosen": -0.27949684858322144,
      "logits/rejected": -1.0097931623458862,
      "logps/chosen": -0.8230732083320618,
      "logps/rejected": -5.000334739685059,
      "loss": 0.8254,
      "odds_ratio_loss": 0.023418055847287178,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08230730891227722,
      "rewards/margins": 0.4177261292934418,
      "rewards/rejected": -0.5000334978103638,
      "sft_loss": 0.8230732083320618,
      "step": 420
    },
    {
      "epoch": 1.3883217639853,
      "grad_norm": 1.8371398448944092,
      "learning_rate": 2.7987282552711973e-05,
      "logits/chosen": -0.32712841033935547,
      "logits/rejected": -1.0544841289520264,
      "logps/chosen": -0.8350850939750671,
      "logps/rejected": -5.021151065826416,
      "loss": 0.837,
      "odds_ratio_loss": 0.01911192014813423,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08350850641727448,
      "rewards/margins": 0.41860657930374146,
      "rewards/rejected": -0.5021151304244995,
      "sft_loss": 0.8350850939750671,
      "step": 425
    },
    {
      "epoch": 1.4046549612086565,
      "grad_norm": 2.2567172050476074,
      "learning_rate": 2.7562154104130178e-05,
      "logits/chosen": -0.39351749420166016,
      "logits/rejected": -1.124728798866272,
      "logps/chosen": -0.7841497659683228,
      "logps/rejected": -5.364629745483398,
      "loss": 0.7859,
      "odds_ratio_loss": 0.017894607037305832,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07841498404741287,
      "rewards/margins": 0.458048015832901,
      "rewards/rejected": -0.5364629626274109,
      "sft_loss": 0.7841497659683228,
      "step": 430
    },
    {
      "epoch": 1.4209881584320132,
      "grad_norm": 2.822150945663452,
      "learning_rate": 2.7136275503943626e-05,
      "logits/chosen": -0.14898474514484406,
      "logits/rejected": -0.778265655040741,
      "logps/chosen": -0.8347780108451843,
      "logps/rejected": -5.595808506011963,
      "loss": 0.8356,
      "odds_ratio_loss": 0.007863040082156658,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08347781002521515,
      "rewards/margins": 0.4761030673980713,
      "rewards/rejected": -0.55958092212677,
      "sft_loss": 0.8347780108451843,
      "step": 435
    },
    {
      "epoch": 1.4373213556553694,
      "grad_norm": 1.9301971197128296,
      "learning_rate": 2.670977144157475e-05,
      "logits/chosen": -0.3294993042945862,
      "logits/rejected": -0.8833429217338562,
      "logps/chosen": -0.8435837030410767,
      "logps/rejected": -5.550654888153076,
      "loss": 0.845,
      "odds_ratio_loss": 0.013694020919501781,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08435836434364319,
      "rewards/margins": 0.47070708870887756,
      "rewards/rejected": -0.5550654530525208,
      "sft_loss": 0.8435837030410767,
      "step": 440
    },
    {
      "epoch": 1.4536545528787261,
      "grad_norm": 1.864906907081604,
      "learning_rate": 2.628276678956974e-05,
      "logits/chosen": -0.38186246156692505,
      "logits/rejected": -0.9432054758071899,
      "logps/chosen": -0.7808457612991333,
      "logps/rejected": -5.465119361877441,
      "loss": 0.7817,
      "odds_ratio_loss": 0.008780199103057384,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07808457314968109,
      "rewards/margins": 0.4684273600578308,
      "rewards/rejected": -0.5465119481086731,
      "sft_loss": 0.7808457612991333,
      "step": 445
    },
    {
      "epoch": 1.4699877501020824,
      "grad_norm": 1.652700424194336,
      "learning_rate": 2.5855386567038304e-05,
      "logits/chosen": -0.2546243369579315,
      "logits/rejected": -0.8346729278564453,
      "logps/chosen": -0.8080782890319824,
      "logps/rejected": -5.827437400817871,
      "loss": 0.8087,
      "odds_ratio_loss": 0.005900132469832897,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08080783486366272,
      "rewards/margins": 0.5019358992576599,
      "rewards/rejected": -0.582743763923645,
      "sft_loss": 0.8080782890319824,
      "step": 450
    },
    {
      "epoch": 1.486320947325439,
      "grad_norm": 1.7943625450134277,
      "learning_rate": 2.5427755903050232e-05,
      "logits/chosen": -0.3007858395576477,
      "logits/rejected": -0.751599907875061,
      "logps/chosen": -0.8181053400039673,
      "logps/rejected": -5.938103199005127,
      "loss": 0.8187,
      "odds_ratio_loss": 0.005451642908155918,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08181053400039673,
      "rewards/margins": 0.511999785900116,
      "rewards/rejected": -0.5938103199005127,
      "sft_loss": 0.8181053400039673,
      "step": 455
    },
    {
      "epoch": 1.5026541445487953,
      "grad_norm": 1.9405500888824463,
      "learning_rate": 2.5e-05,
      "logits/chosen": -0.3018740713596344,
      "logits/rejected": -0.8147015571594238,
      "logps/chosen": -0.8223198652267456,
      "logps/rejected": -6.007750034332275,
      "loss": 0.8231,
      "odds_ratio_loss": 0.007952779531478882,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08223198354244232,
      "rewards/margins": 0.518543004989624,
      "rewards/rejected": -0.6007749438285828,
      "sft_loss": 0.8223198652267456,
      "step": 460
    },
    {
      "epoch": 1.518987341772152,
      "grad_norm": 29.944835662841797,
      "learning_rate": 2.4572244096949774e-05,
      "logits/chosen": -0.3322751522064209,
      "logits/rejected": -0.8299162983894348,
      "logps/chosen": -0.8652318120002747,
      "logps/rejected": -5.8731489181518555,
      "loss": 0.8665,
      "odds_ratio_loss": 0.012552870437502861,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08652317523956299,
      "rewards/margins": 0.5007917881011963,
      "rewards/rejected": -0.5873149633407593,
      "sft_loss": 0.8652318120002747,
      "step": 465
    },
    {
      "epoch": 1.5353205389955082,
      "grad_norm": 4.769914150238037,
      "learning_rate": 2.41446134329617e-05,
      "logits/chosen": -0.30929869413375854,
      "logits/rejected": -0.7486665844917297,
      "logps/chosen": -0.8595030903816223,
      "logps/rejected": -6.054928779602051,
      "loss": 0.8602,
      "odds_ratio_loss": 0.007209992501884699,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08595031499862671,
      "rewards/margins": 0.5195425748825073,
      "rewards/rejected": -0.6054929494857788,
      "sft_loss": 0.8595030903816223,
      "step": 470
    },
    {
      "epoch": 1.551653736218865,
      "grad_norm": 2.072922468185425,
      "learning_rate": 2.3717233210430256e-05,
      "logits/chosen": -0.3498794436454773,
      "logits/rejected": -0.9254905581474304,
      "logps/chosen": -0.836539626121521,
      "logps/rejected": -5.841341495513916,
      "loss": 0.8373,
      "odds_ratio_loss": 0.007826647721230984,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08365396410226822,
      "rewards/margins": 0.5004801750183105,
      "rewards/rejected": -0.5841341018676758,
      "sft_loss": 0.836539626121521,
      "step": 475
    },
    {
      "epoch": 1.5679869334422212,
      "grad_norm": 2.006734609603882,
      "learning_rate": 2.3290228558425264e-05,
      "logits/chosen": -0.21343150734901428,
      "logits/rejected": -0.997966468334198,
      "logps/chosen": -0.8389822244644165,
      "logps/rejected": -5.127771854400635,
      "loss": 0.8408,
      "odds_ratio_loss": 0.018000155687332153,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08389821648597717,
      "rewards/margins": 0.4288790225982666,
      "rewards/rejected": -0.5127772092819214,
      "sft_loss": 0.8389822244644165,
      "step": 480
    },
    {
      "epoch": 1.5843201306655779,
      "grad_norm": 2.0465636253356934,
      "learning_rate": 2.2863724496056377e-05,
      "logits/chosen": -0.36094948649406433,
      "logits/rejected": -1.0251775979995728,
      "logps/chosen": -0.8593729138374329,
      "logps/rejected": -5.164275169372559,
      "loss": 0.8608,
      "odds_ratio_loss": 0.014115150086581707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08593729883432388,
      "rewards/margins": 0.43049025535583496,
      "rewards/rejected": -0.5164275169372559,
      "sft_loss": 0.8593729138374329,
      "step": 485
    },
    {
      "epoch": 1.600653327888934,
      "grad_norm": 1.6602107286453247,
      "learning_rate": 2.2437845895869828e-05,
      "logits/chosen": -0.35683882236480713,
      "logits/rejected": -1.1014772653579712,
      "logps/chosen": -0.8006218671798706,
      "logps/rejected": -4.8285603523254395,
      "loss": 0.8022,
      "odds_ratio_loss": 0.01606990396976471,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08006219565868378,
      "rewards/margins": 0.402793824672699,
      "rewards/rejected": -0.48285597562789917,
      "sft_loss": 0.8006218671798706,
      "step": 490
    },
    {
      "epoch": 1.6169865251122908,
      "grad_norm": 2.5427048206329346,
      "learning_rate": 2.2012717447288033e-05,
      "logits/chosen": -0.3278926610946655,
      "logits/rejected": -1.0268025398254395,
      "logps/chosen": -0.8360313177108765,
      "logps/rejected": -4.992895126342773,
      "loss": 0.8377,
      "odds_ratio_loss": 0.017158646136522293,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0836031436920166,
      "rewards/margins": 0.41568630933761597,
      "rewards/rejected": -0.49928945302963257,
      "sft_loss": 0.8360313177108765,
      "step": 495
    },
    {
      "epoch": 1.633319722335647,
      "grad_norm": 2.533067226409912,
      "learning_rate": 2.1588463620102796e-05,
      "logits/chosen": -0.3250111937522888,
      "logits/rejected": -0.9380800127983093,
      "logps/chosen": -0.7811130881309509,
      "logps/rejected": -5.476039409637451,
      "loss": 0.7821,
      "odds_ratio_loss": 0.009653602726757526,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07811131328344345,
      "rewards/margins": 0.46949267387390137,
      "rewards/rejected": -0.547603964805603,
      "sft_loss": 0.7811130881309509,
      "step": 500
    },
    {
      "epoch": 1.6496529195590037,
      "grad_norm": 2.0488734245300293,
      "learning_rate": 2.116520862803286e-05,
      "logits/chosen": -0.21985015273094177,
      "logits/rejected": -0.9144424200057983,
      "logps/chosen": -0.7239242792129517,
      "logps/rejected": -5.2707695960998535,
      "loss": 0.726,
      "odds_ratio_loss": 0.0210262518376112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07239242643117905,
      "rewards/margins": 0.4546845555305481,
      "rewards/rejected": -0.5270770192146301,
      "sft_loss": 0.7239242792129517,
      "step": 505
    },
    {
      "epoch": 1.6659861167823602,
      "grad_norm": 1.8625127077102661,
      "learning_rate": 2.074307639235645e-05,
      "logits/chosen": -0.16022160649299622,
      "logits/rejected": -0.7577518224716187,
      "logps/chosen": -0.8772943615913391,
      "logps/rejected": -5.823426246643066,
      "loss": 0.8781,
      "odds_ratio_loss": 0.007620103657245636,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08772943168878555,
      "rewards/margins": 0.49461308121681213,
      "rewards/rejected": -0.5823425650596619,
      "sft_loss": 0.8772943615913391,
      "step": 510
    },
    {
      "epoch": 1.6823193140057167,
      "grad_norm": 2.045032024383545,
      "learning_rate": 2.0322190505629297e-05,
      "logits/chosen": -0.3373351991176605,
      "logits/rejected": -0.8180400729179382,
      "logps/chosen": -0.7550835013389587,
      "logps/rejected": -6.03420877456665,
      "loss": 0.7556,
      "odds_ratio_loss": 0.004965050611644983,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07550835609436035,
      "rewards/margins": 0.5279124975204468,
      "rewards/rejected": -0.6034208536148071,
      "sft_loss": 0.7550835013389587,
      "step": 515
    },
    {
      "epoch": 1.6986525112290731,
      "grad_norm": 2.0969460010528564,
      "learning_rate": 1.9902674195499145e-05,
      "logits/chosen": -0.16994231939315796,
      "logits/rejected": -0.7098522782325745,
      "logps/chosen": -0.8867838978767395,
      "logps/rejected": -6.680023193359375,
      "loss": 0.8871,
      "odds_ratio_loss": 0.0027744858525693417,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08867839723825455,
      "rewards/margins": 0.5793240070343018,
      "rewards/rejected": -0.6680023670196533,
      "sft_loss": 0.8867838978767395,
      "step": 520
    },
    {
      "epoch": 1.7149857084524296,
      "grad_norm": 2.6975438594818115,
      "learning_rate": 1.948465028862686e-05,
      "logits/chosen": -0.2635698616504669,
      "logits/rejected": -0.7826798558235168,
      "logps/chosen": -0.8126001358032227,
      "logps/rejected": -6.0560736656188965,
      "loss": 0.8132,
      "odds_ratio_loss": 0.005659271962940693,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08126001060009003,
      "rewards/margins": 0.5243473052978516,
      "rewards/rejected": -0.6056073904037476,
      "sft_loss": 0.8126001358032227,
      "step": 525
    },
    {
      "epoch": 1.731318905675786,
      "grad_norm": 2.2599716186523438,
      "learning_rate": 1.906824117472509e-05,
      "logits/chosen": -0.3497006297111511,
      "logits/rejected": -0.8874444961547852,
      "logps/chosen": -0.8235942721366882,
      "logps/rejected": -5.622652530670166,
      "loss": 0.8246,
      "odds_ratio_loss": 0.009956205263733864,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0823594257235527,
      "rewards/margins": 0.47990575432777405,
      "rewards/rejected": -0.5622652173042297,
      "sft_loss": 0.8235942721366882,
      "step": 530
    },
    {
      "epoch": 1.7476521028991425,
      "grad_norm": 2.015336275100708,
      "learning_rate": 1.8653568770724806e-05,
      "logits/chosen": -0.2165544331073761,
      "logits/rejected": -0.780961811542511,
      "logps/chosen": -0.8200712203979492,
      "logps/rejected": -6.040628910064697,
      "loss": 0.8208,
      "odds_ratio_loss": 0.006871505174785852,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08200712502002716,
      "rewards/margins": 0.5220558643341064,
      "rewards/rejected": -0.6040629148483276,
      "sft_loss": 0.8200712203979492,
      "step": 535
    },
    {
      "epoch": 1.763985300122499,
      "grad_norm": 2.2979736328125,
      "learning_rate": 1.8240754485080242e-05,
      "logits/chosen": -0.32368141412734985,
      "logits/rejected": -0.7435051798820496,
      "logps/chosen": -0.864768385887146,
      "logps/rejected": -6.1721649169921875,
      "loss": 0.8654,
      "odds_ratio_loss": 0.006089936941862106,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08647684752941132,
      "rewards/margins": 0.5307397246360779,
      "rewards/rejected": -0.6172165274620056,
      "sft_loss": 0.864768385887146,
      "step": 540
    },
    {
      "epoch": 1.7803184973458555,
      "grad_norm": 1.8173251152038574,
      "learning_rate": 1.7829919182222752e-05,
      "logits/chosen": -0.1074652448296547,
      "logits/rejected": -0.6679927706718445,
      "logps/chosen": -0.8761852383613586,
      "logps/rejected": -6.271355628967285,
      "loss": 0.8768,
      "odds_ratio_loss": 0.005977124907076359,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08761852234601974,
      "rewards/margins": 0.5395171046257019,
      "rewards/rejected": -0.6271356344223022,
      "sft_loss": 0.8761852383613586,
      "step": 545
    },
    {
      "epoch": 1.796651694569212,
      "grad_norm": 2.2009291648864746,
      "learning_rate": 1.7421183147173915e-05,
      "logits/chosen": -0.27911990880966187,
      "logits/rejected": -0.7374569177627563,
      "logps/chosen": -0.843457043170929,
      "logps/rejected": -6.139245986938477,
      "loss": 0.8441,
      "odds_ratio_loss": 0.006457020528614521,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08434571325778961,
      "rewards/margins": 0.5295789241790771,
      "rewards/rejected": -0.6139246225357056,
      "sft_loss": 0.843457043170929,
      "step": 550
    },
    {
      "epoch": 1.8129848917925684,
      "grad_norm": 1.7380826473236084,
      "learning_rate": 1.7014666050328325e-05,
      "logits/chosen": -0.3113337457180023,
      "logits/rejected": -0.786532998085022,
      "logps/chosen": -0.7784915566444397,
      "logps/rejected": -5.818180084228516,
      "loss": 0.7793,
      "odds_ratio_loss": 0.007720579393208027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07784914970397949,
      "rewards/margins": 0.5039688944816589,
      "rewards/rejected": -0.5818179845809937,
      "sft_loss": 0.7784915566444397,
      "step": 555
    },
    {
      "epoch": 1.8293180890159249,
      "grad_norm": 2.6397993564605713,
      "learning_rate": 1.661048691241625e-05,
      "logits/chosen": -0.27005431056022644,
      "logits/rejected": -0.7860145568847656,
      "logps/chosen": -0.9139931797981262,
      "logps/rejected": -6.222220420837402,
      "loss": 0.9148,
      "odds_ratio_loss": 0.007696300745010376,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09139931201934814,
      "rewards/margins": 0.53082275390625,
      "rewards/rejected": -0.6222220659255981,
      "sft_loss": 0.9139931797981262,
      "step": 560
    },
    {
      "epoch": 1.8456512862392813,
      "grad_norm": 2.7970798015594482,
      "learning_rate": 1.620876406965658e-05,
      "logits/chosen": -0.2970743775367737,
      "logits/rejected": -0.8503972291946411,
      "logps/chosen": -0.8780502080917358,
      "logps/rejected": -6.108642578125,
      "loss": 0.8797,
      "odds_ratio_loss": 0.016725538298487663,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08780501782894135,
      "rewards/margins": 0.5230591893196106,
      "rewards/rejected": -0.6108641624450684,
      "sft_loss": 0.8780502080917358,
      "step": 565
    },
    {
      "epoch": 1.8619844834626378,
      "grad_norm": 2.1944961547851562,
      "learning_rate": 1.5809615139110172e-05,
      "logits/chosen": -0.30784982442855835,
      "logits/rejected": -0.7813681364059448,
      "logps/chosen": -0.8596214056015015,
      "logps/rejected": -6.107638835906982,
      "loss": 0.8601,
      "odds_ratio_loss": 0.0052293105982244015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08596214652061462,
      "rewards/margins": 0.5248017907142639,
      "rewards/rejected": -0.6107639074325562,
      "sft_loss": 0.8596214056015015,
      "step": 570
    },
    {
      "epoch": 1.8783176806859943,
      "grad_norm": 2.0301718711853027,
      "learning_rate": 1.5413156984243716e-05,
      "logits/chosen": -0.3718656003475189,
      "logits/rejected": -0.8364275097846985,
      "logps/chosen": -0.8916218876838684,
      "logps/rejected": -5.7218194007873535,
      "loss": 0.893,
      "odds_ratio_loss": 0.013976702466607094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0891621857881546,
      "rewards/margins": 0.48301976919174194,
      "rewards/rejected": -0.5721819996833801,
      "sft_loss": 0.8916218876838684,
      "step": 575
    },
    {
      "epoch": 1.8946508779093507,
      "grad_norm": 2.0284810066223145,
      "learning_rate": 1.5019505680714233e-05,
      "logits/chosen": -0.2422557771205902,
      "logits/rejected": -0.9137754440307617,
      "logps/chosen": -0.819043755531311,
      "logps/rejected": -5.523108005523682,
      "loss": 0.82,
      "odds_ratio_loss": 0.009329824708402157,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08190438896417618,
      "rewards/margins": 0.4704064428806305,
      "rewards/rejected": -0.5523108243942261,
      "sft_loss": 0.819043755531311,
      "step": 580
    },
    {
      "epoch": 1.9109840751327072,
      "grad_norm": 1.9808765649795532,
      "learning_rate": 1.462877648238432e-05,
      "logits/chosen": -0.4207962453365326,
      "logits/rejected": -0.9078248739242554,
      "logps/chosen": -0.7534202337265015,
      "logps/rejected": -5.3436737060546875,
      "loss": 0.7547,
      "odds_ratio_loss": 0.012538093142211437,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07534202188253403,
      "rewards/margins": 0.4590253233909607,
      "rewards/rejected": -0.5343673825263977,
      "sft_loss": 0.7534202337265015,
      "step": 585
    },
    {
      "epoch": 1.9273172723560636,
      "grad_norm": 2.0298492908477783,
      "learning_rate": 1.4241083787577923e-05,
      "logits/chosen": -0.3211058974266052,
      "logits/rejected": -0.8285148739814758,
      "logps/chosen": -0.8483121991157532,
      "logps/rejected": -5.782259941101074,
      "loss": 0.85,
      "odds_ratio_loss": 0.01721656322479248,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08483122289180756,
      "rewards/margins": 0.49339476227760315,
      "rewards/rejected": -0.5782259702682495,
      "sft_loss": 0.8483121991157532,
      "step": 590
    },
    {
      "epoch": 1.9436504695794201,
      "grad_norm": 2.123544454574585,
      "learning_rate": 1.3856541105586545e-05,
      "logits/chosen": -0.3325098752975464,
      "logits/rejected": -0.9181059002876282,
      "logps/chosen": -0.8079050779342651,
      "logps/rejected": -5.831791877746582,
      "loss": 0.8089,
      "odds_ratio_loss": 0.010058022104203701,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08079050481319427,
      "rewards/margins": 0.5023886561393738,
      "rewards/rejected": -0.5831791758537292,
      "sft_loss": 0.8079050779342651,
      "step": 595
    },
    {
      "epoch": 1.9599836668027768,
      "grad_norm": 1.670109748840332,
      "learning_rate": 1.3475261023435909e-05,
      "logits/chosen": -0.2341674566268921,
      "logits/rejected": -0.7504159808158875,
      "logps/chosen": -0.8234028816223145,
      "logps/rejected": -6.169764041900635,
      "loss": 0.8238,
      "odds_ratio_loss": 0.004405100364238024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0823402926325798,
      "rewards/margins": 0.5346360206604004,
      "rewards/rejected": -0.6169763803482056,
      "sft_loss": 0.8234028816223145,
      "step": 600
    },
    {
      "epoch": 1.976316864026133,
      "grad_norm": 1.6875087022781372,
      "learning_rate": 1.3097355172922499e-05,
      "logits/chosen": -0.2776256203651428,
      "logits/rejected": -0.7649735808372498,
      "logps/chosen": -0.8112476468086243,
      "logps/rejected": -6.096004962921143,
      "loss": 0.8118,
      "odds_ratio_loss": 0.0060133906081318855,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08112476766109467,
      "rewards/margins": 0.5284757614135742,
      "rewards/rejected": -0.6096004843711853,
      "sft_loss": 0.8112476468086243,
      "step": 605
    },
    {
      "epoch": 1.9926500612494897,
      "grad_norm": 1.9893207550048828,
      "learning_rate": 1.2722934197929803e-05,
      "logits/chosen": -0.2685665786266327,
      "logits/rejected": -0.7786232233047485,
      "logps/chosen": -0.7959750890731812,
      "logps/rejected": -6.182962417602539,
      "loss": 0.7964,
      "odds_ratio_loss": 0.004221998620778322,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07959751784801483,
      "rewards/margins": 0.5386987328529358,
      "rewards/rejected": -0.6182962656021118,
      "sft_loss": 0.7959750890731812,
      "step": 610
    },
    {
      "epoch": 2.008983258472846,
      "grad_norm": 1.7965635061264038,
      "learning_rate": 1.2352107722033843e-05,
      "logits/chosen": -0.2184493988752365,
      "logits/rejected": -0.8009710311889648,
      "logps/chosen": -0.8106815218925476,
      "logps/rejected": -5.654906272888184,
      "loss": 0.8116,
      "odds_ratio_loss": 0.009499135427176952,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08106815814971924,
      "rewards/margins": 0.48442238569259644,
      "rewards/rejected": -0.5654906034469604,
      "sft_loss": 0.8106815218925476,
      "step": 615
    },
    {
      "epoch": 2.0253164556962027,
      "grad_norm": 2.163811445236206,
      "learning_rate": 1.198498431640743e-05,
      "logits/chosen": -0.2861095070838928,
      "logits/rejected": -0.8159316778182983,
      "logps/chosen": -0.7896136045455933,
      "logps/rejected": -5.978821754455566,
      "loss": 0.7903,
      "odds_ratio_loss": 0.007189574651420116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07896135747432709,
      "rewards/margins": 0.5189208984375,
      "rewards/rejected": -0.5978822708129883,
      "sft_loss": 0.7896136045455933,
      "step": 620
    },
    {
      "epoch": 2.041649652919559,
      "grad_norm": 2.019101142883301,
      "learning_rate": 1.1621671468032493e-05,
      "logits/chosen": -0.39903494715690613,
      "logits/rejected": -0.8325216174125671,
      "logps/chosen": -0.807002067565918,
      "logps/rejected": -5.897282600402832,
      "loss": 0.8075,
      "odds_ratio_loss": 0.005388813093304634,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08070021867752075,
      "rewards/margins": 0.5090280771255493,
      "rewards/rejected": -0.5897282958030701,
      "sft_loss": 0.807002067565918,
      "step": 625
    },
    {
      "epoch": 2.0579828501429156,
      "grad_norm": 2.1805834770202637,
      "learning_rate": 1.126227554822985e-05,
      "logits/chosen": -0.2608413100242615,
      "logits/rejected": -0.7501794099807739,
      "logps/chosen": -0.7259417772293091,
      "logps/rejected": -6.224379062652588,
      "loss": 0.7263,
      "odds_ratio_loss": 0.0037105106748640537,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07259417325258255,
      "rewards/margins": 0.5498436689376831,
      "rewards/rejected": -0.6224378943443298,
      "sft_loss": 0.7259417772293091,
      "step": 630
    },
    {
      "epoch": 2.074316047366272,
      "grad_norm": 2.2151620388031006,
      "learning_rate": 1.0906901781515694e-05,
      "logits/chosen": -0.2219965159893036,
      "logits/rejected": -0.7585871815681458,
      "logps/chosen": -0.7999061346054077,
      "logps/rejected": -6.1256184577941895,
      "loss": 0.8005,
      "odds_ratio_loss": 0.005652335938066244,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07999061048030853,
      "rewards/margins": 0.5325711965560913,
      "rewards/rejected": -0.612561821937561,
      "sft_loss": 0.7999061346054077,
      "step": 635
    },
    {
      "epoch": 2.0906492445896285,
      "grad_norm": 2.2990260124206543,
      "learning_rate": 1.0555654214793722e-05,
      "logits/chosen": -0.2750963270664215,
      "logits/rejected": -0.7611300945281982,
      "logps/chosen": -0.8194829225540161,
      "logps/rejected": -6.067640781402588,
      "loss": 0.8201,
      "odds_ratio_loss": 0.006496583577245474,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08194829523563385,
      "rewards/margins": 0.5248157978057861,
      "rewards/rejected": -0.6067641377449036,
      "sft_loss": 0.8194829225540161,
      "step": 640
    },
    {
      "epoch": 2.1069824418129848,
      "grad_norm": 2.1134228706359863,
      "learning_rate": 1.0208635686892202e-05,
      "logits/chosen": -0.2696288526058197,
      "logits/rejected": -0.8589851260185242,
      "logps/chosen": -0.7490558624267578,
      "logps/rejected": -5.564435005187988,
      "loss": 0.7501,
      "odds_ratio_loss": 0.01047680526971817,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07490558922290802,
      "rewards/margins": 0.48153796792030334,
      "rewards/rejected": -0.5564435720443726,
      "sft_loss": 0.7490558624267578,
      "step": 645
    },
    {
      "epoch": 2.1233156390363415,
      "grad_norm": 2.0989155769348145,
      "learning_rate": 9.865947798454556e-06,
      "logits/chosen": -0.24364769458770752,
      "logits/rejected": -0.804131031036377,
      "logps/chosen": -0.8394583463668823,
      "logps/rejected": -6.04466438293457,
      "loss": 0.84,
      "odds_ratio_loss": 0.00556190824136138,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08394583314657211,
      "rewards/margins": 0.5205205678939819,
      "rewards/rejected": -0.6044663786888123,
      "sft_loss": 0.8394583463668823,
      "step": 650
    },
    {
      "epoch": 2.1396488362596977,
      "grad_norm": 3.0253589153289795,
      "learning_rate": 9.527690882192636e-06,
      "logits/chosen": -0.34022632241249084,
      "logits/rejected": -0.8489152789115906,
      "logps/chosen": -0.7610163688659668,
      "logps/rejected": -5.664425849914551,
      "loss": 0.7623,
      "odds_ratio_loss": 0.01285813469439745,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0761016383767128,
      "rewards/margins": 0.4903409481048584,
      "rewards/rejected": -0.566442608833313,
      "sft_loss": 0.7610163688659668,
      "step": 655
    },
    {
      "epoch": 2.1559820334830544,
      "grad_norm": 1.6050690412521362,
      "learning_rate": 9.193963973511135e-06,
      "logits/chosen": -0.2724407911300659,
      "logits/rejected": -0.7652899026870728,
      "logps/chosen": -0.7721027135848999,
      "logps/rejected": -5.853890419006348,
      "loss": 0.7728,
      "odds_ratio_loss": 0.006839917507022619,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07721027731895447,
      "rewards/margins": 0.5081788301467896,
      "rewards/rejected": -0.5853890180587769,
      "sft_loss": 0.7721027135848999,
      "step": 660
    },
    {
      "epoch": 2.1723152307064106,
      "grad_norm": 2.1909117698669434,
      "learning_rate": 8.864864781511807e-06,
      "logits/chosen": -0.41523709893226624,
      "logits/rejected": -0.7764497995376587,
      "logps/chosen": -0.8339493870735168,
      "logps/rejected": -6.235623836517334,
      "loss": 0.8346,
      "odds_ratio_loss": 0.006118964869529009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08339492976665497,
      "rewards/margins": 0.5401674509048462,
      "rewards/rejected": -0.6235624551773071,
      "sft_loss": 0.8339493870735168,
      "step": 665
    },
    {
      "epoch": 2.1886484279297673,
      "grad_norm": 2.204056978225708,
      "learning_rate": 8.540489660386063e-06,
      "logits/chosen": -0.28157439827919006,
      "logits/rejected": -0.7564243674278259,
      "logps/chosen": -0.8656519055366516,
      "logps/rejected": -6.216989517211914,
      "loss": 0.8666,
      "odds_ratio_loss": 0.0093773752450943,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08656518906354904,
      "rewards/margins": 0.5351337790489197,
      "rewards/rejected": -0.6216989755630493,
      "sft_loss": 0.8656519055366516,
      "step": 670
    },
    {
      "epoch": 2.2049816251531236,
      "grad_norm": 2.3478028774261475,
      "learning_rate": 8.220933581204257e-06,
      "logits/chosen": -0.28393420577049255,
      "logits/rejected": -0.873609185218811,
      "logps/chosen": -0.846001923084259,
      "logps/rejected": -5.715620517730713,
      "loss": 0.847,
      "odds_ratio_loss": 0.009764907881617546,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08460019528865814,
      "rewards/margins": 0.48696184158325195,
      "rewards/rejected": -0.5715619921684265,
      "sft_loss": 0.846001923084259,
      "step": 675
    },
    {
      "epoch": 2.2213148223764803,
      "grad_norm": 2.1057419776916504,
      "learning_rate": 7.906290104109875e-06,
      "logits/chosen": -0.21088667213916779,
      "logits/rejected": -0.8024004697799683,
      "logps/chosen": -0.8920997381210327,
      "logps/rejected": -5.927838325500488,
      "loss": 0.8927,
      "odds_ratio_loss": 0.0057823387905955315,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08920996636152267,
      "rewards/margins": 0.5035739541053772,
      "rewards/rejected": -0.5927838683128357,
      "sft_loss": 0.8920997381210327,
      "step": 680
    },
    {
      "epoch": 2.2376480195998365,
      "grad_norm": 2.2970566749572754,
      "learning_rate": 7.5966513509268365e-06,
      "logits/chosen": -0.24143846333026886,
      "logits/rejected": -0.7781409025192261,
      "logps/chosen": -0.7809386253356934,
      "logps/rejected": -5.8026275634765625,
      "loss": 0.7819,
      "odds_ratio_loss": 0.009480984881520271,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07809387147426605,
      "rewards/margins": 0.5021688342094421,
      "rewards/rejected": -0.5802627801895142,
      "sft_loss": 0.7809386253356934,
      "step": 685
    },
    {
      "epoch": 2.253981216823193,
      "grad_norm": 2.1630165576934814,
      "learning_rate": 7.292107978187951e-06,
      "logits/chosen": -0.27860409021377563,
      "logits/rejected": -0.8003829717636108,
      "logps/chosen": -0.8690743446350098,
      "logps/rejected": -5.686347007751465,
      "loss": 0.87,
      "odds_ratio_loss": 0.00926794670522213,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08690743893384933,
      "rewards/margins": 0.48172727227211,
      "rewards/rejected": -0.5686346888542175,
      "sft_loss": 0.8690743446350098,
      "step": 690
    },
    {
      "epoch": 2.2703144140465494,
      "grad_norm": 1.9345569610595703,
      "learning_rate": 6.992749150592343e-06,
      "logits/chosen": -0.2534225583076477,
      "logits/rejected": -0.7979742288589478,
      "logps/chosen": -0.8852649927139282,
      "logps/rejected": -5.78798770904541,
      "loss": 0.8865,
      "odds_ratio_loss": 0.011952772736549377,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08852650225162506,
      "rewards/margins": 0.49027228355407715,
      "rewards/rejected": -0.578798770904541,
      "sft_loss": 0.8852649927139282,
      "step": 695
    },
    {
      "epoch": 2.286647611269906,
      "grad_norm": 1.9216334819793701,
      "learning_rate": 6.69866251489964e-06,
      "logits/chosen": -0.15063397586345673,
      "logits/rejected": -0.7254005670547485,
      "logps/chosen": -0.8156722784042358,
      "logps/rejected": -5.895369052886963,
      "loss": 0.8165,
      "odds_ratio_loss": 0.007838285528123379,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08156722784042358,
      "rewards/margins": 0.5079696774482727,
      "rewards/rejected": -0.5895368456840515,
      "sft_loss": 0.8156722784042358,
      "step": 700
    }
  ],
  "logging_steps": 5,
  "max_steps": 918,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100,
  "total_flos": 2.9087955577523405e+17,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}