{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.3066557778685177,
  "eval_steps": 500,
  "global_step": 400,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.016333197223356473,
      "grad_norm": 1.1593419313430786,
      "learning_rate": 4.9996340229869763e-05,
      "logits/chosen": -1.0346699953079224,
      "logits/rejected": -2.6695446968078613,
      "logps/chosen": -1.101169466972351,
      "logps/rejected": -1.0235556364059448,
      "loss": 1.178,
      "odds_ratio_loss": 0.768240213394165,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.11011694371700287,
      "rewards/margins": -0.007761377841234207,
      "rewards/rejected": -0.10235557705163956,
      "sft_loss": 1.101169466972351,
      "step": 5
    },
    {
      "epoch": 0.032666394446712946,
      "grad_norm": 1.1402696371078491,
      "learning_rate": 4.9985361990992455e-05,
      "logits/chosen": -0.8193758726119995,
      "logits/rejected": -2.842833995819092,
      "logps/chosen": -1.0217273235321045,
      "logps/rejected": -1.0100481510162354,
      "loss": 1.0927,
      "odds_ratio_loss": 0.7092453837394714,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.10217274725437164,
      "rewards/margins": -0.0011679243762046099,
      "rewards/rejected": -0.10100481659173965,
      "sft_loss": 1.0217273235321045,
      "step": 10
    },
    {
      "epoch": 0.04899959167006942,
      "grad_norm": 1.3554140329360962,
      "learning_rate": 4.996706849759453e-05,
      "logits/chosen": -0.7696830034255981,
      "logits/rejected": -2.703421115875244,
      "logps/chosen": -1.0294106006622314,
      "logps/rejected": -1.0599125623703003,
      "loss": 1.0993,
      "odds_ratio_loss": 0.6984036564826965,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.10294107347726822,
      "rewards/margins": 0.003050185739994049,
      "rewards/rejected": -0.10599125921726227,
      "sft_loss": 1.0294106006622314,
      "step": 15
    },
    {
      "epoch": 0.06533278889342589,
      "grad_norm": 1.406853199005127,
      "learning_rate": 4.9941465105674435e-05,
      "logits/chosen": -0.698488712310791,
      "logits/rejected": -2.808516263961792,
      "logps/chosen": -1.0651928186416626,
      "logps/rejected": -0.996777355670929,
      "loss": 1.1402,
      "odds_ratio_loss": 0.7504132390022278,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.10651928186416626,
      "rewards/margins": -0.006841552909463644,
      "rewards/rejected": -0.09967772662639618,
      "sft_loss": 1.0651928186416626,
      "step": 20
    },
    {
      "epoch": 0.08166598611678236,
      "grad_norm": 1.238513469696045,
      "learning_rate": 4.99085593114345e-05,
      "logits/chosen": -0.6306881904602051,
      "logits/rejected": -2.698362350463867,
      "logps/chosen": -1.0242538452148438,
      "logps/rejected": -1.0320734977722168,
      "loss": 1.0937,
      "odds_ratio_loss": 0.6945221424102783,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.10242538154125214,
      "rewards/margins": 0.0007819652673788369,
      "rewards/rejected": -0.10320734977722168,
      "sft_loss": 1.0242538452148438,
      "step": 25
    },
    {
      "epoch": 0.09799918334013884,
      "grad_norm": 1.9588699340820312,
      "learning_rate": 4.986836074908616e-05,
      "logits/chosen": -0.5534936189651489,
      "logits/rejected": -2.678889036178589,
      "logps/chosen": -1.1003000736236572,
      "logps/rejected": -1.0425399541854858,
      "loss": 1.1745,
      "odds_ratio_loss": 0.7416313290596008,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11003003269433975,
      "rewards/margins": -0.005776026751846075,
      "rewards/rejected": -0.10425399243831635,
      "sft_loss": 1.1003000736236572,
      "step": 30
    },
    {
      "epoch": 0.1143323805634953,
      "grad_norm": 1.1857821941375732,
      "learning_rate": 4.982088118802922e-05,
      "logits/chosen": -0.6129628419876099,
      "logits/rejected": -2.876364231109619,
      "logps/chosen": -0.9628928303718567,
      "logps/rejected": -0.952060341835022,
      "loss": 1.0336,
      "odds_ratio_loss": 0.7068274617195129,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.09628929197788239,
      "rewards/margins": -0.001083258306607604,
      "rewards/rejected": -0.09520602226257324,
      "sft_loss": 0.9628928303718567,
      "step": 35
    },
    {
      "epoch": 0.13066557778685178,
      "grad_norm": 1.33924400806427,
      "learning_rate": 4.976613452940604e-05,
      "logits/chosen": -0.7091814279556274,
      "logits/rejected": -2.6595664024353027,
      "logps/chosen": -0.89775550365448,
      "logps/rejected": -0.935952365398407,
      "loss": 0.9642,
      "odds_ratio_loss": 0.6644185781478882,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08977555483579636,
      "rewards/margins": 0.003819683101028204,
      "rewards/rejected": -0.0935952290892601,
      "sft_loss": 0.89775550365448,
      "step": 40
    },
    {
      "epoch": 0.14699877501020825,
      "grad_norm": 1.136667013168335,
      "learning_rate": 4.9704136802031485e-05,
      "logits/chosen": -0.7233263850212097,
      "logits/rejected": -2.6673388481140137,
      "logps/chosen": -0.9036429524421692,
      "logps/rejected": -0.9419530630111694,
      "loss": 0.9699,
      "odds_ratio_loss": 0.662254273891449,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09036429226398468,
      "rewards/margins": 0.0038310133386403322,
      "rewards/rejected": -0.09419531375169754,
      "sft_loss": 0.9036429524421692,
      "step": 45
    },
    {
      "epoch": 0.16333197223356472,
      "grad_norm": 1.4290399551391602,
      "learning_rate": 4.9634906157700036e-05,
      "logits/chosen": -0.6113058924674988,
      "logits/rejected": -2.705373764038086,
      "logps/chosen": -0.9550830125808716,
      "logps/rejected": -0.9817631840705872,
      "loss": 1.0223,
      "odds_ratio_loss": 0.6725824475288391,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.09550830721855164,
      "rewards/margins": 0.002668013097718358,
      "rewards/rejected": -0.09817632287740707,
      "sft_loss": 0.9550830125808716,
      "step": 50
    },
    {
      "epoch": 0.17966516945692118,
      "grad_norm": 1.4945589303970337,
      "learning_rate": 4.955846286587122e-05,
      "logits/chosen": -0.647710919380188,
      "logits/rejected": -2.7208316326141357,
      "logps/chosen": -0.9462896585464478,
      "logps/rejected": -0.9507497549057007,
      "loss": 1.0148,
      "odds_ratio_loss": 0.6852728128433228,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0946289673447609,
      "rewards/margins": 0.0004460157942958176,
      "rewards/rejected": -0.09507497400045395,
      "sft_loss": 0.9462896585464478,
      "step": 55
    },
    {
      "epoch": 0.19599836668027767,
      "grad_norm": 1.3553156852722168,
      "learning_rate": 4.9474829307735115e-05,
      "logits/chosen": -0.6279743909835815,
      "logits/rejected": -2.6854496002197266,
      "logps/chosen": -0.8896121978759766,
      "logps/rejected": -0.9993942975997925,
      "loss": 0.9509,
      "odds_ratio_loss": 0.6129180192947388,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.08896122127771378,
      "rewards/margins": 0.010978218168020248,
      "rewards/rejected": -0.09993942826986313,
      "sft_loss": 0.8896121978759766,
      "step": 60
    },
    {
      "epoch": 0.21233156390363414,
      "grad_norm": 1.4247411489486694,
      "learning_rate": 4.9384029969659573e-05,
      "logits/chosen": -0.698381781578064,
      "logits/rejected": -2.7967753410339355,
      "logps/chosen": -0.9455701112747192,
      "logps/rejected": -0.9564310908317566,
      "loss": 1.0147,
      "odds_ratio_loss": 0.6911824941635132,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0945570170879364,
      "rewards/margins": 0.0010860899928957224,
      "rewards/rejected": -0.09564311057329178,
      "sft_loss": 0.9455701112747192,
      "step": 65
    },
    {
      "epoch": 0.2286647611269906,
      "grad_norm": 1.6154508590698242,
      "learning_rate": 4.9286091436021015e-05,
      "logits/chosen": -0.595032274723053,
      "logits/rejected": -2.740784168243408,
      "logps/chosen": -0.9214372634887695,
      "logps/rejected": -1.0210167169570923,
      "loss": 0.9836,
      "odds_ratio_loss": 0.6212525963783264,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09214373677968979,
      "rewards/margins": 0.00995794590562582,
      "rewards/rejected": -0.10210166871547699,
      "sft_loss": 0.9214372634887695,
      "step": 70
    },
    {
      "epoch": 0.24499795835034707,
      "grad_norm": 1.8231102228164673,
      "learning_rate": 4.918104238142104e-05,
      "logits/chosen": -0.5186889171600342,
      "logits/rejected": -2.7576358318328857,
      "logps/chosen": -0.9789311289787292,
      "logps/rejected": -1.0169849395751953,
      "loss": 1.0454,
      "odds_ratio_loss": 0.6644267439842224,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09789310395717621,
      "rewards/margins": 0.0038053765892982483,
      "rewards/rejected": -0.10169849544763565,
      "sft_loss": 0.9789311289787292,
      "step": 75
    },
    {
      "epoch": 0.26133115557370357,
      "grad_norm": 2.05936336517334,
      "learning_rate": 4.906891356229103e-05,
      "logits/chosen": -0.47254711389541626,
      "logits/rejected": -2.68280029296875,
      "logps/chosen": -0.9620391726493835,
      "logps/rejected": -1.0026928186416626,
      "loss": 1.0294,
      "odds_ratio_loss": 0.6731926202774048,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.09620390832424164,
      "rewards/margins": 0.004065364599227905,
      "rewards/rejected": -0.10026928037405014,
      "sft_loss": 0.9620391726493835,
      "step": 80
    },
    {
      "epoch": 0.27766435279706003,
      "grad_norm": 1.6302626132965088,
      "learning_rate": 4.894973780788722e-05,
      "logits/chosen": -0.5315309762954712,
      "logits/rejected": -2.7227425575256348,
      "logps/chosen": -0.9423310160636902,
      "logps/rejected": -1.071781039237976,
      "loss": 1.0032,
      "odds_ratio_loss": 0.6090766191482544,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.09423311054706573,
      "rewards/margins": 0.012944999150931835,
      "rewards/rejected": -0.10717810690402985,
      "sft_loss": 0.9423310160636902,
      "step": 85
    },
    {
      "epoch": 0.2939975500204165,
      "grad_norm": 1.7891793251037598,
      "learning_rate": 4.882355001067892e-05,
      "logits/chosen": -0.5275685787200928,
      "logits/rejected": -2.6127567291259766,
      "logps/chosen": -0.9351356625556946,
      "logps/rejected": -1.0083874464035034,
      "loss": 0.9995,
      "odds_ratio_loss": 0.6435801386833191,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09351354837417603,
      "rewards/margins": 0.0073251971043646336,
      "rewards/rejected": -0.10083875805139542,
      "sft_loss": 0.9351356625556946,
      "step": 90
    },
    {
      "epoch": 0.31033074724377296,
      "grad_norm": 1.852216124534607,
      "learning_rate": 4.869038711613259e-05,
      "logits/chosen": -0.5760647058486938,
      "logits/rejected": -2.722364664077759,
      "logps/chosen": -0.9301847219467163,
      "logps/rejected": -1.0749822854995728,
      "loss": 0.9903,
      "odds_ratio_loss": 0.601297914981842,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09301847964525223,
      "rewards/margins": 0.014479748904705048,
      "rewards/rejected": -0.10749821364879608,
      "sft_loss": 0.9301847219467163,
      "step": 95
    },
    {
      "epoch": 0.32666394446712943,
      "grad_norm": 1.8597307205200195,
      "learning_rate": 4.855028811189496e-05,
      "logits/chosen": -0.5871164202690125,
      "logits/rejected": -2.6969285011291504,
      "logps/chosen": -1.0701930522918701,
      "logps/rejected": -1.0556358098983765,
      "loss": 1.1427,
      "odds_ratio_loss": 0.7247555255889893,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.10701930522918701,
      "rewards/margins": -0.0014557272661477327,
      "rewards/rejected": -0.10556356608867645,
      "sft_loss": 1.0701930522918701,
      "step": 100
    },
    {
      "epoch": 0.3429971416904859,
      "grad_norm": 1.6869758367538452,
      "learning_rate": 4.8403294016378095e-05,
      "logits/chosen": -0.4466473460197449,
      "logits/rejected": -2.501394510269165,
      "logps/chosen": -0.880301833152771,
      "logps/rejected": -1.0003767013549805,
      "loss": 0.941,
      "odds_ratio_loss": 0.60676509141922,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.08803018927574158,
      "rewards/margins": 0.012007484212517738,
      "rewards/rejected": -0.10003767162561417,
      "sft_loss": 0.880301833152771,
      "step": 105
    },
    {
      "epoch": 0.35933033891384236,
      "grad_norm": 1.8963792324066162,
      "learning_rate": 4.8249447866750025e-05,
      "logits/chosen": -0.41199809312820435,
      "logits/rejected": -2.594975471496582,
      "logps/chosen": -0.8528003692626953,
      "logps/rejected": -0.9772283434867859,
      "loss": 0.9128,
      "odds_ratio_loss": 0.6004671454429626,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.08528003096580505,
      "rewards/margins": 0.012442806735634804,
      "rewards/rejected": -0.0977228432893753,
      "sft_loss": 0.8528003692626953,
      "step": 110
    },
    {
      "epoch": 0.3756635361371989,
      "grad_norm": 2.6958725452423096,
      "learning_rate": 4.808879470633417e-05,
      "logits/chosen": -0.43985629081726074,
      "logits/rejected": -2.6999452114105225,
      "logps/chosen": -0.9486901164054871,
      "logps/rejected": -1.0987169742584229,
      "loss": 1.0076,
      "odds_ratio_loss": 0.5889819264411926,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.09486902505159378,
      "rewards/margins": 0.015002680011093616,
      "rewards/rejected": -0.10987170040607452,
      "sft_loss": 0.9486901164054871,
      "step": 115
    },
    {
      "epoch": 0.39199673336055535,
      "grad_norm": 1.6014004945755005,
      "learning_rate": 4.792138157142158e-05,
      "logits/chosen": -0.43066906929016113,
      "logits/rejected": -2.634669065475464,
      "logps/chosen": -0.8806023597717285,
      "logps/rejected": -1.0933107137680054,
      "loss": 0.9352,
      "odds_ratio_loss": 0.5455727577209473,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.08806024491786957,
      "rewards/margins": 0.021270830184221268,
      "rewards/rejected": -0.10933107137680054,
      "sft_loss": 0.8806023597717285,
      "step": 120
    },
    {
      "epoch": 0.4083299305839118,
      "grad_norm": 3.4042723178863525,
      "learning_rate": 4.7747257477499474e-05,
      "logits/chosen": -0.4240710735321045,
      "logits/rejected": -2.664668560028076,
      "logps/chosen": -0.8993316888809204,
      "logps/rejected": -1.0890860557556152,
      "loss": 0.9553,
      "odds_ratio_loss": 0.5601183772087097,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.08993317186832428,
      "rewards/margins": 0.018975432962179184,
      "rewards/rejected": -0.10890860855579376,
      "sft_loss": 0.8993316888809204,
      "step": 125
    },
    {
      "epoch": 0.4246631278072683,
      "grad_norm": 1.724076509475708,
      "learning_rate": 4.75664734049005e-05,
      "logits/chosen": -0.3910972476005554,
      "logits/rejected": -2.571296215057373,
      "logps/chosen": -0.9113162755966187,
      "logps/rejected": -1.094262719154358,
      "loss": 0.9682,
      "odds_ratio_loss": 0.5683911442756653,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09113162755966187,
      "rewards/margins": 0.018294643610715866,
      "rewards/rejected": -0.10942627489566803,
      "sft_loss": 0.9113162755966187,
      "step": 130
    },
    {
      "epoch": 0.44099632503062475,
      "grad_norm": 1.6363219022750854,
      "learning_rate": 4.7379082283876566e-05,
      "logits/chosen": -0.5021435022354126,
      "logits/rejected": -2.6572070121765137,
      "logps/chosen": -0.8906953930854797,
      "logps/rejected": -1.0344655513763428,
      "loss": 0.9498,
      "odds_ratio_loss": 0.5914202928543091,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08906953036785126,
      "rewards/margins": 0.014377007260918617,
      "rewards/rejected": -0.10344654321670532,
      "sft_loss": 0.8906953930854797,
      "step": 135
    },
    {
      "epoch": 0.4573295222539812,
      "grad_norm": 1.7812143564224243,
      "learning_rate": 4.7185138979101864e-05,
      "logits/chosen": -0.507682740688324,
      "logits/rejected": -2.593475580215454,
      "logps/chosen": -0.9167561531066895,
      "logps/rejected": -1.1608246564865112,
      "loss": 0.9698,
      "odds_ratio_loss": 0.5303933024406433,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.09167560935020447,
      "rewards/margins": 0.024406855925917625,
      "rewards/rejected": -0.11608246713876724,
      "sft_loss": 0.9167561531066895,
      "step": 140
    },
    {
      "epoch": 0.4736627194773377,
      "grad_norm": 1.823789119720459,
      "learning_rate": 4.6984700273609495e-05,
      "logits/chosen": -0.3446227014064789,
      "logits/rejected": -2.5670173168182373,
      "logps/chosen": -0.8484543561935425,
      "logps/rejected": -1.0493099689483643,
      "loss": 0.903,
      "odds_ratio_loss": 0.5457809567451477,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.08484544605016708,
      "rewards/margins": 0.020085560157895088,
      "rewards/rejected": -0.10493101179599762,
      "sft_loss": 0.8484543561935425,
      "step": 145
    },
    {
      "epoch": 0.48999591670069415,
      "grad_norm": 1.889367699623108,
      "learning_rate": 4.677782485216644e-05,
      "logits/chosen": -0.44083428382873535,
      "logits/rejected": -2.573638439178467,
      "logps/chosen": -0.833287239074707,
      "logps/rejected": -1.1050159931182861,
      "loss": 0.8838,
      "odds_ratio_loss": 0.5049694180488586,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0833287239074707,
      "rewards/margins": 0.02717287465929985,
      "rewards/rejected": -0.11050160229206085,
      "sft_loss": 0.833287239074707,
      "step": 150
    },
    {
      "epoch": 0.5063291139240507,
      "grad_norm": 1.9978969097137451,
      "learning_rate": 4.6564573284091765e-05,
      "logits/chosen": -0.37475016713142395,
      "logits/rejected": -2.530078887939453,
      "logps/chosen": -0.9055852890014648,
      "logps/rejected": -1.3165007829666138,
      "loss": 0.9498,
      "odds_ratio_loss": 0.4424561858177185,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.09055852890014648,
      "rewards/margins": 0.04109155014157295,
      "rewards/rejected": -0.13165007531642914,
      "sft_loss": 0.9055852890014648,
      "step": 155
    },
    {
      "epoch": 0.5226623111474071,
      "grad_norm": 2.2659225463867188,
      "learning_rate": 4.6345008005522966e-05,
      "logits/chosen": -0.27824097871780396,
      "logits/rejected": -2.1937553882598877,
      "logps/chosen": -0.7978348731994629,
      "logps/rejected": -1.7242333889007568,
      "loss": 0.8243,
      "odds_ratio_loss": 0.26432332396507263,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07978348433971405,
      "rewards/margins": 0.09263985604047775,
      "rewards/rejected": -0.1724233478307724,
      "sft_loss": 0.7978348731994629,
      "step": 160
    },
    {
      "epoch": 0.5389955083707636,
      "grad_norm": 2.497178792953491,
      "learning_rate": 4.611919330113592e-05,
      "logits/chosen": -0.1297352910041809,
      "logits/rejected": -1.7525418996810913,
      "logps/chosen": -0.9057181477546692,
      "logps/rejected": -2.926887035369873,
      "loss": 0.9163,
      "odds_ratio_loss": 0.10600633919239044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0905718058347702,
      "rewards/margins": 0.20211689174175262,
      "rewards/rejected": -0.2926887273788452,
      "sft_loss": 0.9057181477546692,
      "step": 165
    },
    {
      "epoch": 0.5553287055941201,
      "grad_norm": 1.9597887992858887,
      "learning_rate": 4.588719528532342e-05,
      "logits/chosen": -0.20320752263069153,
      "logits/rejected": -1.4767993688583374,
      "logps/chosen": -0.9343756437301636,
      "logps/rejected": -3.7731080055236816,
      "loss": 0.9406,
      "odds_ratio_loss": 0.062325619161129,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0934375673532486,
      "rewards/margins": 0.28387323021888733,
      "rewards/rejected": -0.3773108124732971,
      "sft_loss": 0.9343756437301636,
      "step": 170
    },
    {
      "epoch": 0.5716619028174765,
      "grad_norm": 1.9177374839782715,
      "learning_rate": 4.564908188283814e-05,
      "logits/chosen": -0.16985416412353516,
      "logits/rejected": -1.3144887685775757,
      "logps/chosen": -0.8834988474845886,
      "logps/rejected": -4.147261619567871,
      "loss": 0.8871,
      "odds_ratio_loss": 0.03579668328166008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08834987133741379,
      "rewards/margins": 0.3263762891292572,
      "rewards/rejected": -0.414726197719574,
      "sft_loss": 0.8834988474845886,
      "step": 175
    },
    {
      "epoch": 0.587995100040833,
      "grad_norm": 1.9115794897079468,
      "learning_rate": 4.540492280890555e-05,
      "logits/chosen": -0.34701991081237793,
      "logits/rejected": -1.1767631769180298,
      "logps/chosen": -0.8725343942642212,
      "logps/rejected": -4.597090721130371,
      "loss": 0.8753,
      "odds_ratio_loss": 0.027656957507133484,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08725343644618988,
      "rewards/margins": 0.3724556267261505,
      "rewards/rejected": -0.4597090780735016,
      "sft_loss": 0.8725343942642212,
      "step": 180
    },
    {
      "epoch": 0.6043282972641895,
      "grad_norm": 1.7303317785263062,
      "learning_rate": 4.515478954881248e-05,
      "logits/chosen": -0.37866610288619995,
      "logits/rejected": -1.20633065700531,
      "logps/chosen": -0.8602287173271179,
      "logps/rejected": -4.4016923904418945,
      "loss": 0.8625,
      "odds_ratio_loss": 0.022269519045948982,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08602287620306015,
      "rewards/margins": 0.35414639115333557,
      "rewards/rejected": -0.4401692748069763,
      "sft_loss": 0.8602287173271179,
      "step": 185
    },
    {
      "epoch": 0.6206614944875459,
      "grad_norm": 5.377653121948242,
      "learning_rate": 4.4898755336977673e-05,
      "logits/chosen": -0.32614630460739136,
      "logits/rejected": -1.1425362825393677,
      "logps/chosen": -0.9115999937057495,
      "logps/rejected": -4.731467247009277,
      "loss": 0.9149,
      "odds_ratio_loss": 0.0329320952296257,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09115998446941376,
      "rewards/margins": 0.38198670744895935,
      "rewards/rejected": -0.4731467366218567,
      "sft_loss": 0.9115999937057495,
      "step": 190
    },
    {
      "epoch": 0.6369946917109024,
      "grad_norm": 1.6315754652023315,
      "learning_rate": 4.463689513550997e-05,
      "logits/chosen": -0.3550247550010681,
      "logits/rejected": -1.241008996963501,
      "logps/chosen": -0.8556908369064331,
      "logps/rejected": -4.392051696777344,
      "loss": 0.8582,
      "odds_ratio_loss": 0.024758270010352135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08556908369064331,
      "rewards/margins": 0.35363608598709106,
      "rewards/rejected": -0.4392051696777344,
      "sft_loss": 0.8556908369064331,
      "step": 195
    },
    {
      "epoch": 0.6533278889342589,
      "grad_norm": 1.8591830730438232,
      "learning_rate": 4.436928561226087e-05,
      "logits/chosen": -0.3138996660709381,
      "logits/rejected": -1.2280880212783813,
      "logps/chosen": -0.901713490486145,
      "logps/rejected": -4.212186336517334,
      "loss": 0.9045,
      "odds_ratio_loss": 0.027884572744369507,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09017134457826614,
      "rewards/margins": 0.3310473561286926,
      "rewards/rejected": -0.4212186932563782,
      "sft_loss": 0.901713490486145,
      "step": 200
    },
    {
      "epoch": 0.6696610861576153,
      "grad_norm": 1.8346513509750366,
      "learning_rate": 4.409600511837756e-05,
      "logits/chosen": -0.23156340420246124,
      "logits/rejected": -1.138501524925232,
      "logps/chosen": -0.9389227032661438,
      "logps/rejected": -4.50938606262207,
      "loss": 0.9416,
      "odds_ratio_loss": 0.02684590592980385,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09389226883649826,
      "rewards/margins": 0.35704633593559265,
      "rewards/rejected": -0.4509386122226715,
      "sft_loss": 0.9389227032661438,
      "step": 205
    },
    {
      "epoch": 0.6859942833809718,
      "grad_norm": 1.5812890529632568,
      "learning_rate": 4.381713366536311e-05,
      "logits/chosen": -0.24586188793182373,
      "logits/rejected": -1.1232062578201294,
      "logps/chosen": -0.859118640422821,
      "logps/rejected": -4.883942127227783,
      "loss": 0.8609,
      "odds_ratio_loss": 0.018183354288339615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08591187745332718,
      "rewards/margins": 0.4024823307991028,
      "rewards/rejected": -0.48839420080184937,
      "sft_loss": 0.859118640422821,
      "step": 210
    },
    {
      "epoch": 0.7023274806043283,
      "grad_norm": 1.8375471830368042,
      "learning_rate": 4.353275290165066e-05,
      "logits/chosen": -0.15520325303077698,
      "logits/rejected": -0.985150933265686,
      "logps/chosen": -0.8214719891548157,
      "logps/rejected": -4.9630866050720215,
      "loss": 0.823,
      "odds_ratio_loss": 0.014944888651371002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08214719593524933,
      "rewards/margins": 0.4141615033149719,
      "rewards/rejected": -0.49630871415138245,
      "sft_loss": 0.8214719891548157,
      "step": 215
    },
    {
      "epoch": 0.7186606778276847,
      "grad_norm": 2.556330919265747,
      "learning_rate": 4.324294608869817e-05,
      "logits/chosen": -0.3221624791622162,
      "logits/rejected": -0.9542757272720337,
      "logps/chosen": -0.9679568409919739,
      "logps/rejected": -5.273629188537598,
      "loss": 0.9693,
      "odds_ratio_loss": 0.013113953173160553,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0967956930398941,
      "rewards/margins": 0.4305671751499176,
      "rewards/rejected": -0.5273628830909729,
      "sft_loss": 0.9679568409919739,
      "step": 220
    },
    {
      "epoch": 0.7349938750510412,
      "grad_norm": 2.346754312515259,
      "learning_rate": 4.294779807661105e-05,
      "logits/chosen": -0.2690557539463043,
      "logits/rejected": -1.0382682085037231,
      "logps/chosen": -0.8765946626663208,
      "logps/rejected": -5.27601432800293,
      "loss": 0.8783,
      "odds_ratio_loss": 0.016597110778093338,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08765946328639984,
      "rewards/margins": 0.43994197249412537,
      "rewards/rejected": -0.527601420879364,
      "sft_loss": 0.8765946626663208,
      "step": 225
    },
    {
      "epoch": 0.7513270722743978,
      "grad_norm": 1.543803095817566,
      "learning_rate": 4.270789202181204e-05,
      "logits/chosen": -0.3406456410884857,
      "logits/rejected": -1.0243726968765259,
      "logps/chosen": -0.938999354839325,
      "logps/rejected": -5.179265022277832,
      "loss": 0.9411,
      "odds_ratio_loss": 0.021242566406726837,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09389995038509369,
      "rewards/margins": 0.4240265488624573,
      "rewards/rejected": -0.5179265141487122,
      "sft_loss": 0.938999354839325,
      "step": 230
    },
    {
      "epoch": 0.7676602694977542,
      "grad_norm": 1.8233140707015991,
      "learning_rate": 4.2403348649073174e-05,
      "logits/chosen": -0.2546311020851135,
      "logits/rejected": -0.9541198015213013,
      "logps/chosen": -0.7934538722038269,
      "logps/rejected": -5.206686973571777,
      "loss": 0.7944,
      "odds_ratio_loss": 0.00939333625137806,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07934539020061493,
      "rewards/margins": 0.4413233697414398,
      "rewards/rejected": -0.5206687450408936,
      "sft_loss": 0.7934538722038269,
      "step": 235
    },
    {
      "epoch": 0.7839934667211107,
      "grad_norm": 1.5795395374298096,
      "learning_rate": 4.209370989589013e-05,
      "logits/chosen": -0.3499365448951721,
      "logits/rejected": -1.0148460865020752,
      "logps/chosen": -0.7999550104141235,
      "logps/rejected": -5.092876434326172,
      "loss": 0.8013,
      "odds_ratio_loss": 0.013519616797566414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07999551296234131,
      "rewards/margins": 0.4292922019958496,
      "rewards/rejected": -0.5092877149581909,
      "sft_loss": 0.7999550104141235,
      "step": 240
    },
    {
      "epoch": 0.8003266639444672,
      "grad_norm": 1.8533421754837036,
      "learning_rate": 4.177906641879575e-05,
      "logits/chosen": -0.3404558598995209,
      "logits/rejected": -1.0349606275558472,
      "logps/chosen": -0.9274562001228333,
      "logps/rejected": -4.853649616241455,
      "loss": 0.9298,
      "odds_ratio_loss": 0.023337265476584435,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09274561703205109,
      "rewards/margins": 0.3926192820072174,
      "rewards/rejected": -0.4853649139404297,
      "sft_loss": 0.9274562001228333,
      "step": 245
    },
    {
      "epoch": 0.8166598611678236,
      "grad_norm": 1.792864441871643,
      "learning_rate": 4.145951033961395e-05,
      "logits/chosen": -0.39237070083618164,
      "logits/rejected": -1.20218825340271,
      "logps/chosen": -0.8956495523452759,
      "logps/rejected": -4.416346549987793,
      "loss": 0.8988,
      "odds_ratio_loss": 0.03134030103683472,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08956494927406311,
      "rewards/margins": 0.3520697355270386,
      "rewards/rejected": -0.4416346549987793,
      "sft_loss": 0.8956495523452759,
      "step": 250
    },
    {
      "epoch": 0.8329930583911801,
      "grad_norm": 1.388491153717041,
      "learning_rate": 4.1135135218488206e-05,
      "logits/chosen": -0.4966661036014557,
      "logits/rejected": -1.301102638244629,
      "logps/chosen": -0.822670578956604,
      "logps/rejected": -4.420852184295654,
      "loss": 0.8251,
      "odds_ratio_loss": 0.02416636422276497,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08226706087589264,
      "rewards/margins": 0.3598182201385498,
      "rewards/rejected": -0.4420852065086365,
      "sft_loss": 0.822670578956604,
      "step": 255
    },
    {
      "epoch": 0.8493262556145366,
      "grad_norm": 1.7927485704421997,
      "learning_rate": 4.080603602648887e-05,
      "logits/chosen": -0.38711199164390564,
      "logits/rejected": -1.1762360334396362,
      "logps/chosen": -0.8747900128364563,
      "logps/rejected": -4.844551086425781,
      "loss": 0.8772,
      "odds_ratio_loss": 0.024142403155565262,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08747899532318115,
      "rewards/margins": 0.3969760835170746,
      "rewards/rejected": -0.4844551086425781,
      "sft_loss": 0.8747900128364563,
      "step": 260
    },
    {
      "epoch": 0.865659452837893,
      "grad_norm": 1.763637661933899,
      "learning_rate": 4.047230911780737e-05,
      "logits/chosen": -0.4611227512359619,
      "logits/rejected": -1.092810034751892,
      "logps/chosen": -0.8380341529846191,
      "logps/rejected": -4.902915000915527,
      "loss": 0.8414,
      "odds_ratio_loss": 0.03330729901790619,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08380343019962311,
      "rewards/margins": 0.4064881205558777,
      "rewards/rejected": -0.4902915358543396,
      "sft_loss": 0.8380341529846191,
      "step": 265
    },
    {
      "epoch": 0.8819926500612495,
      "grad_norm": 1.7203322649002075,
      "learning_rate": 4.0134052201545447e-05,
      "logits/chosen": -0.24241454899311066,
      "logits/rejected": -0.9308720827102661,
      "logps/chosen": -0.8675628900527954,
      "logps/rejected": -5.390278339385986,
      "loss": 0.8685,
      "odds_ratio_loss": 0.009246011264622211,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08675628900527954,
      "rewards/margins": 0.4522715210914612,
      "rewards/rejected": -0.5390278100967407,
      "sft_loss": 0.8675628900527954,
      "step": 270
    },
    {
      "epoch": 0.898325847284606,
      "grad_norm": 1.9116899967193604,
      "learning_rate": 3.9791364313107804e-05,
      "logits/chosen": -0.401073157787323,
      "logits/rejected": -0.9959608912467957,
      "logps/chosen": -0.8706862330436707,
      "logps/rejected": -5.204241752624512,
      "loss": 0.872,
      "odds_ratio_loss": 0.0133589506149292,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08706863224506378,
      "rewards/margins": 0.4333556294441223,
      "rewards/rejected": -0.5204241871833801,
      "sft_loss": 0.8706862330436707,
      "step": 275
    },
    {
      "epoch": 0.9146590445079624,
      "grad_norm": 1.9296520948410034,
      "learning_rate": 3.9444345785206285e-05,
      "logits/chosen": -0.40887147188186646,
      "logits/rejected": -0.9652082324028015,
      "logps/chosen": -0.8309628367424011,
      "logps/rejected": -5.662296295166016,
      "loss": 0.8318,
      "odds_ratio_loss": 0.008708246983587742,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08309628069400787,
      "rewards/margins": 0.48313331604003906,
      "rewards/rejected": -0.5662296414375305,
      "sft_loss": 0.8309628367424011,
      "step": 280
    },
    {
      "epoch": 0.9309922417313189,
      "grad_norm": 3.055554151535034,
      "learning_rate": 3.909309821848432e-05,
      "logits/chosen": -0.3440246284008026,
      "logits/rejected": -0.8914414644241333,
      "logps/chosen": -0.8914461135864258,
      "logps/rejected": -5.75334358215332,
      "loss": 0.8921,
      "odds_ratio_loss": 0.0068351454101502895,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08914461731910706,
      "rewards/margins": 0.48618975281715393,
      "rewards/rejected": -0.575334370136261,
      "sft_loss": 0.8914461135864258,
      "step": 285
    },
    {
      "epoch": 0.9473254389546754,
      "grad_norm": 1.5384926795959473,
      "learning_rate": 3.873772445177015e-05,
      "logits/chosen": -0.37930798530578613,
      "logits/rejected": -0.9451101422309875,
      "logps/chosen": -0.8741447329521179,
      "logps/rejected": -5.84790563583374,
      "loss": 0.8751,
      "odds_ratio_loss": 0.009465084411203861,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08741448074579239,
      "rewards/margins": 0.49737611413002014,
      "rewards/rejected": -0.5847905874252319,
      "sft_loss": 0.8741447329521179,
      "step": 290
    },
    {
      "epoch": 0.9636586361780318,
      "grad_norm": 2.4715216159820557,
      "learning_rate": 3.837832853196751e-05,
      "logits/chosen": -0.3153218626976013,
      "logits/rejected": -0.9612942934036255,
      "logps/chosen": -0.8935874700546265,
      "logps/rejected": -5.140650749206543,
      "loss": 0.8956,
      "odds_ratio_loss": 0.01970650628209114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08935876190662384,
      "rewards/margins": 0.4247063100337982,
      "rewards/rejected": -0.514065146446228,
      "sft_loss": 0.8935874700546265,
      "step": 295
    },
    {
      "epoch": 0.9799918334013883,
      "grad_norm": 1.9313322305679321,
      "learning_rate": 3.8015015683592575e-05,
      "logits/chosen": -0.30432718992233276,
      "logits/rejected": -1.099277377128601,
      "logps/chosen": -0.8711298108100891,
      "logps/rejected": -4.722751140594482,
      "loss": 0.873,
      "odds_ratio_loss": 0.018964512273669243,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08711298555135727,
      "rewards/margins": 0.3851621747016907,
      "rewards/rejected": -0.47227516770362854,
      "sft_loss": 0.8711298108100891,
      "step": 300
    },
    {
      "epoch": 0.9963250306247448,
      "grad_norm": 2.2331244945526123,
      "learning_rate": 3.764789227796616e-05,
      "logits/chosen": -0.3758210837841034,
      "logits/rejected": -1.1826483011245728,
      "logps/chosen": -0.8142495155334473,
      "logps/rejected": -4.7738471031188965,
      "loss": 0.816,
      "odds_ratio_loss": 0.017485955730080605,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08142495155334473,
      "rewards/margins": 0.3959597945213318,
      "rewards/rejected": -0.4773847460746765,
      "sft_loss": 0.8142495155334473,
      "step": 305
    },
    {
      "epoch": 1.0126582278481013,
      "grad_norm": 1.7077336311340332,
      "learning_rate": 3.727706580207021e-05,
      "logits/chosen": -0.38093143701553345,
      "logits/rejected": -1.096470594406128,
      "logps/chosen": -0.8485842943191528,
      "logps/rejected": -4.818546295166016,
      "loss": 0.8503,
      "odds_ratio_loss": 0.01746557094156742,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08485842496156693,
      "rewards/margins": 0.3969961702823639,
      "rewards/rejected": -0.4818546175956726,
      "sft_loss": 0.8485842943191528,
      "step": 310
    },
    {
      "epoch": 1.0289914250714578,
      "grad_norm": 1.8160737752914429,
      "learning_rate": 3.69026448270775e-05,
      "logits/chosen": -0.37943610548973083,
      "logits/rejected": -1.0238980054855347,
      "logps/chosen": -0.8509234189987183,
      "logps/rejected": -5.104341506958008,
      "loss": 0.8521,
      "odds_ratio_loss": 0.011288631707429886,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08509235084056854,
      "rewards/margins": 0.4253418445587158,
      "rewards/rejected": -0.5104341506958008,
      "sft_loss": 0.8509234189987183,
      "step": 315
    },
    {
      "epoch": 1.0453246222948143,
      "grad_norm": 2.9576852321624756,
      "learning_rate": 3.652473897656409e-05,
      "logits/chosen": -0.30405062437057495,
      "logits/rejected": -0.9463305473327637,
      "logps/chosen": -0.9074727296829224,
      "logps/rejected": -5.486522674560547,
      "loss": 0.9089,
      "odds_ratio_loss": 0.013948513194918633,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09074728190898895,
      "rewards/margins": 0.45790499448776245,
      "rewards/rejected": -0.5486522912979126,
      "sft_loss": 0.9074727296829224,
      "step": 320
    },
    {
      "epoch": 1.0616578195181707,
      "grad_norm": 1.9609084129333496,
      "learning_rate": 3.6143458894413465e-05,
      "logits/chosen": -0.39042356610298157,
      "logits/rejected": -1.0480425357818604,
      "logps/chosen": -0.7837303876876831,
      "logps/rejected": -5.29976749420166,
      "loss": 0.7846,
      "odds_ratio_loss": 0.008555177599191666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0783730298280716,
      "rewards/margins": 0.45160380005836487,
      "rewards/rejected": -0.5299768447875977,
      "sft_loss": 0.7837303876876831,
      "step": 325
    },
    {
      "epoch": 1.0779910167415272,
      "grad_norm": 2.609851360321045,
      "learning_rate": 3.575891621242209e-05,
      "logits/chosen": -0.35802769660949707,
      "logits/rejected": -0.9912473559379578,
      "logps/chosen": -0.8010882139205933,
      "logps/rejected": -5.255009651184082,
      "loss": 0.8022,
      "odds_ratio_loss": 0.010739787481725216,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08010882139205933,
      "rewards/margins": 0.4453921914100647,
      "rewards/rejected": -0.525501012802124,
      "sft_loss": 0.8010882139205933,
      "step": 330
    },
    {
      "epoch": 1.0943242139648837,
      "grad_norm": 1.9164317846298218,
      "learning_rate": 3.537122351761568e-05,
      "logits/chosen": -0.29361772537231445,
      "logits/rejected": -0.9894407987594604,
      "logps/chosen": -0.8663752675056458,
      "logps/rejected": -5.1811041831970215,
      "loss": 0.8682,
      "odds_ratio_loss": 0.01779218763113022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08663753420114517,
      "rewards/margins": 0.43147286772727966,
      "rewards/rejected": -0.5181103944778442,
      "sft_loss": 0.8663752675056458,
      "step": 335
    },
    {
      "epoch": 1.1106574111882401,
      "grad_norm": 1.8551751375198364,
      "learning_rate": 3.4980494319285775e-05,
      "logits/chosen": -0.2934134900569916,
      "logits/rejected": -0.9298968315124512,
      "logps/chosen": -0.8729950189590454,
      "logps/rejected": -5.402509689331055,
      "loss": 0.8739,
      "odds_ratio_loss": 0.009193124249577522,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08729950338602066,
      "rewards/margins": 0.45295149087905884,
      "rewards/rejected": -0.5402510166168213,
      "sft_loss": 0.8729950189590454,
      "step": 340
    },
    {
      "epoch": 1.1269906084115966,
      "grad_norm": 1.8746849298477173,
      "learning_rate": 3.45868430157563e-05,
      "logits/chosen": -0.27233654260635376,
      "logits/rejected": -0.9365426301956177,
      "logps/chosen": -0.8479824066162109,
      "logps/rejected": -5.225064754486084,
      "loss": 0.8493,
      "odds_ratio_loss": 0.013331398367881775,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08479823917150497,
      "rewards/margins": 0.43770831823349,
      "rewards/rejected": -0.522506594657898,
      "sft_loss": 0.8479824066162109,
      "step": 345
    },
    {
      "epoch": 1.143323805634953,
      "grad_norm": 1.7111650705337524,
      "learning_rate": 3.4190384860889824e-05,
      "logits/chosen": -0.366852343082428,
      "logits/rejected": -1.0381531715393066,
      "logps/chosen": -0.8182533383369446,
      "logps/rejected": -5.278301239013672,
      "loss": 0.8204,
      "odds_ratio_loss": 0.02150268480181694,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08182532340288162,
      "rewards/margins": 0.4460047781467438,
      "rewards/rejected": -0.5278300642967224,
      "sft_loss": 0.8182533383369446,
      "step": 350
    },
    {
      "epoch": 1.1596570028583095,
      "grad_norm": 1.9346224069595337,
      "learning_rate": 3.379123593034342e-05,
      "logits/chosen": -0.2924169898033142,
      "logits/rejected": -1.028192400932312,
      "logps/chosen": -0.9392467737197876,
      "logps/rejected": -5.247817516326904,
      "loss": 0.9409,
      "odds_ratio_loss": 0.0163104310631752,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09392467886209488,
      "rewards/margins": 0.4308570921421051,
      "rewards/rejected": -0.5247817635536194,
      "sft_loss": 0.9392467737197876,
      "step": 355
    },
    {
      "epoch": 1.175990200081666,
      "grad_norm": 1.9083410501480103,
      "learning_rate": 3.3389513087583754e-05,
      "logits/chosen": -0.1519567221403122,
      "logits/rejected": -1.0742218494415283,
      "logps/chosen": -0.8435864448547363,
      "logps/rejected": -4.644381523132324,
      "loss": 0.8465,
      "odds_ratio_loss": 0.028982138261198997,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08435864001512527,
      "rewards/margins": 0.3800795078277588,
      "rewards/rejected": -0.4644381105899811,
      "sft_loss": 0.8435864448547363,
      "step": 360
    },
    {
      "epoch": 1.1923233973050225,
      "grad_norm": 1.8590240478515625,
      "learning_rate": 3.2985333949671685e-05,
      "logits/chosen": -0.43951839208602905,
      "logits/rejected": -1.1476421356201172,
      "logps/chosen": -0.8573881387710571,
      "logps/rejected": -4.641860485076904,
      "loss": 0.8605,
      "odds_ratio_loss": 0.030726104974746704,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08573882281780243,
      "rewards/margins": 0.3784472346305847,
      "rewards/rejected": -0.4641861021518707,
      "sft_loss": 0.8573881387710571,
      "step": 365
    },
    {
      "epoch": 1.208656594528379,
      "grad_norm": 2.064635753631592,
      "learning_rate": 3.257881685282609e-05,
      "logits/chosen": -0.2586503028869629,
      "logits/rejected": -1.1886835098266602,
      "logps/chosen": -0.7855366468429565,
      "logps/rejected": -4.374009132385254,
      "loss": 0.7886,
      "odds_ratio_loss": 0.03049846924841404,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07855366170406342,
      "rewards/margins": 0.3588472604751587,
      "rewards/rejected": -0.4374009668827057,
      "sft_loss": 0.7855366468429565,
      "step": 370
    },
    {
      "epoch": 1.2249897917517354,
      "grad_norm": 1.986666202545166,
      "learning_rate": 3.217008081777726e-05,
      "logits/chosen": -0.3716577887535095,
      "logits/rejected": -1.2067687511444092,
      "logps/chosen": -0.8298207521438599,
      "logps/rejected": -4.59980583190918,
      "loss": 0.8315,
      "odds_ratio_loss": 0.016955239698290825,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08298207074403763,
      "rewards/margins": 0.3769984841346741,
      "rewards/rejected": -0.4599805772304535,
      "sft_loss": 0.8298207521438599,
      "step": 375
    },
    {
      "epoch": 1.2413229889750919,
      "grad_norm": 1.8263311386108398,
      "learning_rate": 3.175924551491977e-05,
      "logits/chosen": -0.3963642716407776,
      "logits/rejected": -1.1271188259124756,
      "logps/chosen": -0.8212121725082397,
      "logps/rejected": -4.6982741355896,
      "loss": 0.8234,
      "odds_ratio_loss": 0.02219826728105545,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08212121576070786,
      "rewards/margins": 0.3877061605453491,
      "rewards/rejected": -0.4698273539543152,
      "sft_loss": 0.8212121725082397,
      "step": 380
    },
    {
      "epoch": 1.2576561861984483,
      "grad_norm": 1.946442723274231,
      "learning_rate": 3.13464312292752e-05,
      "logits/chosen": -0.343479186296463,
      "logits/rejected": -0.9710591435432434,
      "logps/chosen": -0.8622458577156067,
      "logps/rejected": -5.370631217956543,
      "loss": 0.8646,
      "odds_ratio_loss": 0.023645440116524696,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08622457832098007,
      "rewards/margins": 0.450838565826416,
      "rewards/rejected": -0.5370631217956543,
      "sft_loss": 0.8622458577156067,
      "step": 385
    },
    {
      "epoch": 1.2739893834218048,
      "grad_norm": 1.7212151288986206,
      "learning_rate": 3.093175882527491e-05,
      "logits/chosen": -0.3060234785079956,
      "logits/rejected": -1.022698163986206,
      "logps/chosen": -0.8830709457397461,
      "logps/rejected": -5.35630464553833,
      "loss": 0.8848,
      "odds_ratio_loss": 0.01716751977801323,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08830709010362625,
      "rewards/margins": 0.44732338190078735,
      "rewards/rejected": -0.535630464553833,
      "sft_loss": 0.8830709457397461,
      "step": 390
    },
    {
      "epoch": 1.2903225806451613,
      "grad_norm": 2.029407024383545,
      "learning_rate": 3.0515349711373146e-05,
      "logits/chosen": -0.13591887056827545,
      "logits/rejected": -0.9809386134147644,
      "logps/chosen": -0.880872368812561,
      "logps/rejected": -5.28397798538208,
      "loss": 0.8826,
      "odds_ratio_loss": 0.01720813475549221,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08808723837137222,
      "rewards/margins": 0.4403105676174164,
      "rewards/rejected": -0.5283978581428528,
      "sft_loss": 0.880872368812561,
      "step": 395
    },
    {
      "epoch": 1.3066557778685177,
      "grad_norm": 1.8320516347885132,
      "learning_rate": 3.0097325804500864e-05,
      "logits/chosen": -0.3814406991004944,
      "logits/rejected": -1.0283949375152588,
      "logps/chosen": -0.7927986979484558,
      "logps/rejected": -5.100050926208496,
      "loss": 0.7945,
      "odds_ratio_loss": 0.016741547733545303,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07927986979484558,
      "rewards/margins": 0.4307252764701843,
      "rewards/rejected": -0.5100051164627075,
      "sft_loss": 0.7927986979484558,
      "step": 400
    }
  ],
  "logging_steps": 5,
  "max_steps": 918,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100,
  "total_flos": 1.663517229609124e+17,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}