{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 726,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 6.84931506849315e-09,
      "logits/generated": -3.1861095428466797,
      "logits/real": -2.9340975284576416,
      "logps/generated": -105.21376037597656,
      "logps/real": -164.82391357421875,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/generated": 0.0,
      "rewards/margins": 0.0,
      "rewards/real": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 6.84931506849315e-08,
      "logits/generated": -3.0733141899108887,
      "logits/real": -2.8458805084228516,
      "logps/generated": -100.71701049804688,
      "logps/real": -112.07172393798828,
      "loss": 0.5947,
      "rewards/accuracies": 0.8472222089767456,
      "rewards/generated": -0.17832139134407043,
      "rewards/margins": 0.22786876559257507,
      "rewards/real": 0.049547407776117325,
      "step": 10
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.36986301369863e-07,
      "logits/generated": -3.0346713066101074,
      "logits/real": -2.7779345512390137,
      "logps/generated": -114.3007583618164,
      "logps/real": -119.0119400024414,
      "loss": 0.1735,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -1.5705761909484863,
      "rewards/margins": 1.957914113998413,
      "rewards/real": 0.38733819127082825,
      "step": 20
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.054794520547945e-07,
      "logits/generated": -2.9741339683532715,
      "logits/real": -2.7195401191711426,
      "logps/generated": -124.48112487792969,
      "logps/real": -117.0356674194336,
      "loss": 0.0677,
      "rewards/accuracies": 1.0,
      "rewards/generated": -2.8203659057617188,
      "rewards/margins": 3.6289095878601074,
      "rewards/real": 0.808543860912323,
      "step": 30
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.73972602739726e-07,
      "logits/generated": -2.898865222930908,
      "logits/real": -2.7205348014831543,
      "logps/generated": -145.01361083984375,
      "logps/real": -122.72618103027344,
      "loss": 0.0377,
      "rewards/accuracies": 1.0,
      "rewards/generated": -4.186580181121826,
      "rewards/margins": 5.1676530838012695,
      "rewards/real": 0.9810730814933777,
      "step": 40
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.424657534246575e-07,
      "logits/generated": -2.9261574745178223,
      "logits/real": -2.659583806991577,
      "logps/generated": -144.99615478515625,
      "logps/real": -105.9330062866211,
      "loss": 0.0265,
      "rewards/accuracies": 1.0,
      "rewards/generated": -4.944008827209473,
      "rewards/margins": 5.950331687927246,
      "rewards/real": 1.0063225030899048,
      "step": 50
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.10958904109589e-07,
      "logits/generated": -2.8922829627990723,
      "logits/real": -2.7115025520324707,
      "logps/generated": -149.0370330810547,
      "logps/real": -123.13972473144531,
      "loss": 0.0213,
      "rewards/accuracies": 1.0,
      "rewards/generated": -5.328394889831543,
      "rewards/margins": 6.367083549499512,
      "rewards/real": 1.0386886596679688,
      "step": 60
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.794520547945205e-07,
      "logits/generated": -2.886918544769287,
      "logits/real": -2.680799722671509,
      "logps/generated": -163.05960083007812,
      "logps/real": -119.51402282714844,
      "loss": 0.011,
      "rewards/accuracies": 1.0,
      "rewards/generated": -6.317183494567871,
      "rewards/margins": 7.2489824295043945,
      "rewards/real": 0.9317992329597473,
      "step": 70
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.946401225114854e-07,
      "logits/generated": -2.8313755989074707,
      "logits/real": -2.657686471939087,
      "logps/generated": -169.63497924804688,
      "logps/real": -131.515625,
      "loss": 0.0116,
      "rewards/accuracies": 1.0,
      "rewards/generated": -7.619760990142822,
      "rewards/margins": 8.610966682434082,
      "rewards/real": 0.9912055134773254,
      "step": 80
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.869831546707504e-07,
      "logits/generated": -2.812246799468994,
      "logits/real": -2.5690789222717285,
      "logps/generated": -178.45095825195312,
      "logps/real": -121.57679748535156,
      "loss": 0.009,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -8.4043550491333,
      "rewards/margins": 9.33082389831543,
      "rewards/real": 0.9264704585075378,
      "step": 90
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.793261868300153e-07,
      "logits/generated": -2.7975776195526123,
      "logits/real": -2.5637221336364746,
      "logps/generated": -188.71710205078125,
      "logps/real": -113.59111022949219,
      "loss": 0.006,
      "rewards/accuracies": 1.0,
      "rewards/generated": -8.973402976989746,
      "rewards/margins": 9.793700218200684,
      "rewards/real": 0.82029789686203,
      "step": 100
    },
    {
      "epoch": 0.14,
      "eval_logits/generated": -2.7878057956695557,
      "eval_logits/real": -2.5727546215057373,
      "eval_logps/generated": -183.79859924316406,
      "eval_logps/real": -119.36829376220703,
      "eval_loss": 0.010346903465688229,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/generated": -8.943349838256836,
      "eval_rewards/margins": 9.82614517211914,
      "eval_rewards/real": 0.8827962279319763,
      "eval_runtime": 161.2178,
      "eval_samples_per_second": 14.645,
      "eval_steps_per_second": 0.459,
      "step": 100
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.7166921898928023e-07,
      "logits/generated": -2.7586779594421387,
      "logits/real": -2.4634366035461426,
      "logps/generated": -180.26075744628906,
      "logps/real": -109.0326919555664,
      "loss": 0.0103,
      "rewards/accuracies": 1.0,
      "rewards/generated": -8.940742492675781,
      "rewards/margins": 9.569414138793945,
      "rewards/real": 0.628670334815979,
      "step": 110
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.640122511485451e-07,
      "logits/generated": -2.798537492752075,
      "logits/real": -2.526486873626709,
      "logps/generated": -205.32275390625,
      "logps/real": -125.221435546875,
      "loss": 0.0043,
      "rewards/accuracies": 1.0,
      "rewards/generated": -10.294915199279785,
      "rewards/margins": 11.142276763916016,
      "rewards/real": 0.8473619222640991,
      "step": 120
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.563552833078101e-07,
      "logits/generated": -2.754697561264038,
      "logits/real": -2.4235503673553467,
      "logps/generated": -207.21493530273438,
      "logps/real": -115.43656158447266,
      "loss": 0.0066,
      "rewards/accuracies": 1.0,
      "rewards/generated": -10.628061294555664,
      "rewards/margins": 10.883023262023926,
      "rewards/real": 0.2549612820148468,
      "step": 130
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.4869831546707505e-07,
      "logits/generated": -2.7149157524108887,
      "logits/real": -2.3472886085510254,
      "logps/generated": -184.88845825195312,
      "logps/real": -130.86801147460938,
      "loss": 0.0086,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -9.176107406616211,
      "rewards/margins": 10.01699161529541,
      "rewards/real": 0.8408846855163574,
      "step": 140
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.4104134762633994e-07,
      "logits/generated": -2.7166011333465576,
      "logits/real": -2.2654507160186768,
      "logps/generated": -204.56402587890625,
      "logps/real": -107.61639404296875,
      "loss": 0.0068,
      "rewards/accuracies": 1.0,
      "rewards/generated": -11.028618812561035,
      "rewards/margins": 11.473901748657227,
      "rewards/real": 0.44528260827064514,
      "step": 150
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.333843797856049e-07,
      "logits/generated": -2.6574273109436035,
      "logits/real": -2.2875142097473145,
      "logps/generated": -231.9933624267578,
      "logps/real": -116.0688705444336,
      "loss": 0.0063,
      "rewards/accuracies": 1.0,
      "rewards/generated": -12.995275497436523,
      "rewards/margins": 13.312971115112305,
      "rewards/real": 0.3176959753036499,
      "step": 160
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.257274119448698e-07,
      "logits/generated": -2.7084014415740967,
      "logits/real": -2.394038677215576,
      "logps/generated": -216.3066864013672,
      "logps/real": -125.7446060180664,
      "loss": 0.005,
      "rewards/accuracies": 1.0,
      "rewards/generated": -12.051506042480469,
      "rewards/margins": 12.512662887573242,
      "rewards/real": 0.4611574113368988,
      "step": 170
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.180704441041347e-07,
      "logits/generated": -2.6622118949890137,
      "logits/real": -2.2468948364257812,
      "logps/generated": -225.3508758544922,
      "logps/real": -117.0610122680664,
      "loss": 0.0045,
      "rewards/accuracies": 1.0,
      "rewards/generated": -12.793501853942871,
      "rewards/margins": 13.075098037719727,
      "rewards/real": 0.2815958559513092,
      "step": 180
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.1041347626339966e-07,
      "logits/generated": -2.6490254402160645,
      "logits/real": -2.247366428375244,
      "logps/generated": -226.062255859375,
      "logps/real": -122.72492980957031,
      "loss": 0.0014,
      "rewards/accuracies": 1.0,
      "rewards/generated": -13.04449462890625,
      "rewards/margins": 13.306245803833008,
      "rewards/real": 0.2617505192756653,
      "step": 190
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.027565084226646e-07,
      "logits/generated": -2.656899929046631,
      "logits/real": -2.357235908508301,
      "logps/generated": -225.40185546875,
      "logps/real": -132.7057342529297,
      "loss": 0.0049,
      "rewards/accuracies": 1.0,
      "rewards/generated": -13.09276294708252,
      "rewards/margins": 13.024332046508789,
      "rewards/real": -0.06843126565217972,
      "step": 200
    },
    {
      "epoch": 0.28,
      "eval_logits/generated": -2.6820476055145264,
      "eval_logits/real": -2.3354389667510986,
      "eval_logps/generated": -222.9388427734375,
      "eval_logps/real": -131.47967529296875,
      "eval_loss": 0.00588552001863718,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/generated": -12.85737419128418,
      "eval_rewards/margins": 12.529033660888672,
      "eval_rewards/real": -0.3283398747444153,
      "eval_runtime": 158.2352,
      "eval_samples_per_second": 14.921,
      "eval_steps_per_second": 0.468,
      "step": 200
    },
    {
      "epoch": 0.29,
      "learning_rate": 3.9509954058192954e-07,
      "logits/generated": -2.6826717853546143,
      "logits/real": -2.3030552864074707,
      "logps/generated": -230.2223663330078,
      "logps/real": -134.5888214111328,
      "loss": 0.0029,
      "rewards/accuracies": 1.0,
      "rewards/generated": -13.429452896118164,
      "rewards/margins": 13.069856643676758,
      "rewards/real": -0.3595971167087555,
      "step": 210
    },
    {
      "epoch": 0.3,
      "learning_rate": 3.874425727411945e-07,
      "logits/generated": -2.6574368476867676,
      "logits/real": -2.2179970741271973,
      "logps/generated": -244.1143341064453,
      "logps/real": -144.1733856201172,
      "loss": 0.0027,
      "rewards/accuracies": 1.0,
      "rewards/generated": -14.808183670043945,
      "rewards/margins": 14.093585014343262,
      "rewards/real": -0.7145983576774597,
      "step": 220
    },
    {
      "epoch": 0.32,
      "learning_rate": 3.797856049004594e-07,
      "logits/generated": -2.634281873703003,
      "logits/real": -2.218191385269165,
      "logps/generated": -256.09527587890625,
      "logps/real": -127.336181640625,
      "loss": 0.0032,
      "rewards/accuracies": 1.0,
      "rewards/generated": -15.782757759094238,
      "rewards/margins": 15.013738632202148,
      "rewards/real": -0.7690176367759705,
      "step": 230
    },
    {
      "epoch": 0.33,
      "learning_rate": 3.7212863705972436e-07,
      "logits/generated": -2.6072771549224854,
      "logits/real": -2.244497060775757,
      "logps/generated": -253.4292755126953,
      "logps/real": -136.6222381591797,
      "loss": 0.0014,
      "rewards/accuracies": 1.0,
      "rewards/generated": -15.693078994750977,
      "rewards/margins": 15.163601875305176,
      "rewards/real": -0.5294772386550903,
      "step": 240
    },
    {
      "epoch": 0.34,
      "learning_rate": 3.6447166921898925e-07,
      "logits/generated": -2.556964159011841,
      "logits/real": -2.236912727355957,
      "logps/generated": -249.5274658203125,
      "logps/real": -137.39012145996094,
      "loss": 0.0019,
      "rewards/accuracies": 1.0,
      "rewards/generated": -15.941569328308105,
      "rewards/margins": 15.793266296386719,
      "rewards/real": -0.1483028382062912,
      "step": 250
    },
    {
      "epoch": 0.36,
      "learning_rate": 3.568147013782542e-07,
      "logits/generated": -2.5733115673065186,
      "logits/real": -2.228128433227539,
      "logps/generated": -269.9759216308594,
      "logps/real": -143.72103881835938,
      "loss": 0.0014,
      "rewards/accuracies": 1.0,
      "rewards/generated": -17.370920181274414,
      "rewards/margins": 17.03384017944336,
      "rewards/real": -0.33707910776138306,
      "step": 260
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.4915773353751913e-07,
      "logits/generated": -2.5637764930725098,
      "logits/real": -2.186643600463867,
      "logps/generated": -274.34075927734375,
      "logps/real": -135.88766479492188,
      "loss": 0.0009,
      "rewards/accuracies": 1.0,
      "rewards/generated": -17.811744689941406,
      "rewards/margins": 17.153844833374023,
      "rewards/real": -0.6578981876373291,
      "step": 270
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.41500765696784e-07,
      "logits/generated": -2.5118956565856934,
      "logits/real": -2.225782871246338,
      "logps/generated": -296.8309631347656,
      "logps/real": -149.24644470214844,
      "loss": 0.0031,
      "rewards/accuracies": 1.0,
      "rewards/generated": -19.82471466064453,
      "rewards/margins": 18.795507431030273,
      "rewards/real": -1.029209852218628,
      "step": 280
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.33843797856049e-07,
      "logits/generated": -2.6325273513793945,
      "logits/real": -2.219564199447632,
      "logps/generated": -274.51068115234375,
      "logps/real": -128.84051513671875,
      "loss": 0.0039,
      "rewards/accuracies": 1.0,
      "rewards/generated": -17.409244537353516,
      "rewards/margins": 17.205947875976562,
      "rewards/real": -0.2032967507839203,
      "step": 290
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.2618683001531396e-07,
      "logits/generated": -2.5958046913146973,
      "logits/real": -2.189000129699707,
      "logps/generated": -265.55938720703125,
      "logps/real": -134.06478881835938,
      "loss": 0.0008,
      "rewards/accuracies": 1.0,
      "rewards/generated": -17.167810440063477,
      "rewards/margins": 16.994720458984375,
      "rewards/real": -0.17308813333511353,
      "step": 300
    },
    {
      "epoch": 0.41,
      "eval_logits/generated": -2.61017107963562,
      "eval_logits/real": -2.2508018016815186,
      "eval_logps/generated": -265.8119812011719,
      "eval_logps/real": -131.28457641601562,
      "eval_loss": 0.0046236044727265835,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/generated": -17.144683837890625,
      "eval_rewards/margins": 16.835853576660156,
      "eval_rewards/real": -0.3088315427303314,
      "eval_runtime": 158.0301,
      "eval_samples_per_second": 14.94,
      "eval_steps_per_second": 0.468,
      "step": 300
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.1852986217457885e-07,
      "logits/generated": -2.628693103790283,
      "logits/real": -2.0444977283477783,
      "logps/generated": -268.41900634765625,
      "logps/real": -112.76142883300781,
      "loss": 0.0008,
      "rewards/accuracies": 1.0,
      "rewards/generated": -16.913591384887695,
      "rewards/margins": 16.794240951538086,
      "rewards/real": -0.11935219913721085,
      "step": 310
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.108728943338438e-07,
      "logits/generated": -2.578032970428467,
      "logits/real": -2.2168657779693604,
      "logps/generated": -287.4766845703125,
      "logps/real": -131.40872192382812,
      "loss": 0.0016,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.100343704223633,
      "rewards/margins": 17.883678436279297,
      "rewards/real": -0.21666450798511505,
      "step": 320
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.0321592649310873e-07,
      "logits/generated": -2.542041778564453,
      "logits/real": -2.211026430130005,
      "logps/generated": -275.9089660644531,
      "logps/real": -157.68031311035156,
      "loss": 0.0012,
      "rewards/accuracies": 1.0,
      "rewards/generated": -17.96003532409668,
      "rewards/margins": 17.602741241455078,
      "rewards/real": -0.3572925925254822,
      "step": 330
    },
    {
      "epoch": 0.47,
      "learning_rate": 2.955589586523736e-07,
      "logits/generated": -2.547131299972534,
      "logits/real": -2.023761034011841,
      "logps/generated": -292.7474060058594,
      "logps/real": -135.03585815429688,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/generated": -19.637466430664062,
      "rewards/margins": 19.64037322998047,
      "rewards/real": 0.0029046223498880863,
      "step": 340
    },
    {
      "epoch": 0.48,
      "learning_rate": 2.8790199081163856e-07,
      "logits/generated": -2.4998486042022705,
      "logits/real": -2.0819802284240723,
      "logps/generated": -302.1818542480469,
      "logps/real": -130.52798461914062,
      "loss": 0.0017,
      "rewards/accuracies": 1.0,
      "rewards/generated": -20.483074188232422,
      "rewards/margins": 20.199382781982422,
      "rewards/real": -0.2836933135986328,
      "step": 350
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.802450229709035e-07,
      "logits/generated": -2.56754994392395,
      "logits/real": -2.195291757583618,
      "logps/generated": -271.90869140625,
      "logps/real": -145.84982299804688,
      "loss": 0.01,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -17.539487838745117,
      "rewards/margins": 17.163063049316406,
      "rewards/real": -0.37642478942871094,
      "step": 360
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.725880551301684e-07,
      "logits/generated": -2.7496495246887207,
      "logits/real": -2.407241106033325,
      "logps/generated": -239.4289093017578,
      "logps/real": -133.95068359375,
      "loss": 0.0013,
      "rewards/accuracies": 1.0,
      "rewards/generated": -13.810670852661133,
      "rewards/margins": 13.17461109161377,
      "rewards/real": -0.6360600590705872,
      "step": 370
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.649310872894334e-07,
      "logits/generated": -2.729011297225952,
      "logits/real": -2.2989113330841064,
      "logps/generated": -237.5382537841797,
      "logps/real": -129.49656677246094,
      "loss": 0.0022,
      "rewards/accuracies": 1.0,
      "rewards/generated": -14.785906791687012,
      "rewards/margins": 13.848393440246582,
      "rewards/real": -0.937515377998352,
      "step": 380
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.572741194486983e-07,
      "logits/generated": -2.7313809394836426,
      "logits/real": -2.293959856033325,
      "logps/generated": -245.7303466796875,
      "logps/real": -142.39744567871094,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/generated": -15.5045804977417,
      "rewards/margins": 14.365577697753906,
      "rewards/real": -1.1390012502670288,
      "step": 390
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.496171516079632e-07,
      "logits/generated": -2.715420722961426,
      "logits/real": -2.184450626373291,
      "logps/generated": -266.47711181640625,
      "logps/real": -141.78045654296875,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/generated": -16.85863494873047,
      "rewards/margins": 15.755969047546387,
      "rewards/real": -1.1026637554168701,
      "step": 400
    },
    {
      "epoch": 0.55,
      "eval_logits/generated": -2.7045209407806396,
      "eval_logits/real": -2.269023895263672,
      "eval_logps/generated": -259.7652282714844,
      "eval_logps/real": -139.35308837890625,
      "eval_loss": 0.003965151496231556,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/generated": -16.540008544921875,
      "eval_rewards/margins": 15.424325942993164,
      "eval_rewards/real": -1.1156824827194214,
      "eval_runtime": 157.4039,
      "eval_samples_per_second": 15.0,
      "eval_steps_per_second": 0.47,
      "step": 400
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4196018376722816e-07,
      "logits/generated": -2.665651559829712,
      "logits/real": -2.456461191177368,
      "logps/generated": -261.137939453125,
      "logps/real": -178.57186889648438,
      "loss": 0.0014,
      "rewards/accuracies": 1.0,
      "rewards/generated": -16.91732406616211,
      "rewards/margins": 15.843770980834961,
      "rewards/real": -1.0735524892807007,
      "step": 410
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.343032159264931e-07,
      "logits/generated": -2.702195644378662,
      "logits/real": -2.3238885402679443,
      "logps/generated": -269.06353759765625,
      "logps/real": -146.2577362060547,
      "loss": 0.0007,
      "rewards/accuracies": 1.0,
      "rewards/generated": -17.51862335205078,
      "rewards/margins": 16.02815818786621,
      "rewards/real": -1.4904645681381226,
      "step": 420
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.26646248085758e-07,
      "logits/generated": -2.6736180782318115,
      "logits/real": -2.168036699295044,
      "logps/generated": -282.86785888671875,
      "logps/real": -139.81103515625,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.390249252319336,
      "rewards/margins": 17.148635864257812,
      "rewards/real": -1.2416141033172607,
      "step": 430
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.1898928024502298e-07,
      "logits/generated": -2.669506788253784,
      "logits/real": -2.2485787868499756,
      "logps/generated": -277.55511474609375,
      "logps/real": -138.58358764648438,
      "loss": 0.0027,
      "rewards/accuracies": 1.0,
      "rewards/generated": -17.349313735961914,
      "rewards/margins": 16.14107894897461,
      "rewards/real": -1.2082334756851196,
      "step": 440
    },
    {
      "epoch": 0.62,
      "learning_rate": 2.113323124042879e-07,
      "logits/generated": -2.6607108116149902,
      "logits/real": -2.161827564239502,
      "logps/generated": -276.84698486328125,
      "logps/real": -134.13027954101562,
      "loss": 0.0034,
      "rewards/accuracies": 1.0,
      "rewards/generated": -17.76062774658203,
      "rewards/margins": 17.239999771118164,
      "rewards/real": -0.5206273794174194,
      "step": 450
    },
    {
      "epoch": 0.63,
      "learning_rate": 2.036753445635528e-07,
      "logits/generated": -2.646111249923706,
      "logits/real": -2.132166862487793,
      "logps/generated": -291.7902526855469,
      "logps/real": -139.6673583984375,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/generated": -19.08199691772461,
      "rewards/margins": 18.260135650634766,
      "rewards/real": -0.8218600153923035,
      "step": 460
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.9601837672281775e-07,
      "logits/generated": -2.6296095848083496,
      "logits/real": -2.2007174491882324,
      "logps/generated": -293.5247497558594,
      "logps/real": -130.4656524658203,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.928367614746094,
      "rewards/margins": 18.38241958618164,
      "rewards/real": -0.5459470748901367,
      "step": 470
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.883614088820827e-07,
      "logits/generated": -2.653764009475708,
      "logits/real": -2.192340612411499,
      "logps/generated": -278.79925537109375,
      "logps/real": -137.2264404296875,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/generated": -17.596893310546875,
      "rewards/margins": 17.32227897644043,
      "rewards/real": -0.274616003036499,
      "step": 480
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.807044410413476e-07,
      "logits/generated": -2.619284152984619,
      "logits/real": -2.195411443710327,
      "logps/generated": -274.4403991699219,
      "logps/real": -131.1782684326172,
      "loss": 0.0014,
      "rewards/accuracies": 1.0,
      "rewards/generated": -17.91330337524414,
      "rewards/margins": 17.0344181060791,
      "rewards/real": -0.878883957862854,
      "step": 490
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.7304747320061255e-07,
      "logits/generated": -2.5468859672546387,
      "logits/real": -2.16808819770813,
      "logps/generated": -296.60052490234375,
      "logps/real": -142.17849731445312,
      "loss": 0.0013,
      "rewards/accuracies": 1.0,
      "rewards/generated": -19.563961029052734,
      "rewards/margins": 18.861263275146484,
      "rewards/real": -0.7026968002319336,
      "step": 500
    },
    {
      "epoch": 0.69,
      "eval_logits/generated": -2.576681613922119,
      "eval_logits/real": -2.1513924598693848,
      "eval_logps/generated": -289.69342041015625,
      "eval_logps/real": -137.5797119140625,
      "eval_loss": 0.004443956073373556,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/generated": -19.5328311920166,
      "eval_rewards/margins": 18.594486236572266,
      "eval_rewards/real": -0.9383437633514404,
      "eval_runtime": 157.7519,
      "eval_samples_per_second": 14.967,
      "eval_steps_per_second": 0.469,
      "step": 500
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.6539050535987747e-07,
      "logits/generated": -2.590022325515747,
      "logits/real": -2.149199962615967,
      "logps/generated": -315.4823913574219,
      "logps/real": -154.1179656982422,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.407405853271484,
      "rewards/margins": 20.304718017578125,
      "rewards/real": -1.102686882019043,
      "step": 510
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.5773353751914243e-07,
      "logits/generated": -2.5654349327087402,
      "logits/real": -2.137620210647583,
      "logps/generated": -291.37237548828125,
      "logps/real": -140.8650360107422,
      "loss": 0.0018,
      "rewards/accuracies": 1.0,
      "rewards/generated": -19.41069984436035,
      "rewards/margins": 18.840763092041016,
      "rewards/real": -0.5699384808540344,
      "step": 520
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.5007656967840735e-07,
      "logits/generated": -2.5372917652130127,
      "logits/real": -2.0500893592834473,
      "logps/generated": -294.9787902832031,
      "logps/real": -124.251708984375,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/generated": -20.343381881713867,
      "rewards/margins": 19.414480209350586,
      "rewards/real": -0.9289019703865051,
      "step": 530
    },
    {
      "epoch": 0.74,
      "learning_rate": 1.4241960183767226e-07,
      "logits/generated": -2.5766797065734863,
      "logits/real": -2.147850513458252,
      "logps/generated": -311.2908630371094,
      "logps/real": -130.3789825439453,
      "loss": 0.0035,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.165952682495117,
      "rewards/margins": 19.730859756469727,
      "rewards/real": -1.4350941181182861,
      "step": 540
    },
    {
      "epoch": 0.76,
      "learning_rate": 1.347626339969372e-07,
      "logits/generated": -2.554757595062256,
      "logits/real": -2.2075610160827637,
      "logps/generated": -329.0444030761719,
      "logps/real": -152.28512573242188,
      "loss": 0.0011,
      "rewards/accuracies": 1.0,
      "rewards/generated": -22.91840934753418,
      "rewards/margins": 20.643354415893555,
      "rewards/real": -2.275055408477783,
      "step": 550
    },
    {
      "epoch": 0.77,
      "learning_rate": 1.2710566615620215e-07,
      "logits/generated": -2.5634827613830566,
      "logits/real": -2.147305488586426,
      "logps/generated": -311.7596740722656,
      "logps/real": -151.3507843017578,
      "loss": 0.001,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.241247177124023,
      "rewards/margins": 18.893068313598633,
      "rewards/real": -2.348182201385498,
      "step": 560
    },
    {
      "epoch": 0.79,
      "learning_rate": 1.1944869831546706e-07,
      "logits/generated": -2.5457751750946045,
      "logits/real": -2.106954336166382,
      "logps/generated": -328.623779296875,
      "logps/real": -142.12510681152344,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/generated": -23.171123504638672,
      "rewards/margins": 20.756011962890625,
      "rewards/real": -2.415114641189575,
      "step": 570
    },
    {
      "epoch": 0.8,
      "learning_rate": 1.11791730474732e-07,
      "logits/generated": -2.5510199069976807,
      "logits/real": -2.177356243133545,
      "logps/generated": -315.9309997558594,
      "logps/real": -146.3859100341797,
      "loss": 0.0024,
      "rewards/accuracies": 1.0,
      "rewards/generated": -22.496957778930664,
      "rewards/margins": 20.132160186767578,
      "rewards/real": -2.364798069000244,
      "step": 580
    },
    {
      "epoch": 0.81,
      "learning_rate": 1.0413476263399694e-07,
      "logits/generated": -2.5694589614868164,
      "logits/real": -2.1845149993896484,
      "logps/generated": -315.94287109375,
      "logps/real": -136.48629760742188,
      "loss": 0.0015,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.3291072845459,
      "rewards/margins": 19.322040557861328,
      "rewards/real": -2.007070302963257,
      "step": 590
    },
    {
      "epoch": 0.83,
      "learning_rate": 9.647779479326186e-08,
      "logits/generated": -2.5274672508239746,
      "logits/real": -2.2182910442352295,
      "logps/generated": -317.5067443847656,
      "logps/real": -161.26235961914062,
      "loss": 0.0007,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.719104766845703,
      "rewards/margins": 19.16120147705078,
      "rewards/real": -2.5579028129577637,
      "step": 600
    },
    {
      "epoch": 0.83,
      "eval_logits/generated": -2.5353777408599854,
      "eval_logits/real": -2.21551251411438,
      "eval_logps/generated": -314.74267578125,
      "eval_logps/real": -151.55438232421875,
      "eval_loss": 0.004705235827714205,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/generated": -22.037752151489258,
      "eval_rewards/margins": 19.701940536499023,
      "eval_rewards/real": -2.3358113765716553,
      "eval_runtime": 157.4672,
      "eval_samples_per_second": 14.994,
      "eval_steps_per_second": 0.47,
      "step": 600
    },
    {
      "epoch": 0.84,
      "learning_rate": 8.88208269525268e-08,
      "logits/generated": -2.5355331897735596,
      "logits/real": -2.2826364040374756,
      "logps/generated": -333.5545349121094,
      "logps/real": -168.3188934326172,
      "loss": 0.0024,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -23.02947998046875,
      "rewards/margins": 19.85544776916504,
      "rewards/real": -3.17403244972229,
      "step": 610
    },
    {
      "epoch": 0.85,
      "learning_rate": 8.116385911179173e-08,
      "logits/generated": -2.5484328269958496,
      "logits/real": -2.0927655696868896,
      "logps/generated": -329.89495849609375,
      "logps/real": -134.99539184570312,
      "loss": 0.0018,
      "rewards/accuracies": 1.0,
      "rewards/generated": -23.049104690551758,
      "rewards/margins": 20.316526412963867,
      "rewards/real": -2.7325758934020996,
      "step": 620
    },
    {
      "epoch": 0.87,
      "learning_rate": 7.350689127105667e-08,
      "logits/generated": -2.5599124431610107,
      "logits/real": -2.1835923194885254,
      "logps/generated": -327.4054260253906,
      "logps/real": -157.17245483398438,
      "loss": 0.0011,
      "rewards/accuracies": 1.0,
      "rewards/generated": -22.402978897094727,
      "rewards/margins": 20.018922805786133,
      "rewards/real": -2.3840572834014893,
      "step": 630
    },
    {
      "epoch": 0.88,
      "learning_rate": 6.584992343032159e-08,
      "logits/generated": -2.5241808891296387,
      "logits/real": -2.1015708446502686,
      "logps/generated": -323.3908996582031,
      "logps/real": -141.0902557373047,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/generated": -22.945276260375977,
      "rewards/margins": 20.808456420898438,
      "rewards/real": -2.1368181705474854,
      "step": 640
    },
    {
      "epoch": 0.9,
      "learning_rate": 5.819295558958652e-08,
      "logits/generated": -2.5636558532714844,
      "logits/real": -2.1634020805358887,
      "logps/generated": -322.5964660644531,
      "logps/real": -149.52218627929688,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.644861221313477,
      "rewards/margins": 19.158300399780273,
      "rewards/real": -2.4865593910217285,
      "step": 650
    },
    {
      "epoch": 0.91,
      "learning_rate": 5.0535987748851455e-08,
      "logits/generated": -2.509050130844116,
      "logits/real": -2.196336269378662,
      "logps/generated": -312.86407470703125,
      "logps/real": -157.67984008789062,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.931781768798828,
      "rewards/margins": 19.02187728881836,
      "rewards/real": -2.9099009037017822,
      "step": 660
    },
    {
      "epoch": 0.92,
      "learning_rate": 4.287901990811638e-08,
      "logits/generated": -2.5164096355438232,
      "logits/real": -2.170067310333252,
      "logps/generated": -326.18182373046875,
      "logps/real": -149.81845092773438,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/generated": -23.84206771850586,
      "rewards/margins": 21.606847763061523,
      "rewards/real": -2.2352216243743896,
      "step": 670
    },
    {
      "epoch": 0.94,
      "learning_rate": 3.522205206738132e-08,
      "logits/generated": -2.522449254989624,
      "logits/real": -2.093801498413086,
      "logps/generated": -325.7381286621094,
      "logps/real": -151.8564910888672,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/generated": -23.091638565063477,
      "rewards/margins": 20.554058074951172,
      "rewards/real": -2.537578821182251,
      "step": 680
    },
    {
      "epoch": 0.95,
      "learning_rate": 2.7565084226646246e-08,
      "logits/generated": -2.5449929237365723,
      "logits/real": -2.086919069290161,
      "logps/generated": -342.8669738769531,
      "logps/real": -146.0614013671875,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/generated": -24.37696075439453,
      "rewards/margins": 22.004487991333008,
      "rewards/real": -2.3724732398986816,
      "step": 690
    },
    {
      "epoch": 0.96,
      "learning_rate": 1.9908116385911178e-08,
      "logits/generated": -2.4875707626342773,
      "logits/real": -2.102121591567993,
      "logps/generated": -337.271240234375,
      "logps/real": -154.13955688476562,
      "loss": 0.0012,
      "rewards/accuracies": 1.0,
      "rewards/generated": -23.850067138671875,
      "rewards/margins": 21.494661331176758,
      "rewards/real": -2.3554062843322754,
      "step": 700
    },
    {
      "epoch": 0.96,
      "eval_logits/generated": -2.4931468963623047,
      "eval_logits/real": -2.1379408836364746,
      "eval_logps/generated": -328.0981750488281,
      "eval_logps/real": -152.5395965576172,
      "eval_loss": 0.005220742896199226,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/generated": -23.37330436706543,
      "eval_rewards/margins": 20.9389705657959,
      "eval_rewards/real": -2.4343342781066895,
      "eval_runtime": 156.7471,
      "eval_samples_per_second": 15.062,
      "eval_steps_per_second": 0.472,
      "step": 700
    },
    {
      "epoch": 0.98,
      "learning_rate": 1.225114854517611e-08,
      "logits/generated": -2.4835574626922607,
      "logits/real": -2.0778918266296387,
      "logps/generated": -326.81134033203125,
      "logps/real": -148.4219970703125,
      "loss": 0.0013,
      "rewards/accuracies": 1.0,
      "rewards/generated": -23.217302322387695,
      "rewards/margins": 20.590421676635742,
      "rewards/real": -2.626877784729004,
      "step": 710
    },
    {
      "epoch": 0.99,
      "learning_rate": 4.594180704441042e-09,
      "logits/generated": -2.512179374694824,
      "logits/real": -2.0604519844055176,
      "logps/generated": -316.1888732910156,
      "logps/real": -148.81613159179688,
      "loss": 0.0029,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.540176391601562,
      "rewards/margins": 19.17498207092285,
      "rewards/real": -2.3651950359344482,
      "step": 720
    },
    {
      "epoch": 1.0,
      "step": 726,
      "total_flos": 0.0,
      "train_loss": 0.015260709988430512,
      "train_runtime": 4886.4679,
      "train_samples_per_second": 4.753,
      "train_steps_per_second": 0.149
    }
  ],
  "logging_steps": 10,
  "max_steps": 726,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}