diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,18200 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.999297541394882,
+  "eval_steps": 400,
+  "global_step": 5604,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.002676032781401572,
+      "grad_norm": 4.125313107067299,
+      "learning_rate": 8.9126559714795e-09,
+      "logits/chosen": -0.060312606394290924,
+      "logits/rejected": 0.15203741192817688,
+      "logps/chosen": -1.7157971858978271,
+      "logps/rejected": -1.8896640539169312,
+      "loss": 0.2582,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.7157971858978271,
+      "rewards/margins": 0.17386700212955475,
+      "rewards/rejected": -1.8896640539169312,
+      "sft_loss": 1.4683139324188232,
+      "step": 5
+    },
+    {
+      "epoch": 0.005352065562803144,
+      "grad_norm": 2.7467738156031922,
+      "learning_rate": 1.7825311942959e-08,
+      "logits/chosen": 0.011781789362430573,
+      "logits/rejected": 0.13588806986808777,
+      "logps/chosen": -1.8027633428573608,
+      "logps/rejected": -1.8470537662506104,
+      "loss": 0.2651,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.8027633428573608,
+      "rewards/margins": 0.044290412217378616,
+      "rewards/rejected": -1.8470537662506104,
+      "sft_loss": 1.5084987878799438,
+      "step": 10
+    },
+    {
+      "epoch": 0.008028098344204716,
+      "grad_norm": 2.953214775219304,
+      "learning_rate": 2.67379679144385e-08,
+      "logits/chosen": -0.03865582123398781,
+      "logits/rejected": 0.061098456382751465,
+      "logps/chosen": -1.6350253820419312,
+      "logps/rejected": -1.7651439905166626,
+      "loss": 0.302,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.6350253820419312,
+      "rewards/margins": 0.13011865317821503,
+      "rewards/rejected": -1.7651439905166626,
+      "sft_loss": 1.5002126693725586,
+      "step": 15
+    },
+    {
+      "epoch": 0.010704131125606288,
+      "grad_norm": 3.875440232060317,
+      "learning_rate": 3.5650623885918e-08,
+      "logits/chosen": -0.04312217980623245,
+      "logits/rejected": 0.044587552547454834,
+      "logps/chosen": -1.7249486446380615,
+      "logps/rejected": -1.8060178756713867,
+      "loss": 0.2933,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.7249486446380615,
+      "rewards/margins": 0.08106913417577744,
+      "rewards/rejected": -1.8060178756713867,
+      "sft_loss": 1.500407338142395,
+      "step": 20
+    },
+    {
+      "epoch": 0.013380163907007862,
+      "grad_norm": 3.683341007905961,
+      "learning_rate": 4.45632798573975e-08,
+      "logits/chosen": -0.07231198251247406,
+      "logits/rejected": 0.015074786730110645,
+      "logps/chosen": -1.8695526123046875,
+      "logps/rejected": -1.7800153493881226,
+      "loss": 0.322,
+      "rewards/accuracies": 0.3812499940395355,
+      "rewards/chosen": -1.8695526123046875,
+      "rewards/margins": -0.08953739702701569,
+      "rewards/rejected": -1.7800153493881226,
+      "sft_loss": 1.5455690622329712,
+      "step": 25
+    },
+    {
+      "epoch": 0.016056196688409432,
+      "grad_norm": 2.8411343337921076,
+      "learning_rate": 5.3475935828877e-08,
+      "logits/chosen": -0.0865975096821785,
+      "logits/rejected": 0.009093428030610085,
+      "logps/chosen": -1.9094641208648682,
+      "logps/rejected": -1.8325374126434326,
+      "loss": 0.264,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -1.9094641208648682,
+      "rewards/margins": -0.07692664116621017,
+      "rewards/rejected": -1.8325374126434326,
+      "sft_loss": 1.6473287343978882,
+      "step": 30
+    },
+    {
+      "epoch": 0.018732229469811006,
+      "grad_norm": 3.7871810202252782,
+      "learning_rate": 6.23885918003565e-08,
+      "logits/chosen": -0.04631795734167099,
+      "logits/rejected": 0.11639624834060669,
+      "logps/chosen": -1.8485195636749268,
+      "logps/rejected": -1.9989744424819946,
+      "loss": 0.2781,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.8485195636749268,
+      "rewards/margins": 0.15045490860939026,
+      "rewards/rejected": -1.9989744424819946,
+      "sft_loss": 1.5623446702957153,
+      "step": 35
+    },
+    {
+      "epoch": 0.021408262251212576,
+      "grad_norm": 3.209277493286425,
+      "learning_rate": 7.1301247771836e-08,
+      "logits/chosen": 0.03658987209200859,
+      "logits/rejected": 0.2132900059223175,
+      "logps/chosen": -1.8844735622406006,
+      "logps/rejected": -1.7460263967514038,
+      "loss": 0.2934,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -1.8844735622406006,
+      "rewards/margins": -0.1384473741054535,
+      "rewards/rejected": -1.7460263967514038,
+      "sft_loss": 1.5194506645202637,
+      "step": 40
+    },
+    {
+      "epoch": 0.02408429503261415,
+      "grad_norm": 3.7608064055241694,
+      "learning_rate": 8.021390374331551e-08,
+      "logits/chosen": 0.019204024225473404,
+      "logits/rejected": 0.21857735514640808,
+      "logps/chosen": -1.8422836065292358,
+      "logps/rejected": -1.8763787746429443,
+      "loss": 0.285,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.8422836065292358,
+      "rewards/margins": 0.034095339477062225,
+      "rewards/rejected": -1.8763787746429443,
+      "sft_loss": 1.538379430770874,
+      "step": 45
+    },
+    {
+      "epoch": 0.026760327814015723,
+      "grad_norm": 3.4310065887447876,
+      "learning_rate": 8.9126559714795e-08,
+      "logits/chosen": -0.05229135602712631,
+      "logits/rejected": 0.09997323900461197,
+      "logps/chosen": -1.908278226852417,
+      "logps/rejected": -1.7853820323944092,
+      "loss": 0.2806,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.908278226852417,
+      "rewards/margins": -0.12289615720510483,
+      "rewards/rejected": -1.7853820323944092,
+      "sft_loss": 1.5872937440872192,
+      "step": 50
+    },
+    {
+      "epoch": 0.029436360595417294,
+      "grad_norm": 3.1486928978797013,
+      "learning_rate": 9.80392156862745e-08,
+      "logits/chosen": -0.11598268896341324,
+      "logits/rejected": 0.10807422548532486,
+      "logps/chosen": -1.846299409866333,
+      "logps/rejected": -1.8794721364974976,
+      "loss": 0.2676,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.846299409866333,
+      "rewards/margins": 0.03317270055413246,
+      "rewards/rejected": -1.8794721364974976,
+      "sft_loss": 1.5888155698776245,
+      "step": 55
+    },
+    {
+      "epoch": 0.032112393376818864,
+      "grad_norm": 3.225575628852993,
+      "learning_rate": 1.06951871657754e-07,
+      "logits/chosen": -0.08910714834928513,
+      "logits/rejected": 0.10314790904521942,
+      "logps/chosen": -1.8055875301361084,
+      "logps/rejected": -1.9120228290557861,
+      "loss": 0.2595,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.8055875301361084,
+      "rewards/margins": 0.1064353734254837,
+      "rewards/rejected": -1.9120228290557861,
+      "sft_loss": 1.5487940311431885,
+      "step": 60
+    },
+    {
+      "epoch": 0.03478842615822044,
+      "grad_norm": 3.0501140875467243,
+      "learning_rate": 1.158645276292335e-07,
+      "logits/chosen": -0.02393939718604088,
+      "logits/rejected": 0.128209188580513,
+      "logps/chosen": -1.6527059078216553,
+      "logps/rejected": -1.7865177392959595,
+      "loss": 0.2825,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.6527059078216553,
+      "rewards/margins": 0.13381178677082062,
+      "rewards/rejected": -1.7865177392959595,
+      "sft_loss": 1.482107400894165,
+      "step": 65
+    },
+    {
+      "epoch": 0.03746445893962201,
+      "grad_norm": 4.556150358888877,
+      "learning_rate": 1.24777183600713e-07,
+      "logits/chosen": -0.06528159230947495,
+      "logits/rejected": 0.09094108641147614,
+      "logps/chosen": -1.7894785404205322,
+      "logps/rejected": -1.8382999897003174,
+      "loss": 0.2873,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -1.7894785404205322,
+      "rewards/margins": 0.04882138594985008,
+      "rewards/rejected": -1.8382999897003174,
+      "sft_loss": 1.6435940265655518,
+      "step": 70
+    },
+    {
+      "epoch": 0.04014049172102358,
+      "grad_norm": 2.8190971503848,
+      "learning_rate": 1.3368983957219251e-07,
+      "logits/chosen": -0.0369485542178154,
+      "logits/rejected": 0.15109845995903015,
+      "logps/chosen": -1.8229849338531494,
+      "logps/rejected": -2.091214179992676,
+      "loss": 0.2536,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.8229849338531494,
+      "rewards/margins": 0.268229216337204,
+      "rewards/rejected": -2.091214179992676,
+      "sft_loss": 1.584989309310913,
+      "step": 75
+    },
+    {
+      "epoch": 0.04281652450242515,
+      "grad_norm": 2.640922353276211,
+      "learning_rate": 1.42602495543672e-07,
+      "logits/chosen": -0.008784117177128792,
+      "logits/rejected": 0.0963423103094101,
+      "logps/chosen": -1.772080421447754,
+      "logps/rejected": -1.804764986038208,
+      "loss": 0.2817,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.772080421447754,
+      "rewards/margins": 0.0326843187212944,
+      "rewards/rejected": -1.804764986038208,
+      "sft_loss": 1.5509886741638184,
+      "step": 80
+    },
+    {
+      "epoch": 0.04549255728382673,
+      "grad_norm": 2.71350748289859,
+      "learning_rate": 1.5151515151515152e-07,
+      "logits/chosen": -0.1522648185491562,
+      "logits/rejected": 0.09689263254404068,
+      "logps/chosen": -1.8555856943130493,
+      "logps/rejected": -2.046074867248535,
+      "loss": 0.2782,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.8555856943130493,
+      "rewards/margins": 0.1904892474412918,
+      "rewards/rejected": -2.046074867248535,
+      "sft_loss": 1.5202927589416504,
+      "step": 85
+    },
+    {
+      "epoch": 0.0481685900652283,
+      "grad_norm": 2.422361599261421,
+      "learning_rate": 1.6042780748663102e-07,
+      "logits/chosen": 0.09719739854335785,
+      "logits/rejected": 0.05986147001385689,
+      "logps/chosen": -1.8296709060668945,
+      "logps/rejected": -1.8398548364639282,
+      "loss": 0.2889,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -1.8296709060668945,
+      "rewards/margins": 0.010183680802583694,
+      "rewards/rejected": -1.8398548364639282,
+      "sft_loss": 1.4819883108139038,
+      "step": 90
+    },
+    {
+      "epoch": 0.05084462284662987,
+      "grad_norm": 2.418905862195348,
+      "learning_rate": 1.693404634581105e-07,
+      "logits/chosen": -0.07108329236507416,
+      "logits/rejected": 0.0822470635175705,
+      "logps/chosen": -1.9167404174804688,
+      "logps/rejected": -1.9951963424682617,
+      "loss": 0.2626,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.9167404174804688,
+      "rewards/margins": 0.07845588028430939,
+      "rewards/rejected": -1.9951963424682617,
+      "sft_loss": 1.5648051500320435,
+      "step": 95
+    },
+    {
+      "epoch": 0.05352065562803145,
+      "grad_norm": 2.3559369369237277,
+      "learning_rate": 1.7825311942959e-07,
+      "logits/chosen": -0.03189660981297493,
+      "logits/rejected": 0.03379274904727936,
+      "logps/chosen": -1.7909519672393799,
+      "logps/rejected": -1.903272032737732,
+      "loss": 0.263,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.7909519672393799,
+      "rewards/margins": 0.1123199313879013,
+      "rewards/rejected": -1.903272032737732,
+      "sft_loss": 1.5315051078796387,
+      "step": 100
+    },
+    {
+      "epoch": 0.05619668840943302,
+      "grad_norm": 2.272132842693275,
+      "learning_rate": 1.8716577540106952e-07,
+      "logits/chosen": 0.038053132593631744,
+      "logits/rejected": 0.06628037244081497,
+      "logps/chosen": -1.7722151279449463,
+      "logps/rejected": -1.9358274936676025,
+      "loss": 0.2647,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.7722151279449463,
+      "rewards/margins": 0.16361233592033386,
+      "rewards/rejected": -1.9358274936676025,
+      "sft_loss": 1.4914172887802124,
+      "step": 105
+    },
+    {
+      "epoch": 0.05887272119083459,
+      "grad_norm": 2.3939211561963982,
+      "learning_rate": 1.96078431372549e-07,
+      "logits/chosen": 0.019141273573040962,
+      "logits/rejected": 0.1181107759475708,
+      "logps/chosen": -1.8616676330566406,
+      "logps/rejected": -1.9208825826644897,
+      "loss": 0.2764,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.8616676330566406,
+      "rewards/margins": 0.05921504646539688,
+      "rewards/rejected": -1.9208825826644897,
+      "sft_loss": 1.5512902736663818,
+      "step": 110
+    },
+    {
+      "epoch": 0.06154875397223616,
+      "grad_norm": 2.3627131217942043,
+      "learning_rate": 2.049910873440285e-07,
+      "logits/chosen": 0.07100260257720947,
+      "logits/rejected": 0.29280149936676025,
+      "logps/chosen": -1.8520358800888062,
+      "logps/rejected": -2.1854333877563477,
+      "loss": 0.2304,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.8520358800888062,
+      "rewards/margins": 0.33339765667915344,
+      "rewards/rejected": -2.1854333877563477,
+      "sft_loss": 1.6704508066177368,
+      "step": 115
+    },
+    {
+      "epoch": 0.06422478675363773,
+      "grad_norm": 1.735165157614565,
+      "learning_rate": 2.13903743315508e-07,
+      "logits/chosen": -0.07888107746839523,
+      "logits/rejected": 0.10008995234966278,
+      "logps/chosen": -1.9981311559677124,
+      "logps/rejected": -2.1442179679870605,
+      "loss": 0.2481,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.9981311559677124,
+      "rewards/margins": 0.14608684182167053,
+      "rewards/rejected": -2.1442179679870605,
+      "sft_loss": 1.6943069696426392,
+      "step": 120
+    },
+    {
+      "epoch": 0.0669008195350393,
+      "grad_norm": 2.786593260731286,
+      "learning_rate": 2.2281639928698751e-07,
+      "logits/chosen": -0.05085619166493416,
+      "logits/rejected": 0.08655449748039246,
+      "logps/chosen": -1.8938229084014893,
+      "logps/rejected": -1.8082813024520874,
+      "loss": 0.281,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -1.8938229084014893,
+      "rewards/margins": -0.08554168045520782,
+      "rewards/rejected": -1.8082813024520874,
+      "sft_loss": 1.606369972229004,
+      "step": 125
+    },
+    {
+      "epoch": 0.06957685231644088,
+      "grad_norm": 2.5772159985228655,
+      "learning_rate": 2.31729055258467e-07,
+      "logits/chosen": 0.04295843839645386,
+      "logits/rejected": 0.18473029136657715,
+      "logps/chosen": -1.9629977941513062,
+      "logps/rejected": -2.086339235305786,
+      "loss": 0.2481,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.9629977941513062,
+      "rewards/margins": 0.12334122508764267,
+      "rewards/rejected": -2.086339235305786,
+      "sft_loss": 1.6907141208648682,
+      "step": 130
+    },
+    {
+      "epoch": 0.07225288509784245,
+      "grad_norm": 1.9943010581072478,
+      "learning_rate": 2.406417112299465e-07,
+      "logits/chosen": -0.0350356251001358,
+      "logits/rejected": 0.08792857825756073,
+      "logps/chosen": -2.0462894439697266,
+      "logps/rejected": -2.0182530879974365,
+      "loss": 0.2578,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -2.0462894439697266,
+      "rewards/margins": -0.02803630754351616,
+      "rewards/rejected": -2.0182530879974365,
+      "sft_loss": 1.6577552556991577,
+      "step": 135
+    },
+    {
+      "epoch": 0.07492891787924402,
+      "grad_norm": 2.8486191540947807,
+      "learning_rate": 2.49554367201426e-07,
+      "logits/chosen": -0.02075616642832756,
+      "logits/rejected": 0.1543327271938324,
+      "logps/chosen": -2.015401601791382,
+      "logps/rejected": -2.2778303623199463,
+      "loss": 0.2269,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -2.015401601791382,
+      "rewards/margins": 0.2624287009239197,
+      "rewards/rejected": -2.2778303623199463,
+      "sft_loss": 1.6974109411239624,
+      "step": 140
+    },
+    {
+      "epoch": 0.0776049506606456,
+      "grad_norm": 2.1225048898899246,
+      "learning_rate": 2.5846702317290554e-07,
+      "logits/chosen": 0.0070198155008256435,
+      "logits/rejected": 0.17295916378498077,
+      "logps/chosen": -1.9902257919311523,
+      "logps/rejected": -2.140160083770752,
+      "loss": 0.2439,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -1.9902257919311523,
+      "rewards/margins": 0.1499340534210205,
+      "rewards/rejected": -2.140160083770752,
+      "sft_loss": 1.631656289100647,
+      "step": 145
+    },
+    {
+      "epoch": 0.08028098344204716,
+      "grad_norm": 2.338788349764335,
+      "learning_rate": 2.6737967914438503e-07,
+      "logits/chosen": -0.03971818834543228,
+      "logits/rejected": 0.13490387797355652,
+      "logps/chosen": -1.9857639074325562,
+      "logps/rejected": -1.9886020421981812,
+      "loss": 0.2651,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.9857639074325562,
+      "rewards/margins": 0.002838182495906949,
+      "rewards/rejected": -1.9886020421981812,
+      "sft_loss": 1.5006240606307983,
+      "step": 150
+    },
+    {
+      "epoch": 0.08295701622344874,
+      "grad_norm": 2.1835505399539104,
+      "learning_rate": 2.762923351158645e-07,
+      "logits/chosen": -0.027233857661485672,
+      "logits/rejected": 0.025476187467575073,
+      "logps/chosen": -2.1108012199401855,
+      "logps/rejected": -2.1566002368927,
+      "loss": 0.2523,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -2.1108012199401855,
+      "rewards/margins": 0.04579881578683853,
+      "rewards/rejected": -2.1566002368927,
+      "sft_loss": 1.6765267848968506,
+      "step": 155
+    },
+    {
+      "epoch": 0.0856330490048503,
+      "grad_norm": 2.040498416917074,
+      "learning_rate": 2.85204991087344e-07,
+      "logits/chosen": -0.11586171388626099,
+      "logits/rejected": 0.03955882042646408,
+      "logps/chosen": -2.320681095123291,
+      "logps/rejected": -2.297348976135254,
+      "loss": 0.2415,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -2.320681095123291,
+      "rewards/margins": -0.023332182317972183,
+      "rewards/rejected": -2.297348976135254,
+      "sft_loss": 1.771780014038086,
+      "step": 160
+    },
+    {
+      "epoch": 0.08830908178625188,
+      "grad_norm": 1.9928740456508027,
+      "learning_rate": 2.941176470588235e-07,
+      "logits/chosen": -0.04006613790988922,
+      "logits/rejected": 0.14414557814598083,
+      "logps/chosen": -2.0469436645507812,
+      "logps/rejected": -2.3669915199279785,
+      "loss": 0.2436,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -2.0469436645507812,
+      "rewards/margins": 0.3200477659702301,
+      "rewards/rejected": -2.3669915199279785,
+      "sft_loss": 1.6215006113052368,
+      "step": 165
+    },
+    {
+      "epoch": 0.09098511456765346,
+      "grad_norm": 1.9534296167766696,
+      "learning_rate": 3.0303030303030305e-07,
+      "logits/chosen": -0.06098126247525215,
+      "logits/rejected": 0.001214376068674028,
+      "logps/chosen": -2.3339781761169434,
+      "logps/rejected": -2.305957555770874,
+      "loss": 0.2285,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -2.3339781761169434,
+      "rewards/margins": -0.028020773082971573,
+      "rewards/rejected": -2.305957555770874,
+      "sft_loss": 1.726961374282837,
+      "step": 170
+    },
+    {
+      "epoch": 0.09366114734905502,
+      "grad_norm": 1.9588967180799672,
+      "learning_rate": 3.1194295900178254e-07,
+      "logits/chosen": 0.09518565982580185,
+      "logits/rejected": 0.09727490693330765,
+      "logps/chosen": -2.214961290359497,
+      "logps/rejected": -2.278738498687744,
+      "loss": 0.2654,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -2.214961290359497,
+      "rewards/margins": 0.06377717852592468,
+      "rewards/rejected": -2.278738498687744,
+      "sft_loss": 1.7609399557113647,
+      "step": 175
+    },
+    {
+      "epoch": 0.0963371801304566,
+      "grad_norm": 1.6589936848533324,
+      "learning_rate": 3.2085561497326203e-07,
+      "logits/chosen": 0.03179093450307846,
+      "logits/rejected": 0.037368230521678925,
+      "logps/chosen": -2.3154728412628174,
+      "logps/rejected": -2.2699761390686035,
+      "loss": 0.236,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -2.3154728412628174,
+      "rewards/margins": -0.04549693316221237,
+      "rewards/rejected": -2.2699761390686035,
+      "sft_loss": 1.7420244216918945,
+      "step": 180
+    },
+    {
+      "epoch": 0.09901321291185818,
+      "grad_norm": 1.9967555737580138,
+      "learning_rate": 3.297682709447415e-07,
+      "logits/chosen": -0.12511086463928223,
+      "logits/rejected": -0.02812013030052185,
+      "logps/chosen": -2.2482380867004395,
+      "logps/rejected": -2.3399927616119385,
+      "loss": 0.2525,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -2.2482380867004395,
+      "rewards/margins": 0.09175457060337067,
+      "rewards/rejected": -2.3399927616119385,
+      "sft_loss": 1.711207389831543,
+      "step": 185
+    },
+    {
+      "epoch": 0.10168924569325974,
+      "grad_norm": 2.678787481783515,
+      "learning_rate": 3.38680926916221e-07,
+      "logits/chosen": -0.03826170042157173,
+      "logits/rejected": 0.10330984741449356,
+      "logps/chosen": -2.804680347442627,
+      "logps/rejected": -2.641824245452881,
+      "loss": 0.2148,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -2.804680347442627,
+      "rewards/margins": -0.162856325507164,
+      "rewards/rejected": -2.641824245452881,
+      "sft_loss": 1.9906642436981201,
+      "step": 190
+    },
+    {
+      "epoch": 0.10436527847466132,
+      "grad_norm": 2.0306729180588396,
+      "learning_rate": 3.475935828877005e-07,
+      "logits/chosen": 0.04664776846766472,
+      "logits/rejected": 0.22064730525016785,
+      "logps/chosen": -2.1520609855651855,
+      "logps/rejected": -2.210716724395752,
+      "loss": 0.2296,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -2.1520609855651855,
+      "rewards/margins": 0.058656178414821625,
+      "rewards/rejected": -2.210716724395752,
+      "sft_loss": 1.615627646446228,
+      "step": 195
+    },
+    {
+      "epoch": 0.1070413112560629,
+      "grad_norm": 3.1931495872144007,
+      "learning_rate": 3.5650623885918e-07,
+      "logits/chosen": -0.03578418493270874,
+      "logits/rejected": 0.12399481236934662,
+      "logps/chosen": -2.6481387615203857,
+      "logps/rejected": -2.3530163764953613,
+      "loss": 0.2331,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -2.6481387615203857,
+      "rewards/margins": -0.2951226830482483,
+      "rewards/rejected": -2.3530163764953613,
+      "sft_loss": 1.9002326726913452,
+      "step": 200
+    },
+    {
+      "epoch": 0.10971734403746446,
+      "grad_norm": 2.640144568717103,
+      "learning_rate": 3.654188948306595e-07,
+      "logits/chosen": -0.021364711225032806,
+      "logits/rejected": 0.15079109370708466,
+      "logps/chosen": -3.106703281402588,
+      "logps/rejected": -2.7240376472473145,
+      "loss": 0.2138,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -3.106703281402588,
+      "rewards/margins": -0.3826655149459839,
+      "rewards/rejected": -2.7240376472473145,
+      "sft_loss": 1.900535225868225,
+      "step": 205
+    },
+    {
+      "epoch": 0.11239337681886603,
+      "grad_norm": 2.186333179804364,
+      "learning_rate": 3.7433155080213904e-07,
+      "logits/chosen": -0.11891081184148788,
+      "logits/rejected": 0.10270519554615021,
+      "logps/chosen": -2.9581754207611084,
+      "logps/rejected": -3.388826847076416,
+      "loss": 0.1741,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -2.9581754207611084,
+      "rewards/margins": 0.4306512773036957,
+      "rewards/rejected": -3.388826847076416,
+      "sft_loss": 2.0287399291992188,
+      "step": 210
+    },
+    {
+      "epoch": 0.1150694096002676,
+      "grad_norm": 2.0176626763649,
+      "learning_rate": 3.8324420677361853e-07,
+      "logits/chosen": -0.13966991007328033,
+      "logits/rejected": 0.140414297580719,
+      "logps/chosen": -2.749617099761963,
+      "logps/rejected": -2.8892064094543457,
+      "loss": 0.1857,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -2.749617099761963,
+      "rewards/margins": 0.13958922028541565,
+      "rewards/rejected": -2.8892064094543457,
+      "sft_loss": 2.041731357574463,
+      "step": 215
+    },
+    {
+      "epoch": 0.11774544238166917,
+      "grad_norm": 1.998903196870583,
+      "learning_rate": 3.92156862745098e-07,
+      "logits/chosen": 0.0680420845746994,
+      "logits/rejected": 0.18458662927150726,
+      "logps/chosen": -3.2915711402893066,
+      "logps/rejected": -3.790569305419922,
+      "loss": 0.1668,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -3.2915711402893066,
+      "rewards/margins": 0.4989985525608063,
+      "rewards/rejected": -3.790569305419922,
+      "sft_loss": 2.265310764312744,
+      "step": 220
+    },
+    {
+      "epoch": 0.12042147516307075,
+      "grad_norm": 1.9590568738460437,
+      "learning_rate": 4.010695187165775e-07,
+      "logits/chosen": -0.07913367450237274,
+      "logits/rejected": 0.1182810515165329,
+      "logps/chosen": -3.0919346809387207,
+      "logps/rejected": -3.1701674461364746,
+      "loss": 0.1653,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -3.0919346809387207,
+      "rewards/margins": 0.07823298126459122,
+      "rewards/rejected": -3.1701674461364746,
+      "sft_loss": 1.9699609279632568,
+      "step": 225
+    },
+    {
+      "epoch": 0.12309750794447231,
+      "grad_norm": 1.8737010294999485,
+      "learning_rate": 4.09982174688057e-07,
+      "logits/chosen": 0.037992849946022034,
+      "logits/rejected": 0.13747188448905945,
+      "logps/chosen": -3.8204619884490967,
+      "logps/rejected": -3.797633647918701,
+      "loss": 0.1635,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -3.8204619884490967,
+      "rewards/margins": -0.022828320041298866,
+      "rewards/rejected": -3.797633647918701,
+      "sft_loss": 2.1811490058898926,
+      "step": 230
+    },
+    {
+      "epoch": 0.1257735407258739,
+      "grad_norm": 1.8191581628112006,
+      "learning_rate": 4.188948306595365e-07,
+      "logits/chosen": 0.046239614486694336,
+      "logits/rejected": 0.22374077141284943,
+      "logps/chosen": -3.5340847969055176,
+      "logps/rejected": -3.8707435131073,
+      "loss": 0.1426,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -3.5340847969055176,
+      "rewards/margins": 0.3366585373878479,
+      "rewards/rejected": -3.8707435131073,
+      "sft_loss": 2.135493516921997,
+      "step": 235
+    },
+    {
+      "epoch": 0.12844957350727546,
+      "grad_norm": 2.15850506003128,
+      "learning_rate": 4.27807486631016e-07,
+      "logits/chosen": -0.020359747111797333,
+      "logits/rejected": 0.12103061378002167,
+      "logps/chosen": -3.8267650604248047,
+      "logps/rejected": -3.854151487350464,
+      "loss": 0.1603,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -3.8267650604248047,
+      "rewards/margins": 0.02738667093217373,
+      "rewards/rejected": -3.854151487350464,
+      "sft_loss": 2.468606472015381,
+      "step": 240
+    },
+    {
+      "epoch": 0.13112560628867703,
+      "grad_norm": 1.9869901093405427,
+      "learning_rate": 4.3672014260249554e-07,
+      "logits/chosen": 0.09430526196956635,
+      "logits/rejected": 0.2459847629070282,
+      "logps/chosen": -3.5051181316375732,
+      "logps/rejected": -4.297321319580078,
+      "loss": 0.1457,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -3.5051181316375732,
+      "rewards/margins": 0.792202889919281,
+      "rewards/rejected": -4.297321319580078,
+      "sft_loss": 2.459380626678467,
+      "step": 245
+    },
+    {
+      "epoch": 0.1338016390700786,
+      "grad_norm": 1.9104173001829199,
+      "learning_rate": 4.4563279857397503e-07,
+      "logits/chosen": 0.02481432631611824,
+      "logits/rejected": 0.22242239117622375,
+      "logps/chosen": -5.418574333190918,
+      "logps/rejected": -5.279926300048828,
+      "loss": 0.1462,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -5.418574333190918,
+      "rewards/margins": -0.13864776492118835,
+      "rewards/rejected": -5.279926300048828,
+      "sft_loss": 2.6364169120788574,
+      "step": 250
+    },
+    {
+      "epoch": 0.1364776718514802,
+      "grad_norm": 1.2454643359237576,
+      "learning_rate": 4.545454545454545e-07,
+      "logits/chosen": 0.06115083023905754,
+      "logits/rejected": 0.2540772557258606,
+      "logps/chosen": -3.8281192779541016,
+      "logps/rejected": -4.610691547393799,
+      "loss": 0.1384,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -3.8281192779541016,
+      "rewards/margins": 0.7825719714164734,
+      "rewards/rejected": -4.610691547393799,
+      "sft_loss": 2.2860593795776367,
+      "step": 255
+    },
+    {
+      "epoch": 0.13915370463288176,
+      "grad_norm": 0.9849368225532316,
+      "learning_rate": 4.63458110516934e-07,
+      "logits/chosen": -0.10867585986852646,
+      "logits/rejected": 0.02872631885111332,
+      "logps/chosen": -5.133664131164551,
+      "logps/rejected": -4.367121696472168,
+      "loss": 0.1178,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -5.133664131164551,
+      "rewards/margins": -0.7665426135063171,
+      "rewards/rejected": -4.367121696472168,
+      "sft_loss": 2.7160372734069824,
+      "step": 260
+    },
+    {
+      "epoch": 0.1418297374142833,
+      "grad_norm": 0.8961988680228047,
+      "learning_rate": 4.723707664884135e-07,
+      "logits/chosen": 0.07030217349529266,
+      "logits/rejected": 0.17124192416667938,
+      "logps/chosen": -6.0838727951049805,
+      "logps/rejected": -4.811110019683838,
+      "loss": 0.1367,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -6.0838727951049805,
+      "rewards/margins": -1.2727632522583008,
+      "rewards/rejected": -4.811110019683838,
+      "sft_loss": 3.739053726196289,
+      "step": 265
+    },
+    {
+      "epoch": 0.1445057701956849,
+      "grad_norm": 0.8941719839014858,
+      "learning_rate": 4.81283422459893e-07,
+      "logits/chosen": 0.017831971868872643,
+      "logits/rejected": 0.20090460777282715,
+      "logps/chosen": -4.764120578765869,
+      "logps/rejected": -5.134100437164307,
+      "loss": 0.1264,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -4.764120578765869,
+      "rewards/margins": 0.3699795603752136,
+      "rewards/rejected": -5.134100437164307,
+      "sft_loss": 2.7194724082946777,
+      "step": 270
+    },
+    {
+      "epoch": 0.14718180297708647,
+      "grad_norm": 1.222685081948667,
+      "learning_rate": 4.901960784313725e-07,
+      "logits/chosen": 0.16087068617343903,
+      "logits/rejected": 0.2754586338996887,
+      "logps/chosen": -5.333785057067871,
+      "logps/rejected": -6.004696846008301,
+      "loss": 0.1436,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -5.333785057067871,
+      "rewards/margins": 0.6709117293357849,
+      "rewards/rejected": -6.004696846008301,
+      "sft_loss": 3.1917669773101807,
+      "step": 275
+    },
+    {
+      "epoch": 0.14985783575848804,
+      "grad_norm": 0.7370626970557951,
+      "learning_rate": 4.99108734402852e-07,
+      "logits/chosen": 0.007818743586540222,
+      "logits/rejected": 0.22343873977661133,
+      "logps/chosen": -6.119328498840332,
+      "logps/rejected": -5.818517208099365,
+      "loss": 0.1211,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -6.119328498840332,
+      "rewards/margins": -0.3008107542991638,
+      "rewards/rejected": -5.818517208099365,
+      "sft_loss": 3.9426021575927734,
+      "step": 280
+    },
+    {
+      "epoch": 0.15253386853988962,
+      "grad_norm": 1.233957256562051,
+      "learning_rate": 5.080213903743315e-07,
+      "logits/chosen": 0.041990119963884354,
+      "logits/rejected": 0.22771665453910828,
+      "logps/chosen": -5.575260162353516,
+      "logps/rejected": -5.49068546295166,
+      "loss": 0.1275,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -5.575260162353516,
+      "rewards/margins": -0.08457436412572861,
+      "rewards/rejected": -5.49068546295166,
+      "sft_loss": 2.8716347217559814,
+      "step": 285
+    },
+    {
+      "epoch": 0.1552099013212912,
+      "grad_norm": 0.6072672871705348,
+      "learning_rate": 5.169340463458111e-07,
+      "logits/chosen": -0.026344675570726395,
+      "logits/rejected": 0.3459742069244385,
+      "logps/chosen": -4.510623931884766,
+      "logps/rejected": -5.7433624267578125,
+      "loss": 0.0906,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -4.510623931884766,
+      "rewards/margins": 1.2327378988265991,
+      "rewards/rejected": -5.7433624267578125,
+      "sft_loss": 2.9556336402893066,
+      "step": 290
+    },
+    {
+      "epoch": 0.15788593410269275,
+      "grad_norm": 0.7208828481537657,
+      "learning_rate": 5.258467023172905e-07,
+      "logits/chosen": 0.09221816062927246,
+      "logits/rejected": 0.17328932881355286,
+      "logps/chosen": -6.130520820617676,
+      "logps/rejected": -5.456192970275879,
+      "loss": 0.1207,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -6.130520820617676,
+      "rewards/margins": -0.6743276715278625,
+      "rewards/rejected": -5.456192970275879,
+      "sft_loss": 3.7058982849121094,
+      "step": 295
+    },
+    {
+      "epoch": 0.16056196688409433,
+      "grad_norm": 1.340847333736133,
+      "learning_rate": 5.347593582887701e-07,
+      "logits/chosen": 0.009460541419684887,
+      "logits/rejected": 0.23661403357982635,
+      "logps/chosen": -6.522822380065918,
+      "logps/rejected": -6.587447166442871,
+      "loss": 0.1156,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -6.522822380065918,
+      "rewards/margins": 0.06462571769952774,
+      "rewards/rejected": -6.587447166442871,
+      "sft_loss": 3.30534029006958,
+      "step": 300
+    },
+    {
+      "epoch": 0.1632379996654959,
+      "grad_norm": 1.9457407077227884,
+      "learning_rate": 5.436720142602496e-07,
+      "logits/chosen": 0.06960954517126083,
+      "logits/rejected": 0.16367551684379578,
+      "logps/chosen": -5.572967052459717,
+      "logps/rejected": -5.5532331466674805,
+      "loss": 0.1133,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -5.572967052459717,
+      "rewards/margins": -0.01973416842520237,
+      "rewards/rejected": -5.5532331466674805,
+      "sft_loss": 3.6321425437927246,
+      "step": 305
+    },
+    {
+      "epoch": 0.16591403244689748,
+      "grad_norm": 6.893463954081525,
+      "learning_rate": 5.52584670231729e-07,
+      "logits/chosen": -0.13142789900302887,
+      "logits/rejected": 0.0010722450679168105,
+      "logps/chosen": -6.735803127288818,
+      "logps/rejected": -6.812127590179443,
+      "loss": 0.0959,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -6.735803127288818,
+      "rewards/margins": 0.07632424682378769,
+      "rewards/rejected": -6.812127590179443,
+      "sft_loss": 4.3729987144470215,
+      "step": 310
+    },
+    {
+      "epoch": 0.16859006522829906,
+      "grad_norm": 1.2119659575882744,
+      "learning_rate": 5.614973262032086e-07,
+      "logits/chosen": 0.05507947877049446,
+      "logits/rejected": 0.24547457695007324,
+      "logps/chosen": -7.327678680419922,
+      "logps/rejected": -7.448760986328125,
+      "loss": 0.0735,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -7.327678680419922,
+      "rewards/margins": 0.12108228355646133,
+      "rewards/rejected": -7.448760986328125,
+      "sft_loss": 5.221084117889404,
+      "step": 315
+    },
+    {
+      "epoch": 0.1712660980097006,
+      "grad_norm": 1.421852572662818,
+      "learning_rate": 5.70409982174688e-07,
+      "logits/chosen": -0.0505533441901207,
+      "logits/rejected": 0.09875977784395218,
+      "logps/chosen": -7.0575737953186035,
+      "logps/rejected": -6.7295331954956055,
+      "loss": 0.0729,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -7.0575737953186035,
+      "rewards/margins": -0.32803958654403687,
+      "rewards/rejected": -6.7295331954956055,
+      "sft_loss": 5.3815484046936035,
+      "step": 320
+    },
+    {
+      "epoch": 0.17394213079110218,
+      "grad_norm": 2.1768410941997747,
+      "learning_rate": 5.793226381461676e-07,
+      "logits/chosen": -0.06135901063680649,
+      "logits/rejected": 0.12024674564599991,
+      "logps/chosen": -8.038436889648438,
+      "logps/rejected": -8.41388988494873,
+      "loss": 0.0679,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -8.038436889648438,
+      "rewards/margins": 0.3754529356956482,
+      "rewards/rejected": -8.41388988494873,
+      "sft_loss": 5.565326690673828,
+      "step": 325
+    },
+    {
+      "epoch": 0.17661816357250376,
+      "grad_norm": 0.9252033224489392,
+      "learning_rate": 5.88235294117647e-07,
+      "logits/chosen": 0.034027762711048126,
+      "logits/rejected": 0.23385238647460938,
+      "logps/chosen": -6.080555438995361,
+      "logps/rejected": -7.770529270172119,
+      "loss": 0.0713,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -6.080555438995361,
+      "rewards/margins": 1.6899728775024414,
+      "rewards/rejected": -7.770529270172119,
+      "sft_loss": 5.016576290130615,
+      "step": 330
+    },
+    {
+      "epoch": 0.17929419635390534,
+      "grad_norm": 2.0989591544928925,
+      "learning_rate": 5.971479500891266e-07,
+      "logits/chosen": 0.01920177973806858,
+      "logits/rejected": 0.206724613904953,
+      "logps/chosen": -7.403994560241699,
+      "logps/rejected": -7.278214931488037,
+      "loss": 0.0695,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -7.403994560241699,
+      "rewards/margins": -0.12578034400939941,
+      "rewards/rejected": -7.278214931488037,
+      "sft_loss": 4.916281223297119,
+      "step": 335
+    },
+    {
+      "epoch": 0.18197022913530692,
+      "grad_norm": 1.893897803965405,
+      "learning_rate": 6.060606060606061e-07,
+      "logits/chosen": 0.055364273488521576,
+      "logits/rejected": 0.2726927697658539,
+      "logps/chosen": -6.973310947418213,
+      "logps/rejected": -7.325900077819824,
+      "loss": 0.0663,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -6.973310947418213,
+      "rewards/margins": 0.35258910059928894,
+      "rewards/rejected": -7.325900077819824,
+      "sft_loss": 5.119781494140625,
+      "step": 340
+    },
+    {
+      "epoch": 0.1846462619167085,
+      "grad_norm": 2.836372712749154,
+      "learning_rate": 6.149732620320855e-07,
+      "logits/chosen": 0.06766609847545624,
+      "logits/rejected": 0.1415514200925827,
+      "logps/chosen": -6.7723259925842285,
+      "logps/rejected": -6.848855495452881,
+      "loss": 0.0667,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -6.7723259925842285,
+      "rewards/margins": 0.07653047144412994,
+      "rewards/rejected": -6.848855495452881,
+      "sft_loss": 5.026312828063965,
+      "step": 345
+    },
+    {
+      "epoch": 0.18732229469811004,
+      "grad_norm": 2.265144184050898,
+      "learning_rate": 6.238859180035651e-07,
+      "logits/chosen": -0.15822748839855194,
+      "logits/rejected": 0.0024463594891130924,
+      "logps/chosen": -6.168185234069824,
+      "logps/rejected": -5.750182151794434,
+      "loss": 0.0671,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -6.168185234069824,
+      "rewards/margins": -0.4180033802986145,
+      "rewards/rejected": -5.750182151794434,
+      "sft_loss": 4.6768975257873535,
+      "step": 350
+    },
+    {
+      "epoch": 0.18999832747951162,
+      "grad_norm": 2.614388773693835,
+      "learning_rate": 6.327985739750445e-07,
+      "logits/chosen": -0.14725571870803833,
+      "logits/rejected": 0.13961216807365417,
+      "logps/chosen": -6.235496520996094,
+      "logps/rejected": -6.224798202514648,
+      "loss": 0.0614,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -6.235496520996094,
+      "rewards/margins": -0.01069814246147871,
+      "rewards/rejected": -6.224798202514648,
+      "sft_loss": 5.157872200012207,
+      "step": 355
+    },
+    {
+      "epoch": 0.1926743602609132,
+      "grad_norm": 2.7692613642785884,
+      "learning_rate": 6.417112299465241e-07,
+      "logits/chosen": -0.1557673215866089,
+      "logits/rejected": -0.048311877995729446,
+      "logps/chosen": -5.508719444274902,
+      "logps/rejected": -5.668200492858887,
+      "loss": 0.061,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -5.508719444274902,
+      "rewards/margins": 0.15948060154914856,
+      "rewards/rejected": -5.668200492858887,
+      "sft_loss": 4.47888708114624,
+      "step": 360
+    },
+    {
+      "epoch": 0.19535039304231477,
+      "grad_norm": 1.759183414717174,
+      "learning_rate": 6.506238859180035e-07,
+      "logits/chosen": -0.1632741391658783,
+      "logits/rejected": -0.04335709661245346,
+      "logps/chosen": -5.9066925048828125,
+      "logps/rejected": -5.570250988006592,
+      "loss": 0.0622,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -5.9066925048828125,
+      "rewards/margins": -0.33644169569015503,
+      "rewards/rejected": -5.570250988006592,
+      "sft_loss": 5.030333518981934,
+      "step": 365
+    },
+    {
+      "epoch": 0.19802642582371635,
+      "grad_norm": 2.2296092123630236,
+      "learning_rate": 6.59536541889483e-07,
+      "logits/chosen": -0.3708917498588562,
+      "logits/rejected": -0.20164895057678223,
+      "logps/chosen": -5.268651485443115,
+      "logps/rejected": -5.234259128570557,
+      "loss": 0.062,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -5.268651485443115,
+      "rewards/margins": -0.03439173102378845,
+      "rewards/rejected": -5.234259128570557,
+      "sft_loss": 4.815543174743652,
+      "step": 370
+    },
+    {
+      "epoch": 0.2007024586051179,
+      "grad_norm": 2.0146802870823928,
+      "learning_rate": 6.684491978609626e-07,
+      "logits/chosen": -0.47356781363487244,
+      "logits/rejected": -0.21013717353343964,
+      "logps/chosen": -5.339799880981445,
+      "logps/rejected": -5.731418609619141,
+      "loss": 0.0567,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -5.339799880981445,
+      "rewards/margins": 0.39161843061447144,
+      "rewards/rejected": -5.731418609619141,
+      "sft_loss": 4.969027996063232,
+      "step": 375
+    },
+    {
+      "epoch": 0.20337849138651948,
+      "grad_norm": 1.3381881149332266,
+      "learning_rate": 6.77361853832442e-07,
+      "logits/chosen": -0.5766822695732117,
+      "logits/rejected": -0.4294039309024811,
+      "logps/chosen": -4.788380146026611,
+      "logps/rejected": -4.977846622467041,
+      "loss": 0.0566,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.788380146026611,
+      "rewards/margins": 0.18946652114391327,
+      "rewards/rejected": -4.977846622467041,
+      "sft_loss": 4.430079460144043,
+      "step": 380
+    },
+    {
+      "epoch": 0.20605452416792105,
+      "grad_norm": 0.9198991724416198,
+      "learning_rate": 6.862745098039216e-07,
+      "logits/chosen": -0.48545369505882263,
+      "logits/rejected": -0.3366524577140808,
+      "logps/chosen": -5.007147789001465,
+      "logps/rejected": -5.114348888397217,
+      "loss": 0.0569,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -5.007147789001465,
+      "rewards/margins": 0.10720244795084,
+      "rewards/rejected": -5.114348888397217,
+      "sft_loss": 4.7221221923828125,
+      "step": 385
+    },
+    {
+      "epoch": 0.20873055694932263,
+      "grad_norm": 2.7786654323372857,
+      "learning_rate": 6.95187165775401e-07,
+      "logits/chosen": -0.1661207228899002,
+      "logits/rejected": 0.11756277084350586,
+      "logps/chosen": -5.295405387878418,
+      "logps/rejected": -5.372384548187256,
+      "loss": 0.0601,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -5.295405387878418,
+      "rewards/margins": 0.07697971165180206,
+      "rewards/rejected": -5.372384548187256,
+      "sft_loss": 4.778872489929199,
+      "step": 390
+    },
+    {
+      "epoch": 0.2114065897307242,
+      "grad_norm": 1.0788168546978547,
+      "learning_rate": 7.040998217468806e-07,
+      "logits/chosen": -0.4957035183906555,
+      "logits/rejected": -0.25873202085494995,
+      "logps/chosen": -4.942808151245117,
+      "logps/rejected": -5.0668792724609375,
+      "loss": 0.0572,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -4.942808151245117,
+      "rewards/margins": 0.1240713819861412,
+      "rewards/rejected": -5.0668792724609375,
+      "sft_loss": 4.497957706451416,
+      "step": 395
+    },
+    {
+      "epoch": 0.2140826225121258,
+      "grad_norm": 1.2369650710490778,
+      "learning_rate": 7.1301247771836e-07,
+      "logits/chosen": -0.5010001063346863,
+      "logits/rejected": -0.32428526878356934,
+      "logps/chosen": -5.017227649688721,
+      "logps/rejected": -5.143941879272461,
+      "loss": 0.0548,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -5.017227649688721,
+      "rewards/margins": 0.12671446800231934,
+      "rewards/rejected": -5.143941879272461,
+      "sft_loss": 4.438155174255371,
+      "step": 400
+    },
+    {
+      "epoch": 0.2140826225121258,
+      "eval_logits/chosen": -0.02770337089896202,
+      "eval_logits/rejected": 0.10948384553194046,
+      "eval_logps/chosen": -5.346701622009277,
+      "eval_logps/rejected": -5.472263813018799,
+      "eval_loss": 0.05566617101430893,
+      "eval_rewards/accuracies": 0.5326409339904785,
+      "eval_rewards/chosen": -5.346701622009277,
+      "eval_rewards/margins": 0.12556174397468567,
+      "eval_rewards/rejected": -5.472263813018799,
+      "eval_runtime": 43.8831,
+      "eval_samples_per_second": 30.65,
+      "eval_sft_loss": 4.829505443572998,
+      "eval_steps_per_second": 7.679,
+      "step": 400
+    },
+    {
+      "epoch": 0.21675865529352734,
+      "grad_norm": 0.8680376616287312,
+      "learning_rate": 7.219251336898395e-07,
+      "logits/chosen": -0.5568983554840088,
+      "logits/rejected": -0.42599543929100037,
+      "logps/chosen": -5.262287616729736,
+      "logps/rejected": -5.478768825531006,
+      "loss": 0.0577,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -5.262287616729736,
+      "rewards/margins": 0.21648113429546356,
+      "rewards/rejected": -5.478768825531006,
+      "sft_loss": 4.965443134307861,
+      "step": 405
+    },
+    {
+      "epoch": 0.2194346880749289,
+      "grad_norm": 1.3304348425737549,
+      "learning_rate": 7.30837789661319e-07,
+      "logits/chosen": -0.4703024923801422,
+      "logits/rejected": -0.2515341639518738,
+      "logps/chosen": -4.820174694061279,
+      "logps/rejected": -5.074994087219238,
+      "loss": 0.055,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -4.820174694061279,
+      "rewards/margins": 0.25481972098350525,
+      "rewards/rejected": -5.074994087219238,
+      "sft_loss": 4.33463191986084,
+      "step": 410
+    },
+    {
+      "epoch": 0.2221107208563305,
+      "grad_norm": 1.5135280531396909,
+      "learning_rate": 7.397504456327985e-07,
+      "logits/chosen": -0.4607006907463074,
+      "logits/rejected": -0.3690333366394043,
+      "logps/chosen": -5.324876308441162,
+      "logps/rejected": -5.247057914733887,
+      "loss": 0.0574,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -5.324876308441162,
+      "rewards/margins": -0.07781883329153061,
+      "rewards/rejected": -5.247057914733887,
+      "sft_loss": 4.834041595458984,
+      "step": 415
+    },
+    {
+      "epoch": 0.22478675363773207,
+      "grad_norm": 2.6272102535641886,
+      "learning_rate": 7.486631016042781e-07,
+      "logits/chosen": -0.5340300798416138,
+      "logits/rejected": -0.14234408736228943,
+      "logps/chosen": -4.852800369262695,
+      "logps/rejected": -5.007115840911865,
+      "loss": 0.0559,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -4.852800369262695,
+      "rewards/margins": 0.15431593358516693,
+      "rewards/rejected": -5.007115840911865,
+      "sft_loss": 4.486388206481934,
+      "step": 420
+    },
+    {
+      "epoch": 0.22746278641913364,
+      "grad_norm": 1.9315381406431007,
+      "learning_rate": 7.575757575757575e-07,
+      "logits/chosen": -0.43069228529930115,
+      "logits/rejected": -0.1704765111207962,
+      "logps/chosen": -4.909597396850586,
+      "logps/rejected": -5.0253095626831055,
+      "loss": 0.0551,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -4.909597396850586,
+      "rewards/margins": 0.11571161448955536,
+      "rewards/rejected": -5.0253095626831055,
+      "sft_loss": 4.460376739501953,
+      "step": 425
+    },
+    {
+      "epoch": 0.2301388192005352,
+      "grad_norm": 1.137232786466204,
+      "learning_rate": 7.664884135472371e-07,
+      "logits/chosen": -0.5641778707504272,
+      "logits/rejected": -0.2592639625072479,
+      "logps/chosen": -4.983081817626953,
+      "logps/rejected": -5.339555740356445,
+      "loss": 0.0554,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -4.983081817626953,
+      "rewards/margins": 0.35647445917129517,
+      "rewards/rejected": -5.339555740356445,
+      "sft_loss": 4.552998065948486,
+      "step": 430
+    },
+    {
+      "epoch": 0.23281485198193677,
+      "grad_norm": 1.1748183885688712,
+      "learning_rate": 7.754010695187165e-07,
+      "logits/chosen": -0.403970867395401,
+      "logits/rejected": -0.2805110514163971,
+      "logps/chosen": -5.1232590675354,
+      "logps/rejected": -5.035851001739502,
+      "loss": 0.0569,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -5.1232590675354,
+      "rewards/margins": -0.08740736544132233,
+      "rewards/rejected": -5.035851001739502,
+      "sft_loss": 4.523171901702881,
+      "step": 435
+    },
+    {
+      "epoch": 0.23549088476333835,
+      "grad_norm": 1.9134674045638693,
+      "learning_rate": 7.84313725490196e-07,
+      "logits/chosen": -0.39476364850997925,
+      "logits/rejected": -0.21744295954704285,
+      "logps/chosen": -4.964127540588379,
+      "logps/rejected": -5.1540703773498535,
+      "loss": 0.0566,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -4.964127540588379,
+      "rewards/margins": 0.18994323909282684,
+      "rewards/rejected": -5.1540703773498535,
+      "sft_loss": 4.714345455169678,
+      "step": 440
+    },
+    {
+      "epoch": 0.23816691754473993,
+      "grad_norm": 4.445589266572552,
+      "learning_rate": 7.932263814616755e-07,
+      "logits/chosen": -0.3707229495048523,
+      "logits/rejected": -0.2078903168439865,
+      "logps/chosen": -5.059138298034668,
+      "logps/rejected": -5.273739814758301,
+      "loss": 0.0563,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -5.059138298034668,
+      "rewards/margins": 0.21460111439228058,
+      "rewards/rejected": -5.273739814758301,
+      "sft_loss": 4.584336280822754,
+      "step": 445
+    },
+    {
+      "epoch": 0.2408429503261415,
+      "grad_norm": 1.7787613623310712,
+      "learning_rate": 8.02139037433155e-07,
+      "logits/chosen": -0.3272281289100647,
+      "logits/rejected": -0.1507757008075714,
+      "logps/chosen": -4.958096027374268,
+      "logps/rejected": -5.078334808349609,
+      "loss": 0.0555,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -4.958096027374268,
+      "rewards/margins": 0.12023873627185822,
+      "rewards/rejected": -5.078334808349609,
+      "sft_loss": 4.501555442810059,
+      "step": 450
+    },
+    {
+      "epoch": 0.24351898310754308,
+      "grad_norm": 0.9480957363325464,
+      "learning_rate": 8.110516934046346e-07,
+      "logits/chosen": -0.3552139103412628,
+      "logits/rejected": -0.19632229208946228,
+      "logps/chosen": -4.692941188812256,
+      "logps/rejected": -5.041573524475098,
+      "loss": 0.0543,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.692941188812256,
+      "rewards/margins": 0.3486325144767761,
+      "rewards/rejected": -5.041573524475098,
+      "sft_loss": 4.325136184692383,
+      "step": 455
+    },
+    {
+      "epoch": 0.24619501588894463,
+      "grad_norm": 1.9429007679640615,
+      "learning_rate": 8.19964349376114e-07,
+      "logits/chosen": -0.5261731743812561,
+      "logits/rejected": -0.28895407915115356,
+      "logps/chosen": -5.142203330993652,
+      "logps/rejected": -5.269199371337891,
+      "loss": 0.0574,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -5.142203330993652,
+      "rewards/margins": 0.12699629366397858,
+      "rewards/rejected": -5.269199371337891,
+      "sft_loss": 4.904362201690674,
+      "step": 460
+    },
+    {
+      "epoch": 0.2488710486703462,
+      "grad_norm": 1.719077122753352,
+      "learning_rate": 8.288770053475936e-07,
+      "logits/chosen": -0.347450852394104,
+      "logits/rejected": -0.24443945288658142,
+      "logps/chosen": -4.623368263244629,
+      "logps/rejected": -4.905174255371094,
+      "loss": 0.0569,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.623368263244629,
+      "rewards/margins": 0.28180620074272156,
+      "rewards/rejected": -4.905174255371094,
+      "sft_loss": 4.362107753753662,
+      "step": 465
+    },
+    {
+      "epoch": 0.2515470814517478,
+      "grad_norm": 3.797351690873165,
+      "learning_rate": 8.37789661319073e-07,
+      "logits/chosen": -0.23248498141765594,
+      "logits/rejected": -0.33860307931900024,
+      "logps/chosen": -5.063205718994141,
+      "logps/rejected": -5.013503074645996,
+      "loss": 0.0581,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -5.063205718994141,
+      "rewards/margins": -0.049702536314725876,
+      "rewards/rejected": -5.013503074645996,
+      "sft_loss": 4.805975437164307,
+      "step": 470
+    },
+    {
+      "epoch": 0.25422311423314936,
+      "grad_norm": 1.0037746775043357,
+      "learning_rate": 8.467023172905525e-07,
+      "logits/chosen": -0.4212326109409332,
+      "logits/rejected": -0.16105397045612335,
+      "logps/chosen": -4.638871669769287,
+      "logps/rejected": -5.172748565673828,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.638871669769287,
+      "rewards/margins": 0.5338773131370544,
+      "rewards/rejected": -5.172748565673828,
+      "sft_loss": 4.322568893432617,
+      "step": 475
+    },
+    {
+      "epoch": 0.2568991470145509,
+      "grad_norm": 1.5393574024949208,
+      "learning_rate": 8.55614973262032e-07,
+      "logits/chosen": -0.36025190353393555,
+      "logits/rejected": -0.09443429112434387,
+      "logps/chosen": -4.921746730804443,
+      "logps/rejected": -5.215901851654053,
+      "loss": 0.0555,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -4.921746730804443,
+      "rewards/margins": 0.2941551208496094,
+      "rewards/rejected": -5.215901851654053,
+      "sft_loss": 4.582496166229248,
+      "step": 480
+    },
+    {
+      "epoch": 0.2595751797959525,
+      "grad_norm": 2.359407535802298,
+      "learning_rate": 8.645276292335115e-07,
+      "logits/chosen": -0.3606078624725342,
+      "logits/rejected": -0.2609712481498718,
+      "logps/chosen": -5.037476062774658,
+      "logps/rejected": -5.087932586669922,
+      "loss": 0.0559,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -5.037476062774658,
+      "rewards/margins": 0.05045701935887337,
+      "rewards/rejected": -5.087932586669922,
+      "sft_loss": 4.643454551696777,
+      "step": 485
+    },
+    {
+      "epoch": 0.26225121257735406,
+      "grad_norm": 0.9031378126507257,
+      "learning_rate": 8.734402852049911e-07,
+      "logits/chosen": -0.20180901885032654,
+      "logits/rejected": -0.12481292337179184,
+      "logps/chosen": -4.759294033050537,
+      "logps/rejected": -4.890738010406494,
+      "loss": 0.0552,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -4.759294033050537,
+      "rewards/margins": 0.13144339621067047,
+      "rewards/rejected": -4.890738010406494,
+      "sft_loss": 4.474959850311279,
+      "step": 490
+    },
+    {
+      "epoch": 0.26492724535875567,
+      "grad_norm": 0.864766969516503,
+      "learning_rate": 8.823529411764705e-07,
+      "logits/chosen": -0.29709574580192566,
+      "logits/rejected": -0.2729634642601013,
+      "logps/chosen": -4.82390022277832,
+      "logps/rejected": -4.918272972106934,
+      "loss": 0.0563,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -4.82390022277832,
+      "rewards/margins": 0.0943729504942894,
+      "rewards/rejected": -4.918272972106934,
+      "sft_loss": 4.600865840911865,
+      "step": 495
+    },
+    {
+      "epoch": 0.2676032781401572,
+      "grad_norm": 0.6588994171949467,
+      "learning_rate": 8.912655971479501e-07,
+      "logits/chosen": -0.37265440821647644,
+      "logits/rejected": -0.23569945991039276,
+      "logps/chosen": -4.7590532302856445,
+      "logps/rejected": -4.979560375213623,
+      "loss": 0.0546,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -4.7590532302856445,
+      "rewards/margins": 0.22050723433494568,
+      "rewards/rejected": -4.979560375213623,
+      "sft_loss": 4.484901428222656,
+      "step": 500
+    },
+    {
+      "epoch": 0.27027931092155877,
+      "grad_norm": 0.9600651230526771,
+      "learning_rate": 9.001782531194295e-07,
+      "logits/chosen": -0.3922001123428345,
+      "logits/rejected": -0.2255508154630661,
+      "logps/chosen": -4.7787580490112305,
+      "logps/rejected": -4.795541286468506,
+      "loss": 0.0567,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -4.7787580490112305,
+      "rewards/margins": 0.016783803701400757,
+      "rewards/rejected": -4.795541286468506,
+      "sft_loss": 4.484111785888672,
+      "step": 505
+    },
+    {
+      "epoch": 0.2729553437029604,
+      "grad_norm": 0.8779860995345065,
+      "learning_rate": 9.09090909090909e-07,
+      "logits/chosen": -0.25639277696609497,
+      "logits/rejected": -0.17562855780124664,
+      "logps/chosen": -4.838879585266113,
+      "logps/rejected": -5.002278804779053,
+      "loss": 0.0552,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -4.838879585266113,
+      "rewards/margins": 0.16339975595474243,
+      "rewards/rejected": -5.002278804779053,
+      "sft_loss": 4.533398628234863,
+      "step": 510
+    },
+    {
+      "epoch": 0.2756313764843619,
+      "grad_norm": 1.6340742571733686,
+      "learning_rate": 9.180035650623885e-07,
+      "logits/chosen": -0.29652127623558044,
+      "logits/rejected": -0.1382482796907425,
+      "logps/chosen": -4.777513027191162,
+      "logps/rejected": -4.839566230773926,
+      "loss": 0.0554,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -4.777513027191162,
+      "rewards/margins": 0.06205293536186218,
+      "rewards/rejected": -4.839566230773926,
+      "sft_loss": 4.462241172790527,
+      "step": 515
+    },
+    {
+      "epoch": 0.27830740926576353,
+      "grad_norm": 1.467892207273564,
+      "learning_rate": 9.26916221033868e-07,
+      "logits/chosen": -0.39073318243026733,
+      "logits/rejected": -0.17723903059959412,
+      "logps/chosen": -4.805098533630371,
+      "logps/rejected": -4.994345188140869,
+      "loss": 0.055,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -4.805098533630371,
+      "rewards/margins": 0.18924656510353088,
+      "rewards/rejected": -4.994345188140869,
+      "sft_loss": 4.511242866516113,
+      "step": 520
+    },
+    {
+      "epoch": 0.2809834420471651,
+      "grad_norm": 0.769155240820834,
+      "learning_rate": 9.358288770053476e-07,
+      "logits/chosen": -0.2005387246608734,
+      "logits/rejected": -0.061540864408016205,
+      "logps/chosen": -4.82773494720459,
+      "logps/rejected": -5.0809125900268555,
+      "loss": 0.0548,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -4.82773494720459,
+      "rewards/margins": 0.25317686796188354,
+      "rewards/rejected": -5.0809125900268555,
+      "sft_loss": 4.457303047180176,
+      "step": 525
+    },
+    {
+      "epoch": 0.2836594748285666,
+      "grad_norm": 1.1120839605961124,
+      "learning_rate": 9.44741532976827e-07,
+      "logits/chosen": -0.3281249403953552,
+      "logits/rejected": -0.25439485907554626,
+      "logps/chosen": -4.870813369750977,
+      "logps/rejected": -5.039546012878418,
+      "loss": 0.0562,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.870813369750977,
+      "rewards/margins": 0.16873237490653992,
+      "rewards/rejected": -5.039546012878418,
+      "sft_loss": 4.659360408782959,
+      "step": 530
+    },
+    {
+      "epoch": 0.28633550760996823,
+      "grad_norm": 2.0952234650377193,
+      "learning_rate": 9.536541889483066e-07,
+      "logits/chosen": -0.5139321684837341,
+      "logits/rejected": -0.1076575517654419,
+      "logps/chosen": -4.730218410491943,
+      "logps/rejected": -4.953000068664551,
+      "loss": 0.0544,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -4.730218410491943,
+      "rewards/margins": 0.22278185188770294,
+      "rewards/rejected": -4.953000068664551,
+      "sft_loss": 4.485461235046387,
+      "step": 535
+    },
+    {
+      "epoch": 0.2890115403913698,
+      "grad_norm": 0.9124179505228884,
+      "learning_rate": 9.62566844919786e-07,
+      "logits/chosen": -0.3589113652706146,
+      "logits/rejected": -0.20875516533851624,
+      "logps/chosen": -4.740080833435059,
+      "logps/rejected": -4.862841606140137,
+      "loss": 0.0563,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -4.740080833435059,
+      "rewards/margins": 0.12276099622249603,
+      "rewards/rejected": -4.862841606140137,
+      "sft_loss": 4.531832218170166,
+      "step": 540
+    },
+    {
+      "epoch": 0.2916875731727714,
+      "grad_norm": 1.040263178598027,
+      "learning_rate": 9.714795008912655e-07,
+      "logits/chosen": -0.39433753490448,
+      "logits/rejected": -0.11539351940155029,
+      "logps/chosen": -4.739431858062744,
+      "logps/rejected": -4.895320415496826,
+      "loss": 0.0544,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.739431858062744,
+      "rewards/margins": 0.15588879585266113,
+      "rewards/rejected": -4.895320415496826,
+      "sft_loss": 4.4855875968933105,
+      "step": 545
+    },
+    {
+      "epoch": 0.29436360595417294,
+      "grad_norm": 1.5371896765442068,
+      "learning_rate": 9.80392156862745e-07,
+      "logits/chosen": -0.26903194189071655,
+      "logits/rejected": -0.18110091984272003,
+      "logps/chosen": -4.590166091918945,
+      "logps/rejected": -4.735904216766357,
+      "loss": 0.056,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -4.590166091918945,
+      "rewards/margins": 0.14573858678340912,
+      "rewards/rejected": -4.735904216766357,
+      "sft_loss": 4.3490447998046875,
+      "step": 550
+    },
+    {
+      "epoch": 0.2970396387355745,
+      "grad_norm": 1.4915643897189883,
+      "learning_rate": 9.893048128342244e-07,
+      "logits/chosen": -0.36765116453170776,
+      "logits/rejected": -0.19273436069488525,
+      "logps/chosen": -5.064404487609863,
+      "logps/rejected": -5.279716491699219,
+      "loss": 0.0561,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -5.064404487609863,
+      "rewards/margins": 0.2153124362230301,
+      "rewards/rejected": -5.279716491699219,
+      "sft_loss": 4.768413543701172,
+      "step": 555
+    },
+    {
+      "epoch": 0.2997156715169761,
+      "grad_norm": 1.2834538492458512,
+      "learning_rate": 9.98217468805704e-07,
+      "logits/chosen": -0.3625454008579254,
+      "logits/rejected": -0.3016476631164551,
+      "logps/chosen": -4.831945896148682,
+      "logps/rejected": -4.9502854347229,
+      "loss": 0.0557,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -4.831945896148682,
+      "rewards/margins": 0.11833939701318741,
+      "rewards/rejected": -4.9502854347229,
+      "sft_loss": 4.503427028656006,
+      "step": 560
+    },
+    {
+      "epoch": 0.30239170429837764,
+      "grad_norm": 1.6223027628936535,
+      "learning_rate": 9.999984476788462e-07,
+      "logits/chosen": -0.3643776476383209,
+      "logits/rejected": -0.2353856861591339,
+      "logps/chosen": -4.69890832901001,
+      "logps/rejected": -4.904583930969238,
+      "loss": 0.0556,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.69890832901001,
+      "rewards/margins": 0.20567627251148224,
+      "rewards/rejected": -4.904583930969238,
+      "sft_loss": 4.513741493225098,
+      "step": 565
+    },
+    {
+      "epoch": 0.30506773707977924,
+      "grad_norm": 0.9717056305058467,
+      "learning_rate": 9.999921413906797e-07,
+      "logits/chosen": -0.4494122564792633,
+      "logits/rejected": -0.12291695922613144,
+      "logps/chosen": -4.675474643707275,
+      "logps/rejected": -4.861818790435791,
+      "loss": 0.0549,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -4.675474643707275,
+      "rewards/margins": 0.18634450435638428,
+      "rewards/rejected": -4.861818790435791,
+      "sft_loss": 4.422656059265137,
+      "step": 570
+    },
+    {
+      "epoch": 0.3077437698611808,
+      "grad_norm": 0.8102835169481186,
+      "learning_rate": 9.999809841765644e-07,
+      "logits/chosen": -0.5083755850791931,
+      "logits/rejected": -0.468639612197876,
+      "logps/chosen": -4.939351558685303,
+      "logps/rejected": -5.073581218719482,
+      "loss": 0.0562,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -4.939351558685303,
+      "rewards/margins": 0.13423016667366028,
+      "rewards/rejected": -5.073581218719482,
+      "sft_loss": 4.74433708190918,
+      "step": 575
+    },
+    {
+      "epoch": 0.3104198026425824,
+      "grad_norm": 0.484057830475299,
+      "learning_rate": 9.999649761447477e-07,
+      "logits/chosen": -0.47592979669570923,
+      "logits/rejected": -0.19933168590068817,
+      "logps/chosen": -4.55141544342041,
+      "logps/rejected": -4.823070049285889,
+      "loss": 0.0549,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.55141544342041,
+      "rewards/margins": 0.27165549993515015,
+      "rewards/rejected": -4.823070049285889,
+      "sft_loss": 4.323666572570801,
+      "step": 580
+    },
+    {
+      "epoch": 0.31309583542398395,
+      "grad_norm": 0.4954728704364703,
+      "learning_rate": 9.999441174505398e-07,
+      "logits/chosen": -0.5497112274169922,
+      "logits/rejected": -0.40136200189590454,
+      "logps/chosen": -4.930845737457275,
+      "logps/rejected": -5.082024574279785,
+      "loss": 0.0548,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -4.930845737457275,
+      "rewards/margins": 0.15117934346199036,
+      "rewards/rejected": -5.082024574279785,
+      "sft_loss": 4.561702251434326,
+      "step": 585
+    },
+    {
+      "epoch": 0.3157718682053855,
+      "grad_norm": 0.8929142904448083,
+      "learning_rate": 9.999184082963116e-07,
+      "logits/chosen": -0.5046008825302124,
+      "logits/rejected": -0.3427330255508423,
+      "logps/chosen": -4.885519981384277,
+      "logps/rejected": -5.006654739379883,
+      "loss": 0.0551,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.885519981384277,
+      "rewards/margins": 0.12113462388515472,
+      "rewards/rejected": -5.006654739379883,
+      "sft_loss": 4.5403523445129395,
+      "step": 590
+    },
+    {
+      "epoch": 0.3184479009867871,
+      "grad_norm": 0.9240145223243589,
+      "learning_rate": 9.998878489314937e-07,
+      "logits/chosen": -0.46767106652259827,
+      "logits/rejected": -0.23591558635234833,
+      "logps/chosen": -4.70677375793457,
+      "logps/rejected": -4.994019508361816,
+      "loss": 0.055,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.70677375793457,
+      "rewards/margins": 0.28724604845046997,
+      "rewards/rejected": -4.994019508361816,
+      "sft_loss": 4.432908058166504,
+      "step": 595
+    },
+    {
+      "epoch": 0.32112393376818865,
+      "grad_norm": 1.0018417636595902,
+      "learning_rate": 9.99852439652573e-07,
+      "logits/chosen": -0.5504830479621887,
+      "logits/rejected": -0.3432347774505615,
+      "logps/chosen": -4.664198875427246,
+      "logps/rejected": -4.872433662414551,
+      "loss": 0.0546,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -4.664198875427246,
+      "rewards/margins": 0.20823463797569275,
+      "rewards/rejected": -4.872433662414551,
+      "sft_loss": 4.430227756500244,
+      "step": 600
+    },
+    {
+      "epoch": 0.32379996654959026,
+      "grad_norm": 0.6275109758749665,
+      "learning_rate": 9.998121808030904e-07,
+      "logits/chosen": -0.6691871881484985,
+      "logits/rejected": -0.5714275240898132,
+      "logps/chosen": -4.999661445617676,
+      "logps/rejected": -5.115338325500488,
+      "loss": 0.0558,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -4.999661445617676,
+      "rewards/margins": 0.11567720025777817,
+      "rewards/rejected": -5.115338325500488,
+      "sft_loss": 4.749647617340088,
+      "step": 605
+    },
+    {
+      "epoch": 0.3264759993309918,
+      "grad_norm": 1.9533627210971916,
+      "learning_rate": 9.997670727736379e-07,
+      "logits/chosen": -0.5287994146347046,
+      "logits/rejected": -0.2409917414188385,
+      "logps/chosen": -4.611300468444824,
+      "logps/rejected": -4.752462387084961,
+      "loss": 0.0548,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -4.611300468444824,
+      "rewards/margins": 0.14116191864013672,
+      "rewards/rejected": -4.752462387084961,
+      "sft_loss": 4.276651382446289,
+      "step": 610
+    },
+    {
+      "epoch": 0.32915203211239336,
+      "grad_norm": 1.0181190866876273,
+      "learning_rate": 9.99717116001853e-07,
+      "logits/chosen": -0.5561403036117554,
+      "logits/rejected": -0.4006883502006531,
+      "logps/chosen": -4.733808517456055,
+      "logps/rejected": -4.987280368804932,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.733808517456055,
+      "rewards/margins": 0.25347140431404114,
+      "rewards/rejected": -4.987280368804932,
+      "sft_loss": 4.515638828277588,
+      "step": 615
+    },
+    {
+      "epoch": 0.33182806489379496,
+      "grad_norm": 1.8286375874490535,
+      "learning_rate": 9.996623109724173e-07,
+      "logits/chosen": -0.36191409826278687,
+      "logits/rejected": -0.23526597023010254,
+      "logps/chosen": -4.690080165863037,
+      "logps/rejected": -4.8197832107543945,
+      "loss": 0.0553,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.690080165863037,
+      "rewards/margins": 0.12970253825187683,
+      "rewards/rejected": -4.8197832107543945,
+      "sft_loss": 4.448857307434082,
+      "step": 620
+    },
+    {
+      "epoch": 0.3345040976751965,
+      "grad_norm": 0.8430739768632581,
+      "learning_rate": 9.996026582170488e-07,
+      "logits/chosen": -0.40729236602783203,
+      "logits/rejected": -0.15933868288993835,
+      "logps/chosen": -4.661129474639893,
+      "logps/rejected": -5.0198845863342285,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.661129474639893,
+      "rewards/margins": 0.3587549328804016,
+      "rewards/rejected": -5.0198845863342285,
+      "sft_loss": 4.314611434936523,
+      "step": 625
+    },
+    {
+      "epoch": 0.3371801304565981,
+      "grad_norm": 0.41172622641288736,
+      "learning_rate": 9.995381583144996e-07,
+      "logits/chosen": -0.41392308473587036,
+      "logits/rejected": -0.23915371298789978,
+      "logps/chosen": -4.795115947723389,
+      "logps/rejected": -5.012903690338135,
+      "loss": 0.0544,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -4.795115947723389,
+      "rewards/margins": 0.2177872359752655,
+      "rewards/rejected": -5.012903690338135,
+      "sft_loss": 4.513781547546387,
+      "step": 630
+    },
+    {
+      "epoch": 0.33985616323799966,
+      "grad_norm": 1.3659635196177828,
+      "learning_rate": 9.994688118905471e-07,
+      "logits/chosen": -0.40934377908706665,
+      "logits/rejected": -0.06164498254656792,
+      "logps/chosen": -4.684535980224609,
+      "logps/rejected": -5.066915512084961,
+      "loss": 0.0539,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.684535980224609,
+      "rewards/margins": 0.38237953186035156,
+      "rewards/rejected": -5.066915512084961,
+      "sft_loss": 4.450982093811035,
+      "step": 635
+    },
+    {
+      "epoch": 0.3425321960194012,
+      "grad_norm": 0.7655809286311621,
+      "learning_rate": 9.993946196179912e-07,
+      "logits/chosen": -0.4428192973136902,
+      "logits/rejected": -0.13932375609874725,
+      "logps/chosen": -4.84511661529541,
+      "logps/rejected": -5.081221103668213,
+      "loss": 0.055,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -4.84511661529541,
+      "rewards/margins": 0.2361036241054535,
+      "rewards/rejected": -5.081221103668213,
+      "sft_loss": 4.533552646636963,
+      "step": 640
+    },
+    {
+      "epoch": 0.3452082288008028,
+      "grad_norm": 0.753608439880354,
+      "learning_rate": 9.993155822166455e-07,
+      "logits/chosen": -0.4735592305660248,
+      "logits/rejected": -0.3594627380371094,
+      "logps/chosen": -4.647860527038574,
+      "logps/rejected": -4.909556865692139,
+      "loss": 0.0545,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -4.647860527038574,
+      "rewards/margins": 0.2616958022117615,
+      "rewards/rejected": -4.909556865692139,
+      "sft_loss": 4.432980537414551,
+      "step": 645
+    },
+    {
+      "epoch": 0.34788426158220437,
+      "grad_norm": 0.7532799343685316,
+      "learning_rate": 9.992317004533313e-07,
+      "logits/chosen": -0.3656768798828125,
+      "logits/rejected": -0.24436573684215546,
+      "logps/chosen": -4.652246952056885,
+      "logps/rejected": -4.921938896179199,
+      "loss": 0.0551,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -4.652246952056885,
+      "rewards/margins": 0.26969173550605774,
+      "rewards/rejected": -4.921938896179199,
+      "sft_loss": 4.465109825134277,
+      "step": 650
+    },
+    {
+      "epoch": 0.350560294363606,
+      "grad_norm": 0.8544184685749583,
+      "learning_rate": 9.991429751418696e-07,
+      "logits/chosen": -0.38019418716430664,
+      "logits/rejected": -0.35641294717788696,
+      "logps/chosen": -4.640519618988037,
+      "logps/rejected": -4.896888732910156,
+      "loss": 0.0548,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -4.640519618988037,
+      "rewards/margins": 0.256369024515152,
+      "rewards/rejected": -4.896888732910156,
+      "sft_loss": 4.424017429351807,
+      "step": 655
+    },
+    {
+      "epoch": 0.3532363271450075,
+      "grad_norm": 0.7903309832011153,
+      "learning_rate": 9.99049407143074e-07,
+      "logits/chosen": -0.462789922952652,
+      "logits/rejected": -0.25584983825683594,
+      "logps/chosen": -5.043513298034668,
+      "logps/rejected": -5.086087703704834,
+      "loss": 0.0565,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -5.043513298034668,
+      "rewards/margins": 0.04257440194487572,
+      "rewards/rejected": -5.086087703704834,
+      "sft_loss": 4.764005661010742,
+      "step": 660
+    },
+    {
+      "epoch": 0.35591235992640907,
+      "grad_norm": 0.609573225241613,
+      "learning_rate": 9.989509973647416e-07,
+      "logits/chosen": -0.44240602850914,
+      "logits/rejected": -0.2531934976577759,
+      "logps/chosen": -4.6058125495910645,
+      "logps/rejected": -4.872105121612549,
+      "loss": 0.054,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.6058125495910645,
+      "rewards/margins": 0.26629284024238586,
+      "rewards/rejected": -4.872105121612549,
+      "sft_loss": 4.4222412109375,
+      "step": 665
+    },
+    {
+      "epoch": 0.3585883927078107,
+      "grad_norm": 0.4364736762052332,
+      "learning_rate": 9.988477467616445e-07,
+      "logits/chosen": -0.5186976194381714,
+      "logits/rejected": -0.2430642545223236,
+      "logps/chosen": -4.617299556732178,
+      "logps/rejected": -4.82781982421875,
+      "loss": 0.0548,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.617299556732178,
+      "rewards/margins": 0.21052002906799316,
+      "rewards/rejected": -4.82781982421875,
+      "sft_loss": 4.397097110748291,
+      "step": 670
+    },
+    {
+      "epoch": 0.3612644254892122,
+      "grad_norm": 1.137938867391629,
+      "learning_rate": 9.987396563355205e-07,
+      "logits/chosen": -0.517059326171875,
+      "logits/rejected": -0.4032202661037445,
+      "logps/chosen": -4.8206658363342285,
+      "logps/rejected": -5.0536298751831055,
+      "loss": 0.0554,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.8206658363342285,
+      "rewards/margins": 0.2329637110233307,
+      "rewards/rejected": -5.0536298751831055,
+      "sft_loss": 4.577258110046387,
+      "step": 675
+    },
+    {
+      "epoch": 0.36394045827061383,
+      "grad_norm": 1.2897163566434429,
+      "learning_rate": 9.986267271350631e-07,
+      "logits/chosen": -0.440659761428833,
+      "logits/rejected": -0.23207931220531464,
+      "logps/chosen": -4.67221736907959,
+      "logps/rejected": -4.837889194488525,
+      "loss": 0.0558,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -4.67221736907959,
+      "rewards/margins": 0.1656724214553833,
+      "rewards/rejected": -4.837889194488525,
+      "sft_loss": 4.413691997528076,
+      "step": 680
+    },
+    {
+      "epoch": 0.3666164910520154,
+      "grad_norm": 0.539702639060486,
+      "learning_rate": 9.985089602559123e-07,
+      "logits/chosen": -0.4820149838924408,
+      "logits/rejected": -0.21975748240947723,
+      "logps/chosen": -4.768852233886719,
+      "logps/rejected": -5.050930976867676,
+      "loss": 0.054,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.768852233886719,
+      "rewards/margins": 0.28207892179489136,
+      "rewards/rejected": -5.050930976867676,
+      "sft_loss": 4.496817588806152,
+      "step": 685
+    },
+    {
+      "epoch": 0.369292523833417,
+      "grad_norm": 1.5387946865898154,
+      "learning_rate": 9.983863568406428e-07,
+      "logits/chosen": -0.35290640592575073,
+      "logits/rejected": -0.29900383949279785,
+      "logps/chosen": -4.636789798736572,
+      "logps/rejected": -4.796220302581787,
+      "loss": 0.0552,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -4.636789798736572,
+      "rewards/margins": 0.1594308465719223,
+      "rewards/rejected": -4.796220302581787,
+      "sft_loss": 4.371980667114258,
+      "step": 690
+    },
+    {
+      "epoch": 0.37196855661481854,
+      "grad_norm": 0.6503780378168439,
+      "learning_rate": 9.982589180787532e-07,
+      "logits/chosen": -0.4263625741004944,
+      "logits/rejected": -0.2828146815299988,
+      "logps/chosen": -4.904356002807617,
+      "logps/rejected": -5.096182823181152,
+      "loss": 0.0549,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -4.904356002807617,
+      "rewards/margins": 0.19182677567005157,
+      "rewards/rejected": -5.096182823181152,
+      "sft_loss": 4.617617130279541,
+      "step": 695
+    },
+    {
+      "epoch": 0.3746445893962201,
+      "grad_norm": 0.5447784030210343,
+      "learning_rate": 9.981266452066553e-07,
+      "logits/chosen": -0.5558158755302429,
+      "logits/rejected": -0.32897305488586426,
+      "logps/chosen": -4.607967853546143,
+      "logps/rejected": -4.868782043457031,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.607967853546143,
+      "rewards/margins": 0.2608141601085663,
+      "rewards/rejected": -4.868782043457031,
+      "sft_loss": 4.378183841705322,
+      "step": 700
+    },
+    {
+      "epoch": 0.3773206221776217,
+      "grad_norm": 0.6795933899283331,
+      "learning_rate": 9.979895395076608e-07,
+      "logits/chosen": -0.502457857131958,
+      "logits/rejected": -0.19821786880493164,
+      "logps/chosen": -4.631689071655273,
+      "logps/rejected": -4.896653175354004,
+      "loss": 0.0539,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.631689071655273,
+      "rewards/margins": 0.26496395468711853,
+      "rewards/rejected": -4.896653175354004,
+      "sft_loss": 4.34788703918457,
+      "step": 705
+    },
+    {
+      "epoch": 0.37999665495902324,
+      "grad_norm": 0.6873470967068471,
+      "learning_rate": 9.9784760231197e-07,
+      "logits/chosen": -0.5005318522453308,
+      "logits/rejected": -0.3380716145038605,
+      "logps/chosen": -4.7272210121154785,
+      "logps/rejected": -4.948512554168701,
+      "loss": 0.0545,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -4.7272210121154785,
+      "rewards/margins": 0.22129102051258087,
+      "rewards/rejected": -4.948512554168701,
+      "sft_loss": 4.455728054046631,
+      "step": 710
+    },
+    {
+      "epoch": 0.38267268774042484,
+      "grad_norm": 0.38741905047989544,
+      "learning_rate": 9.97700834996658e-07,
+      "logits/chosen": -0.5377181768417358,
+      "logits/rejected": -0.27840954065322876,
+      "logps/chosen": -4.827449798583984,
+      "logps/rejected": -5.016432285308838,
+      "loss": 0.0545,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -4.827449798583984,
+      "rewards/margins": 0.18898220360279083,
+      "rewards/rejected": -5.016432285308838,
+      "sft_loss": 4.430556297302246,
+      "step": 715
+    },
+    {
+      "epoch": 0.3853487205218264,
+      "grad_norm": 0.6920780216870869,
+      "learning_rate": 9.97549238985662e-07,
+      "logits/chosen": -0.49541395902633667,
+      "logits/rejected": -0.19519221782684326,
+      "logps/chosen": -4.899749755859375,
+      "logps/rejected": -5.18363094329834,
+      "loss": 0.0544,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -4.899749755859375,
+      "rewards/margins": 0.28388163447380066,
+      "rewards/rejected": -5.18363094329834,
+      "sft_loss": 4.593430519104004,
+      "step": 720
+    },
+    {
+      "epoch": 0.38802475330322794,
+      "grad_norm": 0.6183567658360065,
+      "learning_rate": 9.973928157497674e-07,
+      "logits/chosen": -0.7071774005889893,
+      "logits/rejected": -0.45961588621139526,
+      "logps/chosen": -4.625585079193115,
+      "logps/rejected": -4.892023086547852,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.625585079193115,
+      "rewards/margins": 0.2664377987384796,
+      "rewards/rejected": -4.892023086547852,
+      "sft_loss": 4.314015865325928,
+      "step": 725
+    },
+    {
+      "epoch": 0.39070078608462955,
+      "grad_norm": 0.8093467088520951,
+      "learning_rate": 9.972315668065927e-07,
+      "logits/chosen": -0.6060682535171509,
+      "logits/rejected": -0.4073333740234375,
+      "logps/chosen": -4.745782375335693,
+      "logps/rejected": -4.918501377105713,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -4.745782375335693,
+      "rewards/margins": 0.17271855473518372,
+      "rewards/rejected": -4.918501377105713,
+      "sft_loss": 4.408670902252197,
+      "step": 730
+    },
+    {
+      "epoch": 0.3933768188660311,
+      "grad_norm": 0.5190267369651727,
+      "learning_rate": 9.97065493720576e-07,
+      "logits/chosen": -0.6034917235374451,
+      "logits/rejected": -0.43217557668685913,
+      "logps/chosen": -4.639141082763672,
+      "logps/rejected": -4.880422115325928,
+      "loss": 0.0547,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.639141082763672,
+      "rewards/margins": 0.24128136038780212,
+      "rewards/rejected": -4.880422115325928,
+      "sft_loss": 4.418794631958008,
+      "step": 735
+    },
+    {
+      "epoch": 0.3960528516474327,
+      "grad_norm": 0.7299713178783463,
+      "learning_rate": 9.968945981029594e-07,
+      "logits/chosen": -0.5461141467094421,
+      "logits/rejected": -0.2863614559173584,
+      "logps/chosen": -4.790996551513672,
+      "logps/rejected": -5.0908708572387695,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.790996551513672,
+      "rewards/margins": 0.299874484539032,
+      "rewards/rejected": -5.0908708572387695,
+      "sft_loss": 4.418035984039307,
+      "step": 740
+    },
+    {
+      "epoch": 0.39872888442883425,
+      "grad_norm": 1.357858967463249,
+      "learning_rate": 9.967188816117726e-07,
+      "logits/chosen": -0.4381503462791443,
+      "logits/rejected": -0.24060969054698944,
+      "logps/chosen": -4.734060287475586,
+      "logps/rejected": -5.104639530181885,
+      "loss": 0.0556,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.734060287475586,
+      "rewards/margins": 0.3705799877643585,
+      "rewards/rejected": -5.104639530181885,
+      "sft_loss": 4.420103549957275,
+      "step": 745
+    },
+    {
+      "epoch": 0.4014049172102358,
+      "grad_norm": 0.5571020275754393,
+      "learning_rate": 9.965383459518179e-07,
+      "logits/chosen": -0.6509796977043152,
+      "logits/rejected": -0.3502160310745239,
+      "logps/chosen": -5.010281085968018,
+      "logps/rejected": -5.239097595214844,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -5.010281085968018,
+      "rewards/margins": 0.22881627082824707,
+      "rewards/rejected": -5.239097595214844,
+      "sft_loss": 4.609499931335449,
+      "step": 750
+    },
+    {
+      "epoch": 0.4040809499916374,
+      "grad_norm": 0.5180996933056597,
+      "learning_rate": 9.963529928746533e-07,
+      "logits/chosen": -0.5796464681625366,
+      "logits/rejected": -0.3192821145057678,
+      "logps/chosen": -4.6411871910095215,
+      "logps/rejected": -4.946473598480225,
+      "loss": 0.0545,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.6411871910095215,
+      "rewards/margins": 0.305286169052124,
+      "rewards/rejected": -4.946473598480225,
+      "sft_loss": 4.233515739440918,
+      "step": 755
+    },
+    {
+      "epoch": 0.40675698277303896,
+      "grad_norm": 0.383125627873452,
+      "learning_rate": 9.961628241785746e-07,
+      "logits/chosen": -0.7499212622642517,
+      "logits/rejected": -0.5933259129524231,
+      "logps/chosen": -4.820398807525635,
+      "logps/rejected": -5.086698055267334,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -4.820398807525635,
+      "rewards/margins": 0.266299307346344,
+      "rewards/rejected": -5.086698055267334,
+      "sft_loss": 4.4908623695373535,
+      "step": 760
+    },
+    {
+      "epoch": 0.40943301555444056,
+      "grad_norm": 0.7665403862710062,
+      "learning_rate": 9.959678417085998e-07,
+      "logits/chosen": -0.684373140335083,
+      "logits/rejected": -0.5438544154167175,
+      "logps/chosen": -4.870017051696777,
+      "logps/rejected": -5.046212196350098,
+      "loss": 0.0552,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -4.870017051696777,
+      "rewards/margins": 0.17619472742080688,
+      "rewards/rejected": -5.046212196350098,
+      "sft_loss": 4.622311115264893,
+      "step": 765
+    },
+    {
+      "epoch": 0.4121090483358421,
+      "grad_norm": 0.41415913294882845,
+      "learning_rate": 9.957680473564493e-07,
+      "logits/chosen": -0.5799676775932312,
+      "logits/rejected": -0.32691285014152527,
+      "logps/chosen": -4.706719398498535,
+      "logps/rejected": -4.918424129486084,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -4.706719398498535,
+      "rewards/margins": 0.21170508861541748,
+      "rewards/rejected": -4.918424129486084,
+      "sft_loss": 4.271069526672363,
+      "step": 770
+    },
+    {
+      "epoch": 0.41478508111724366,
+      "grad_norm": 0.443777235736431,
+      "learning_rate": 9.95563443060529e-07,
+      "logits/chosen": -0.7746785879135132,
+      "logits/rejected": -0.47731414437294006,
+      "logps/chosen": -4.810762405395508,
+      "logps/rejected": -5.0177717208862305,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -4.810762405395508,
+      "rewards/margins": 0.2070087492465973,
+      "rewards/rejected": -5.0177717208862305,
+      "sft_loss": 4.463230609893799,
+      "step": 775
+    },
+    {
+      "epoch": 0.41746111389864526,
+      "grad_norm": 0.5265592870739666,
+      "learning_rate": 9.95354030805911e-07,
+      "logits/chosen": -0.7950371503829956,
+      "logits/rejected": -0.5439642071723938,
+      "logps/chosen": -4.769791603088379,
+      "logps/rejected": -4.9892578125,
+      "loss": 0.054,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -4.769791603088379,
+      "rewards/margins": 0.21946604549884796,
+      "rewards/rejected": -4.9892578125,
+      "sft_loss": 4.466793537139893,
+      "step": 780
+    },
+    {
+      "epoch": 0.4201371466800468,
+      "grad_norm": 0.8208738399246265,
+      "learning_rate": 9.951398126243133e-07,
+      "logits/chosen": -0.6324111223220825,
+      "logits/rejected": -0.486175000667572,
+      "logps/chosen": -4.708271026611328,
+      "logps/rejected": -4.979205131530762,
+      "loss": 0.0544,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.708271026611328,
+      "rewards/margins": 0.2709343433380127,
+      "rewards/rejected": -4.979205131530762,
+      "sft_loss": 4.414024829864502,
+      "step": 785
+    },
+    {
+      "epoch": 0.4228131794614484,
+      "grad_norm": 0.7140836228407493,
+      "learning_rate": 9.94920790594082e-07,
+      "logits/chosen": -0.6593544483184814,
+      "logits/rejected": -0.48353734612464905,
+      "logps/chosen": -4.598227500915527,
+      "logps/rejected": -4.899634838104248,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.598227500915527,
+      "rewards/margins": 0.30140742659568787,
+      "rewards/rejected": -4.899634838104248,
+      "sft_loss": 4.312748908996582,
+      "step": 790
+    },
+    {
+      "epoch": 0.42548921224284997,
+      "grad_norm": 0.4014903013901931,
+      "learning_rate": 9.946969668401696e-07,
+      "logits/chosen": -0.6135523915290833,
+      "logits/rejected": -0.32322195172309875,
+      "logps/chosen": -4.550875663757324,
+      "logps/rejected": -4.979404926300049,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.550875663757324,
+      "rewards/margins": 0.4285293519496918,
+      "rewards/rejected": -4.979404926300049,
+      "sft_loss": 4.304534912109375,
+      "step": 795
+    },
+    {
+      "epoch": 0.4281652450242516,
+      "grad_norm": 1.1777799079408109,
+      "learning_rate": 9.944683435341155e-07,
+      "logits/chosen": -0.433992862701416,
+      "logits/rejected": -0.33607035875320435,
+      "logps/chosen": -4.869369029998779,
+      "logps/rejected": -5.201943397521973,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.869369029998779,
+      "rewards/margins": 0.3325735628604889,
+      "rewards/rejected": -5.201943397521973,
+      "sft_loss": 4.4167160987854,
+      "step": 800
+    },
+    {
+      "epoch": 0.4281652450242516,
+      "eval_logits/chosen": 0.0762772411108017,
+      "eval_logits/rejected": 0.21884751319885254,
+      "eval_logps/chosen": -4.661361217498779,
+      "eval_logps/rejected": -4.99029541015625,
+      "eval_loss": 0.05288328602910042,
+      "eval_rewards/accuracies": 0.6023738980293274,
+      "eval_rewards/chosen": -4.661361217498779,
+      "eval_rewards/margins": 0.32893452048301697,
+      "eval_rewards/rejected": -4.99029541015625,
+      "eval_runtime": 43.6891,
+      "eval_samples_per_second": 30.786,
+      "eval_sft_loss": 4.132972717285156,
+      "eval_steps_per_second": 7.714,
+      "step": 800
+    },
+    {
+      "epoch": 0.4308412778056531,
+      "grad_norm": 1.019461476829721,
+      "learning_rate": 9.942349228940236e-07,
+      "logits/chosen": -0.5534158945083618,
+      "logits/rejected": -0.2658959627151489,
+      "logps/chosen": -4.665492057800293,
+      "logps/rejected": -5.222234725952148,
+      "loss": 0.053,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.665492057800293,
+      "rewards/margins": 0.5567426681518555,
+      "rewards/rejected": -5.222234725952148,
+      "sft_loss": 4.35896635055542,
+      "step": 805
+    },
+    {
+      "epoch": 0.43351731058705467,
+      "grad_norm": 0.6905383851627591,
+      "learning_rate": 9.939967071845424e-07,
+      "logits/chosen": -0.5316272974014282,
+      "logits/rejected": -0.44879570603370667,
+      "logps/chosen": -4.868673324584961,
+      "logps/rejected": -5.094768524169922,
+      "loss": 0.0559,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -4.868673324584961,
+      "rewards/margins": 0.22609524428844452,
+      "rewards/rejected": -5.094768524169922,
+      "sft_loss": 4.565918922424316,
+      "step": 810
+    },
+    {
+      "epoch": 0.4361933433684563,
+      "grad_norm": 0.9640958718568238,
+      "learning_rate": 9.937536987168413e-07,
+      "logits/chosen": -0.5573722124099731,
+      "logits/rejected": -0.34027716517448425,
+      "logps/chosen": -4.796445846557617,
+      "logps/rejected": -5.075738430023193,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -4.796445846557617,
+      "rewards/margins": 0.27929285168647766,
+      "rewards/rejected": -5.075738430023193,
+      "sft_loss": 4.393874645233154,
+      "step": 815
+    },
+    {
+      "epoch": 0.4388693761498578,
+      "grad_norm": 0.45255104653719846,
+      "learning_rate": 9.935058998485896e-07,
+      "logits/chosen": -0.5388621687889099,
+      "logits/rejected": -0.5094455480575562,
+      "logps/chosen": -4.702640533447266,
+      "logps/rejected": -4.948770523071289,
+      "loss": 0.0544,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -4.702640533447266,
+      "rewards/margins": 0.24612972140312195,
+      "rewards/rejected": -4.948770523071289,
+      "sft_loss": 4.344797134399414,
+      "step": 820
+    },
+    {
+      "epoch": 0.44154540893125943,
+      "grad_norm": 1.1306242091401009,
+      "learning_rate": 9.932533129839333e-07,
+      "logits/chosen": -0.7148083448410034,
+      "logits/rejected": -0.5382518172264099,
+      "logps/chosen": -4.7975382804870605,
+      "logps/rejected": -4.953667640686035,
+      "loss": 0.0552,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.7975382804870605,
+      "rewards/margins": 0.15612894296646118,
+      "rewards/rejected": -4.953667640686035,
+      "sft_loss": 4.56071138381958,
+      "step": 825
+    },
+    {
+      "epoch": 0.444221441712661,
+      "grad_norm": 0.6894798373046558,
+      "learning_rate": 9.929959405734711e-07,
+      "logits/chosen": -0.533441424369812,
+      "logits/rejected": -0.3360450863838196,
+      "logps/chosen": -4.676575660705566,
+      "logps/rejected": -4.950590133666992,
+      "loss": 0.0543,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.676575660705566,
+      "rewards/margins": 0.2740144729614258,
+      "rewards/rejected": -4.950590133666992,
+      "sft_loss": 4.489292621612549,
+      "step": 830
+    },
+    {
+      "epoch": 0.44689747449406253,
+      "grad_norm": 0.5615731774657157,
+      "learning_rate": 9.927337851142314e-07,
+      "logits/chosen": -0.535950779914856,
+      "logits/rejected": -0.3678244650363922,
+      "logps/chosen": -4.804347991943359,
+      "logps/rejected": -4.954916954040527,
+      "loss": 0.0554,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -4.804347991943359,
+      "rewards/margins": 0.1505691409111023,
+      "rewards/rejected": -4.954916954040527,
+      "sft_loss": 4.574199199676514,
+      "step": 835
+    },
+    {
+      "epoch": 0.44957350727546413,
+      "grad_norm": 0.5138053605954034,
+      "learning_rate": 9.924668491496474e-07,
+      "logits/chosen": -0.598753809928894,
+      "logits/rejected": -0.30034974217414856,
+      "logps/chosen": -4.644226551055908,
+      "logps/rejected": -4.914678573608398,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.644226551055908,
+      "rewards/margins": 0.270452082157135,
+      "rewards/rejected": -4.914678573608398,
+      "sft_loss": 4.4098100662231445,
+      "step": 840
+    },
+    {
+      "epoch": 0.4522495400568657,
+      "grad_norm": 0.3953000760486427,
+      "learning_rate": 9.92195135269533e-07,
+      "logits/chosen": -0.571025013923645,
+      "logits/rejected": -0.4940189719200134,
+      "logps/chosen": -4.623247146606445,
+      "logps/rejected": -4.7912702560424805,
+      "loss": 0.0547,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.623247146606445,
+      "rewards/margins": 0.16802339255809784,
+      "rewards/rejected": -4.7912702560424805,
+      "sft_loss": 4.395881175994873,
+      "step": 845
+    },
+    {
+      "epoch": 0.4549255728382673,
+      "grad_norm": 0.9784287629098288,
+      "learning_rate": 9.919186461100574e-07,
+      "logits/chosen": -0.7064191699028015,
+      "logits/rejected": -0.5537311434745789,
+      "logps/chosen": -4.647214889526367,
+      "logps/rejected": -4.877057075500488,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.647214889526367,
+      "rewards/margins": 0.22984282672405243,
+      "rewards/rejected": -4.877057075500488,
+      "sft_loss": 4.406468391418457,
+      "step": 850
+    },
+    {
+      "epoch": 0.45760160561966884,
+      "grad_norm": 0.6202522928914502,
+      "learning_rate": 9.9163738435372e-07,
+      "logits/chosen": -0.6961187124252319,
+      "logits/rejected": -0.4674125611782074,
+      "logps/chosen": -4.66948938369751,
+      "logps/rejected": -5.008526802062988,
+      "loss": 0.0547,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -4.66948938369751,
+      "rewards/margins": 0.33903709053993225,
+      "rewards/rejected": -5.008526802062988,
+      "sft_loss": 4.422011852264404,
+      "step": 855
+    },
+    {
+      "epoch": 0.4602776384010704,
+      "grad_norm": 1.1274872046878186,
+      "learning_rate": 9.913513527293234e-07,
+      "logits/chosen": -0.8537279367446899,
+      "logits/rejected": -0.6118448376655579,
+      "logps/chosen": -4.752366065979004,
+      "logps/rejected": -5.068416595458984,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.752366065979004,
+      "rewards/margins": 0.3160504400730133,
+      "rewards/rejected": -5.068416595458984,
+      "sft_loss": 4.435427665710449,
+      "step": 860
+    },
+    {
+      "epoch": 0.462953671182472,
+      "grad_norm": 0.9165527333766942,
+      "learning_rate": 9.910605540119474e-07,
+      "logits/chosen": -0.7939995527267456,
+      "logits/rejected": -0.6034280061721802,
+      "logps/chosen": -4.908832550048828,
+      "logps/rejected": -5.1738457679748535,
+      "loss": 0.0545,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.908832550048828,
+      "rewards/margins": 0.2650133967399597,
+      "rewards/rejected": -5.1738457679748535,
+      "sft_loss": 4.486456394195557,
+      "step": 865
+    },
+    {
+      "epoch": 0.46562970396387354,
+      "grad_norm": 0.42446011187865473,
+      "learning_rate": 9.907649910229227e-07,
+      "logits/chosen": -0.9538064002990723,
+      "logits/rejected": -0.5534366965293884,
+      "logps/chosen": -4.690494537353516,
+      "logps/rejected": -4.987832546234131,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.690494537353516,
+      "rewards/margins": 0.2973388135433197,
+      "rewards/rejected": -4.987832546234131,
+      "sft_loss": 4.383624076843262,
+      "step": 870
+    },
+    {
+      "epoch": 0.46830573674527515,
+      "grad_norm": 0.5446443287919899,
+      "learning_rate": 9.90464666629803e-07,
+      "logits/chosen": -0.6814507246017456,
+      "logits/rejected": -0.545750081539154,
+      "logps/chosen": -4.67910099029541,
+      "logps/rejected": -4.943177223205566,
+      "loss": 0.0551,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -4.67910099029541,
+      "rewards/margins": 0.26407718658447266,
+      "rewards/rejected": -4.943177223205566,
+      "sft_loss": 4.388939380645752,
+      "step": 875
+    },
+    {
+      "epoch": 0.4709817695266767,
+      "grad_norm": 0.7810222448754378,
+      "learning_rate": 9.901595837463363e-07,
+      "logits/chosen": -0.6461896896362305,
+      "logits/rejected": -0.390037477016449,
+      "logps/chosen": -4.738075256347656,
+      "logps/rejected": -5.075900554656982,
+      "loss": 0.0543,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.738075256347656,
+      "rewards/margins": 0.33782586455345154,
+      "rewards/rejected": -5.075900554656982,
+      "sft_loss": 4.471536159515381,
+      "step": 880
+    },
+    {
+      "epoch": 0.47365780230807825,
+      "grad_norm": 0.46368875012727806,
+      "learning_rate": 9.898497453324384e-07,
+      "logits/chosen": -0.6110280752182007,
+      "logits/rejected": -0.501623809337616,
+      "logps/chosen": -4.8805389404296875,
+      "logps/rejected": -5.093916893005371,
+      "loss": 0.054,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.8805389404296875,
+      "rewards/margins": 0.21337802708148956,
+      "rewards/rejected": -5.093916893005371,
+      "sft_loss": 4.527132034301758,
+      "step": 885
+    },
+    {
+      "epoch": 0.47633383508947985,
+      "grad_norm": 0.5772927826805968,
+      "learning_rate": 9.895351543941628e-07,
+      "logits/chosen": -0.6308537721633911,
+      "logits/rejected": -0.45382413268089294,
+      "logps/chosen": -4.536052703857422,
+      "logps/rejected": -4.7554030418396,
+      "loss": 0.0549,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -4.536052703857422,
+      "rewards/margins": 0.21935030817985535,
+      "rewards/rejected": -4.7554030418396,
+      "sft_loss": 4.236863136291504,
+      "step": 890
+    },
+    {
+      "epoch": 0.4790098678708814,
+      "grad_norm": 0.5909700614108395,
+      "learning_rate": 9.892158139836724e-07,
+      "logits/chosen": -0.667130172252655,
+      "logits/rejected": -0.5236655473709106,
+      "logps/chosen": -5.009900093078613,
+      "logps/rejected": -5.105405330657959,
+      "loss": 0.0549,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -5.009900093078613,
+      "rewards/margins": 0.09550575166940689,
+      "rewards/rejected": -5.105405330657959,
+      "sft_loss": 4.719082355499268,
+      "step": 895
+    },
+    {
+      "epoch": 0.481685900652283,
+      "grad_norm": 1.2509012937984993,
+      "learning_rate": 9.88891727199209e-07,
+      "logits/chosen": -0.7147120833396912,
+      "logits/rejected": -0.5922271609306335,
+      "logps/chosen": -4.690701961517334,
+      "logps/rejected": -4.896805763244629,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.690701961517334,
+      "rewards/margins": 0.20610396564006805,
+      "rewards/rejected": -4.896805763244629,
+      "sft_loss": 4.374290466308594,
+      "step": 900
+    },
+    {
+      "epoch": 0.48436193343368455,
+      "grad_norm": 0.5020956780576539,
+      "learning_rate": 9.885628971850641e-07,
+      "logits/chosen": -0.6695514917373657,
+      "logits/rejected": -0.40942034125328064,
+      "logps/chosen": -4.461915016174316,
+      "logps/rejected": -4.818450927734375,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.461915016174316,
+      "rewards/margins": 0.35653623938560486,
+      "rewards/rejected": -4.818450927734375,
+      "sft_loss": 4.175963878631592,
+      "step": 905
+    },
+    {
+      "epoch": 0.48703796621508616,
+      "grad_norm": 0.2706776295092486,
+      "learning_rate": 9.882293271315481e-07,
+      "logits/chosen": -0.7963489294052124,
+      "logits/rejected": -0.6448882818222046,
+      "logps/chosen": -4.85056209564209,
+      "logps/rejected": -5.1240363121032715,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.85056209564209,
+      "rewards/margins": 0.2734738886356354,
+      "rewards/rejected": -5.1240363121032715,
+      "sft_loss": 4.56730842590332,
+      "step": 910
+    },
+    {
+      "epoch": 0.4897139989964877,
+      "grad_norm": 0.7830577955619996,
+      "learning_rate": 9.878910202749589e-07,
+      "logits/chosen": -0.7318392992019653,
+      "logits/rejected": -0.4440035820007324,
+      "logps/chosen": -4.606733322143555,
+      "logps/rejected": -4.9441142082214355,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.606733322143555,
+      "rewards/margins": 0.33738085627555847,
+      "rewards/rejected": -4.9441142082214355,
+      "sft_loss": 4.38753080368042,
+      "step": 915
+    },
+    {
+      "epoch": 0.49239003177788926,
+      "grad_norm": 0.5249861703336051,
+      "learning_rate": 9.875479798975512e-07,
+      "logits/chosen": -0.5288979411125183,
+      "logits/rejected": -0.25817009806632996,
+      "logps/chosen": -4.555031776428223,
+      "logps/rejected": -4.973117828369141,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.555031776428223,
+      "rewards/margins": 0.41808605194091797,
+      "rewards/rejected": -4.973117828369141,
+      "sft_loss": 4.3153204917907715,
+      "step": 920
+    },
+    {
+      "epoch": 0.49506606455929086,
+      "grad_norm": 0.6958792681399926,
+      "learning_rate": 9.87200209327504e-07,
+      "logits/chosen": -0.6519374847412109,
+      "logits/rejected": -0.3630174994468689,
+      "logps/chosen": -4.784842014312744,
+      "logps/rejected": -4.913394927978516,
+      "loss": 0.0546,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -4.784842014312744,
+      "rewards/margins": 0.12855303287506104,
+      "rewards/rejected": -4.913394927978516,
+      "sft_loss": 4.431909561157227,
+      "step": 925
+    },
+    {
+      "epoch": 0.4977420973406924,
+      "grad_norm": 0.6857385655509702,
+      "learning_rate": 9.868477119388894e-07,
+      "logits/chosen": -0.6547890901565552,
+      "logits/rejected": -0.5493338704109192,
+      "logps/chosen": -4.803074836730957,
+      "logps/rejected": -5.25410795211792,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.803074836730957,
+      "rewards/margins": 0.45103350281715393,
+      "rewards/rejected": -5.25410795211792,
+      "sft_loss": 4.5226616859436035,
+      "step": 930
+    },
+    {
+      "epoch": 0.500418130122094,
+      "grad_norm": 0.7494384327463705,
+      "learning_rate": 9.864904911516383e-07,
+      "logits/chosen": -0.5585234761238098,
+      "logits/rejected": -0.4689217209815979,
+      "logps/chosen": -4.659033298492432,
+      "logps/rejected": -4.939300537109375,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -4.659033298492432,
+      "rewards/margins": 0.28026682138442993,
+      "rewards/rejected": -4.939300537109375,
+      "sft_loss": 4.326298713684082,
+      "step": 935
+    },
+    {
+      "epoch": 0.5030941629034956,
+      "grad_norm": 0.5667236667391294,
+      "learning_rate": 9.861285504315084e-07,
+      "logits/chosen": -0.549680233001709,
+      "logits/rejected": -0.4027118682861328,
+      "logps/chosen": -4.72601318359375,
+      "logps/rejected": -5.003291130065918,
+      "loss": 0.0547,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.72601318359375,
+      "rewards/margins": 0.2772785723209381,
+      "rewards/rejected": -5.003291130065918,
+      "sft_loss": 4.4270710945129395,
+      "step": 940
+    },
+    {
+      "epoch": 0.5057701956848971,
+      "grad_norm": 0.3704541163727195,
+      "learning_rate": 9.857618932900502e-07,
+      "logits/chosen": -0.6717230081558228,
+      "logits/rejected": -0.4291006624698639,
+      "logps/chosen": -4.774092197418213,
+      "logps/rejected": -5.1116228103637695,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.774092197418213,
+      "rewards/margins": 0.3375304341316223,
+      "rewards/rejected": -5.1116228103637695,
+      "sft_loss": 4.614851474761963,
+      "step": 945
+    },
+    {
+      "epoch": 0.5084462284662987,
+      "grad_norm": 1.080406869888111,
+      "learning_rate": 9.853905232845727e-07,
+      "logits/chosen": -0.5478076934814453,
+      "logits/rejected": -0.3193608224391937,
+      "logps/chosen": -4.520052909851074,
+      "logps/rejected": -4.752264976501465,
+      "loss": 0.0547,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.520052909851074,
+      "rewards/margins": 0.23221249878406525,
+      "rewards/rejected": -4.752264976501465,
+      "sft_loss": 4.331027507781982,
+      "step": 950
+    },
+    {
+      "epoch": 0.5111222612477003,
+      "grad_norm": 0.47093914637526024,
+      "learning_rate": 9.850144440181095e-07,
+      "logits/chosen": -0.48740583658218384,
+      "logits/rejected": -0.2183370292186737,
+      "logps/chosen": -4.712460994720459,
+      "logps/rejected": -4.964376926422119,
+      "loss": 0.0543,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.712460994720459,
+      "rewards/margins": 0.25191575288772583,
+      "rewards/rejected": -4.964376926422119,
+      "sft_loss": 4.447497367858887,
+      "step": 955
+    },
+    {
+      "epoch": 0.5137982940291018,
+      "grad_norm": 0.9253745426059107,
+      "learning_rate": 9.846336591393832e-07,
+      "logits/chosen": -0.45807933807373047,
+      "logits/rejected": -0.2740600109100342,
+      "logps/chosen": -4.678889751434326,
+      "logps/rejected": -4.9867753982543945,
+      "loss": 0.055,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.678889751434326,
+      "rewards/margins": 0.30788561701774597,
+      "rewards/rejected": -4.9867753982543945,
+      "sft_loss": 4.41696834564209,
+      "step": 960
+    },
+    {
+      "epoch": 0.5164743268105034,
+      "grad_norm": 0.5562158528705252,
+      "learning_rate": 9.842481723427704e-07,
+      "logits/chosen": -0.43384265899658203,
+      "logits/rejected": -0.3705459535121918,
+      "logps/chosen": -4.858570098876953,
+      "logps/rejected": -5.0885772705078125,
+      "loss": 0.0552,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -4.858570098876953,
+      "rewards/margins": 0.23000743985176086,
+      "rewards/rejected": -5.0885772705078125,
+      "sft_loss": 4.645135402679443,
+      "step": 965
+    },
+    {
+      "epoch": 0.519150359591905,
+      "grad_norm": 0.46758288708109713,
+      "learning_rate": 9.838579873682658e-07,
+      "logits/chosen": -0.4290435314178467,
+      "logits/rejected": -0.42416420578956604,
+      "logps/chosen": -4.561893463134766,
+      "logps/rejected": -4.798999786376953,
+      "loss": 0.0547,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -4.561893463134766,
+      "rewards/margins": 0.2371063530445099,
+      "rewards/rejected": -4.798999786376953,
+      "sft_loss": 4.215584754943848,
+      "step": 970
+    },
+    {
+      "epoch": 0.5218263923733065,
+      "grad_norm": 0.9168000242182311,
+      "learning_rate": 9.834631080014457e-07,
+      "logits/chosen": -0.6926376223564148,
+      "logits/rejected": -0.4112038016319275,
+      "logps/chosen": -4.897233486175537,
+      "logps/rejected": -5.223374366760254,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.897233486175537,
+      "rewards/margins": 0.3261413872241974,
+      "rewards/rejected": -5.223374366760254,
+      "sft_loss": 4.6407694816589355,
+      "step": 975
+    },
+    {
+      "epoch": 0.5245024251547081,
+      "grad_norm": 0.8592004725850547,
+      "learning_rate": 9.830635380734312e-07,
+      "logits/chosen": -0.6455878615379333,
+      "logits/rejected": -0.4143894612789154,
+      "logps/chosen": -4.61000919342041,
+      "logps/rejected": -4.840395450592041,
+      "loss": 0.0545,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -4.61000919342041,
+      "rewards/margins": 0.23038557171821594,
+      "rewards/rejected": -4.840395450592041,
+      "sft_loss": 4.316131591796875,
+      "step": 980
+    },
+    {
+      "epoch": 0.5271784579361097,
+      "grad_norm": 0.4155059102576832,
+      "learning_rate": 9.826592814608517e-07,
+      "logits/chosen": -0.5832679867744446,
+      "logits/rejected": -0.30334407091140747,
+      "logps/chosen": -4.680327415466309,
+      "logps/rejected": -4.880679130554199,
+      "loss": 0.0543,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -4.680327415466309,
+      "rewards/margins": 0.20035116374492645,
+      "rewards/rejected": -4.880679130554199,
+      "sft_loss": 4.373396873474121,
+      "step": 985
+    },
+    {
+      "epoch": 0.5298544907175113,
+      "grad_norm": 0.6151022174046136,
+      "learning_rate": 9.822503420858067e-07,
+      "logits/chosen": -0.5774039030075073,
+      "logits/rejected": -0.5873244404792786,
+      "logps/chosen": -4.89186429977417,
+      "logps/rejected": -5.018913269042969,
+      "loss": 0.0546,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -4.89186429977417,
+      "rewards/margins": 0.12704893946647644,
+      "rewards/rejected": -5.018913269042969,
+      "sft_loss": 4.576397895812988,
+      "step": 990
+    },
+    {
+      "epoch": 0.5325305234989128,
+      "grad_norm": 0.9617406539039807,
+      "learning_rate": 9.818367239158277e-07,
+      "logits/chosen": -0.4993751645088196,
+      "logits/rejected": -0.41326743364334106,
+      "logps/chosen": -4.793762683868408,
+      "logps/rejected": -4.96630334854126,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -4.793762683868408,
+      "rewards/margins": 0.1725403368473053,
+      "rewards/rejected": -4.96630334854126,
+      "sft_loss": 4.41725492477417,
+      "step": 995
+    },
+    {
+      "epoch": 0.5352065562803144,
+      "grad_norm": 0.9060064144194804,
+      "learning_rate": 9.8141843096384e-07,
+      "logits/chosen": -0.5664402842521667,
+      "logits/rejected": -0.34098461270332336,
+      "logps/chosen": -4.702371120452881,
+      "logps/rejected": -5.04431676864624,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.702371120452881,
+      "rewards/margins": 0.3419460654258728,
+      "rewards/rejected": -5.04431676864624,
+      "sft_loss": 4.421466827392578,
+      "step": 1000
+    },
+    {
+      "epoch": 0.537882589061716,
+      "grad_norm": 0.5086244800614527,
+      "learning_rate": 9.809954672881237e-07,
+      "logits/chosen": -0.46155256032943726,
+      "logits/rejected": -0.2267765998840332,
+      "logps/chosen": -4.836634159088135,
+      "logps/rejected": -5.148859024047852,
+      "loss": 0.0543,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -4.836634159088135,
+      "rewards/margins": 0.3122252821922302,
+      "rewards/rejected": -5.148859024047852,
+      "sft_loss": 4.532876968383789,
+      "step": 1005
+    },
+    {
+      "epoch": 0.5405586218431175,
+      "grad_norm": 0.5176592683524385,
+      "learning_rate": 9.80567836992274e-07,
+      "logits/chosen": -0.47870248556137085,
+      "logits/rejected": -0.22733981907367706,
+      "logps/chosen": -4.524051666259766,
+      "logps/rejected": -4.881123065948486,
+      "loss": 0.0549,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.524051666259766,
+      "rewards/margins": 0.35707104206085205,
+      "rewards/rejected": -4.881123065948486,
+      "sft_loss": 4.257218837738037,
+      "step": 1010
+    },
+    {
+      "epoch": 0.5432346546245191,
+      "grad_norm": 0.6158973094099502,
+      "learning_rate": 9.801355442251625e-07,
+      "logits/chosen": -0.7883400321006775,
+      "logits/rejected": -0.5459173917770386,
+      "logps/chosen": -4.803179740905762,
+      "logps/rejected": -5.041607856750488,
+      "loss": 0.0545,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.803179740905762,
+      "rewards/margins": 0.23842862248420715,
+      "rewards/rejected": -5.041607856750488,
+      "sft_loss": 4.608723163604736,
+      "step": 1015
+    },
+    {
+      "epoch": 0.5459106874059207,
+      "grad_norm": 0.38857303823842887,
+      "learning_rate": 9.796985931808949e-07,
+      "logits/chosen": -0.8481922149658203,
+      "logits/rejected": -0.5922120809555054,
+      "logps/chosen": -4.708874702453613,
+      "logps/rejected": -4.960658073425293,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.708874702453613,
+      "rewards/margins": 0.25178369879722595,
+      "rewards/rejected": -4.960658073425293,
+      "sft_loss": 4.406599521636963,
+      "step": 1020
+    },
+    {
+      "epoch": 0.5485867201873222,
+      "grad_norm": 0.37995724426168664,
+      "learning_rate": 9.792569880987724e-07,
+      "logits/chosen": -0.7771550416946411,
+      "logits/rejected": -0.6147995591163635,
+      "logps/chosen": -4.492480754852295,
+      "logps/rejected": -4.806388854980469,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.492480754852295,
+      "rewards/margins": 0.3139081597328186,
+      "rewards/rejected": -4.806388854980469,
+      "sft_loss": 4.236155986785889,
+      "step": 1025
+    },
+    {
+      "epoch": 0.5512627529687238,
+      "grad_norm": 0.5433034173994814,
+      "learning_rate": 9.788107332632493e-07,
+      "logits/chosen": -0.7106307744979858,
+      "logits/rejected": -0.619476318359375,
+      "logps/chosen": -4.824548244476318,
+      "logps/rejected": -4.9400177001953125,
+      "loss": 0.0569,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -4.824548244476318,
+      "rewards/margins": 0.11546945571899414,
+      "rewards/rejected": -4.9400177001953125,
+      "sft_loss": 4.580724239349365,
+      "step": 1030
+    },
+    {
+      "epoch": 0.5539387857501255,
+      "grad_norm": 0.39975295156815127,
+      "learning_rate": 9.783598330038924e-07,
+      "logits/chosen": -0.7352392077445984,
+      "logits/rejected": -0.5586301684379578,
+      "logps/chosen": -4.770339488983154,
+      "logps/rejected": -4.9732537269592285,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.770339488983154,
+      "rewards/margins": 0.20291383564472198,
+      "rewards/rejected": -4.9732537269592285,
+      "sft_loss": 4.535826206207275,
+      "step": 1035
+    },
+    {
+      "epoch": 0.5566148185315271,
+      "grad_norm": 0.5679160526366398,
+      "learning_rate": 9.779042916953376e-07,
+      "logits/chosen": -0.6250237226486206,
+      "logits/rejected": -0.35187873244285583,
+      "logps/chosen": -4.4509196281433105,
+      "logps/rejected": -4.852322578430176,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.4509196281433105,
+      "rewards/margins": 0.4014025330543518,
+      "rewards/rejected": -4.852322578430176,
+      "sft_loss": 4.283907413482666,
+      "step": 1040
+    },
+    {
+      "epoch": 0.5592908513129285,
+      "grad_norm": 0.45573423362566345,
+      "learning_rate": 9.774441137572487e-07,
+      "logits/chosen": -0.6780288219451904,
+      "logits/rejected": -0.4629104733467102,
+      "logps/chosen": -4.64642858505249,
+      "logps/rejected": -4.976282596588135,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.64642858505249,
+      "rewards/margins": 0.32985401153564453,
+      "rewards/rejected": -4.976282596588135,
+      "sft_loss": 4.352892875671387,
+      "step": 1045
+    },
+    {
+      "epoch": 0.5619668840943302,
+      "grad_norm": 0.422512298556522,
+      "learning_rate": 9.76979303654274e-07,
+      "logits/chosen": -0.6532023549079895,
+      "logits/rejected": -0.502606213092804,
+      "logps/chosen": -4.712841033935547,
+      "logps/rejected": -5.147298336029053,
+      "loss": 0.0539,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.712841033935547,
+      "rewards/margins": 0.434457391500473,
+      "rewards/rejected": -5.147298336029053,
+      "sft_loss": 4.4879865646362305,
+      "step": 1050
+    },
+    {
+      "epoch": 0.5646429168757318,
+      "grad_norm": 0.8131359180672422,
+      "learning_rate": 9.765098658960035e-07,
+      "logits/chosen": -0.5284560918807983,
+      "logits/rejected": -0.4684422016143799,
+      "logps/chosen": -4.553152561187744,
+      "logps/rejected": -4.902011871337891,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.553152561187744,
+      "rewards/margins": 0.3488597273826599,
+      "rewards/rejected": -4.902011871337891,
+      "sft_loss": 4.281155586242676,
+      "step": 1055
+    },
+    {
+      "epoch": 0.5673189496571333,
+      "grad_norm": 0.9015355821027234,
+      "learning_rate": 9.76035805036924e-07,
+      "logits/chosen": -0.518097996711731,
+      "logits/rejected": -0.28143590688705444,
+      "logps/chosen": -4.709748268127441,
+      "logps/rejected": -5.019460201263428,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.709748268127441,
+      "rewards/margins": 0.30971240997314453,
+      "rewards/rejected": -5.019460201263428,
+      "sft_loss": 4.399357795715332,
+      "step": 1060
+    },
+    {
+      "epoch": 0.5699949824385349,
+      "grad_norm": 0.749882232299179,
+      "learning_rate": 9.755571256763764e-07,
+      "logits/chosen": -0.4936675429344177,
+      "logits/rejected": -0.32630711793899536,
+      "logps/chosen": -4.695061683654785,
+      "logps/rejected": -5.094540596008301,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.695061683654785,
+      "rewards/margins": 0.3994784355163574,
+      "rewards/rejected": -5.094540596008301,
+      "sft_loss": 4.374762535095215,
+      "step": 1065
+    },
+    {
+      "epoch": 0.5726710152199365,
+      "grad_norm": 0.4335609502427654,
+      "learning_rate": 9.750738324585097e-07,
+      "logits/chosen": -0.5365744233131409,
+      "logits/rejected": -0.21823947131633759,
+      "logps/chosen": -4.555490970611572,
+      "logps/rejected": -4.938295364379883,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.555490970611572,
+      "rewards/margins": 0.3828045725822449,
+      "rewards/rejected": -4.938295364379883,
+      "sft_loss": 4.294363975524902,
+      "step": 1070
+    },
+    {
+      "epoch": 0.5753470480013381,
+      "grad_norm": 0.5841385822714422,
+      "learning_rate": 9.74585930072237e-07,
+      "logits/chosen": -0.5023024082183838,
+      "logits/rejected": -0.32627633213996887,
+      "logps/chosen": -4.687676906585693,
+      "logps/rejected": -5.107883453369141,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.687676906585693,
+      "rewards/margins": 0.42020702362060547,
+      "rewards/rejected": -5.107883453369141,
+      "sft_loss": 4.394619464874268,
+      "step": 1075
+    },
+    {
+      "epoch": 0.5780230807827396,
+      "grad_norm": 0.3573018957478717,
+      "learning_rate": 9.740934232511892e-07,
+      "logits/chosen": -0.6413004398345947,
+      "logits/rejected": -0.5170631408691406,
+      "logps/chosen": -4.733739376068115,
+      "logps/rejected": -4.969666957855225,
+      "loss": 0.0544,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.733739376068115,
+      "rewards/margins": 0.23592762649059296,
+      "rewards/rejected": -4.969666957855225,
+      "sft_loss": 4.418457984924316,
+      "step": 1080
+    },
+    {
+      "epoch": 0.5806991135641412,
+      "grad_norm": 0.9387854678624943,
+      "learning_rate": 9.735963167736698e-07,
+      "logits/chosen": -0.5804794430732727,
+      "logits/rejected": -0.3907211422920227,
+      "logps/chosen": -4.788337707519531,
+      "logps/rejected": -5.068944454193115,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.788337707519531,
+      "rewards/margins": 0.28060680627822876,
+      "rewards/rejected": -5.068944454193115,
+      "sft_loss": 4.502654075622559,
+      "step": 1085
+    },
+    {
+      "epoch": 0.5833751463455428,
+      "grad_norm": 0.574041624870192,
+      "learning_rate": 9.730946154626078e-07,
+      "logits/chosen": -0.5329641699790955,
+      "logits/rejected": -0.40476202964782715,
+      "logps/chosen": -4.6259355545043945,
+      "logps/rejected": -4.907242774963379,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.6259355545043945,
+      "rewards/margins": 0.2813071608543396,
+      "rewards/rejected": -4.907242774963379,
+      "sft_loss": 4.300456523895264,
+      "step": 1090
+    },
+    {
+      "epoch": 0.5860511791269443,
+      "grad_norm": 0.7365756470251096,
+      "learning_rate": 9.725883241855117e-07,
+      "logits/chosen": -0.7071422338485718,
+      "logits/rejected": -0.5051488876342773,
+      "logps/chosen": -4.796841621398926,
+      "logps/rejected": -5.196400165557861,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.796841621398926,
+      "rewards/margins": 0.39955854415893555,
+      "rewards/rejected": -5.196400165557861,
+      "sft_loss": 4.557826995849609,
+      "step": 1095
+    },
+    {
+      "epoch": 0.5887272119083459,
+      "grad_norm": 0.5917366709488491,
+      "learning_rate": 9.720774478544218e-07,
+      "logits/chosen": -0.4832594394683838,
+      "logits/rejected": -0.29662543535232544,
+      "logps/chosen": -4.595888614654541,
+      "logps/rejected": -4.967526435852051,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.595888614654541,
+      "rewards/margins": 0.3716380000114441,
+      "rewards/rejected": -4.967526435852051,
+      "sft_loss": 4.249319553375244,
+      "step": 1100
+    },
+    {
+      "epoch": 0.5914032446897475,
+      "grad_norm": 0.3847051965590612,
+      "learning_rate": 9.715619914258624e-07,
+      "logits/chosen": -0.6779504418373108,
+      "logits/rejected": -0.5655364394187927,
+      "logps/chosen": -4.718003273010254,
+      "logps/rejected": -4.955427646636963,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.718003273010254,
+      "rewards/margins": 0.2374243438243866,
+      "rewards/rejected": -4.955427646636963,
+      "sft_loss": 4.384034156799316,
+      "step": 1105
+    },
+    {
+      "epoch": 0.594079277471149,
+      "grad_norm": 0.543269055034267,
+      "learning_rate": 9.710419599007937e-07,
+      "logits/chosen": -0.592439591884613,
+      "logits/rejected": -0.3935183882713318,
+      "logps/chosen": -4.704367637634277,
+      "logps/rejected": -4.9490461349487305,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.704367637634277,
+      "rewards/margins": 0.24467873573303223,
+      "rewards/rejected": -4.9490461349487305,
+      "sft_loss": 4.478358745574951,
+      "step": 1110
+    },
+    {
+      "epoch": 0.5967553102525506,
+      "grad_norm": 0.40955078166194536,
+      "learning_rate": 9.705173583245643e-07,
+      "logits/chosen": -0.6035395860671997,
+      "logits/rejected": -0.35157328844070435,
+      "logps/chosen": -4.598028659820557,
+      "logps/rejected": -4.925137042999268,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.598028659820557,
+      "rewards/margins": 0.32710808515548706,
+      "rewards/rejected": -4.925137042999268,
+      "sft_loss": 4.3422746658325195,
+      "step": 1115
+    },
+    {
+      "epoch": 0.5994313430339522,
+      "grad_norm": 0.2948789244920242,
+      "learning_rate": 9.699881917868609e-07,
+      "logits/chosen": -0.6719862222671509,
+      "logits/rejected": -0.5192317962646484,
+      "logps/chosen": -4.481719970703125,
+      "logps/rejected": -4.777043342590332,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.481719970703125,
+      "rewards/margins": 0.29532328248023987,
+      "rewards/rejected": -4.777043342590332,
+      "sft_loss": 4.248114585876465,
+      "step": 1120
+    },
+    {
+      "epoch": 0.6021073758153538,
+      "grad_norm": 0.8245398173478702,
+      "learning_rate": 9.694544654216594e-07,
+      "logits/chosen": -0.6878781318664551,
+      "logits/rejected": -0.4318612515926361,
+      "logps/chosen": -4.79897403717041,
+      "logps/rejected": -5.210911750793457,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.79897403717041,
+      "rewards/margins": 0.4119381308555603,
+      "rewards/rejected": -5.210911750793457,
+      "sft_loss": 4.552307605743408,
+      "step": 1125
+    },
+    {
+      "epoch": 0.6047834085967553,
+      "grad_norm": 1.14084627054799,
+      "learning_rate": 9.689161844071755e-07,
+      "logits/chosen": -0.3799501657485962,
+      "logits/rejected": -0.2871240973472595,
+      "logps/chosen": -4.454643726348877,
+      "logps/rejected": -4.7599263191223145,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.454643726348877,
+      "rewards/margins": 0.3052830696105957,
+      "rewards/rejected": -4.7599263191223145,
+      "sft_loss": 4.177380084991455,
+      "step": 1130
+    },
+    {
+      "epoch": 0.6074594413781569,
+      "grad_norm": 0.8660037391087473,
+      "learning_rate": 9.683733539658138e-07,
+      "logits/chosen": -0.5278415083885193,
+      "logits/rejected": -0.268408864736557,
+      "logps/chosen": -4.686205863952637,
+      "logps/rejected": -5.040558815002441,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.686205863952637,
+      "rewards/margins": 0.3543532192707062,
+      "rewards/rejected": -5.040558815002441,
+      "sft_loss": 4.330956935882568,
+      "step": 1135
+    },
+    {
+      "epoch": 0.6101354741595585,
+      "grad_norm": 0.4970819248895511,
+      "learning_rate": 9.678259793641178e-07,
+      "logits/chosen": -0.48409318923950195,
+      "logits/rejected": -0.44155415892601013,
+      "logps/chosen": -4.872367858886719,
+      "logps/rejected": -5.101282119750977,
+      "loss": 0.0547,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -4.872367858886719,
+      "rewards/margins": 0.22891390323638916,
+      "rewards/rejected": -5.101282119750977,
+      "sft_loss": 4.596518039703369,
+      "step": 1140
+    },
+    {
+      "epoch": 0.61281150694096,
+      "grad_norm": 0.5812923017369568,
+      "learning_rate": 9.672740659127183e-07,
+      "logits/chosen": -0.5782763361930847,
+      "logits/rejected": -0.40878796577453613,
+      "logps/chosen": -4.521733283996582,
+      "logps/rejected": -4.903876304626465,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.521733283996582,
+      "rewards/margins": 0.382142573595047,
+      "rewards/rejected": -4.903876304626465,
+      "sft_loss": 4.200924396514893,
+      "step": 1145
+    },
+    {
+      "epoch": 0.6154875397223616,
+      "grad_norm": 0.44409718097577794,
+      "learning_rate": 9.667176189662818e-07,
+      "logits/chosen": -0.5531161427497864,
+      "logits/rejected": -0.3877137005329132,
+      "logps/chosen": -4.67879581451416,
+      "logps/rejected": -4.9793853759765625,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.67879581451416,
+      "rewards/margins": 0.3005899488925934,
+      "rewards/rejected": -4.9793853759765625,
+      "sft_loss": 4.385974407196045,
+      "step": 1150
+    },
+    {
+      "epoch": 0.6181635725037632,
+      "grad_norm": 0.5761368695976793,
+      "learning_rate": 9.661566439234592e-07,
+      "logits/chosen": -0.4771701395511627,
+      "logits/rejected": -0.3725683093070984,
+      "logps/chosen": -4.723606586456299,
+      "logps/rejected": -4.984592914581299,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.723606586456299,
+      "rewards/margins": 0.26098623871803284,
+      "rewards/rejected": -4.984592914581299,
+      "sft_loss": 4.454628944396973,
+      "step": 1155
+    },
+    {
+      "epoch": 0.6208396052851648,
+      "grad_norm": 0.5556713770938857,
+      "learning_rate": 9.655911462268327e-07,
+      "logits/chosen": -0.4481441378593445,
+      "logits/rejected": -0.30789104104042053,
+      "logps/chosen": -4.546295166015625,
+      "logps/rejected": -4.988908767700195,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.546295166015625,
+      "rewards/margins": 0.44261303544044495,
+      "rewards/rejected": -4.988908767700195,
+      "sft_loss": 4.256467819213867,
+      "step": 1160
+    },
+    {
+      "epoch": 0.6235156380665663,
+      "grad_norm": 0.4875769722274501,
+      "learning_rate": 9.650211313628636e-07,
+      "logits/chosen": -0.5960179567337036,
+      "logits/rejected": -0.4614837169647217,
+      "logps/chosen": -4.707175254821777,
+      "logps/rejected": -4.891497611999512,
+      "loss": 0.0553,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -4.707175254821777,
+      "rewards/margins": 0.18432240188121796,
+      "rewards/rejected": -4.891497611999512,
+      "sft_loss": 4.4733428955078125,
+      "step": 1165
+    },
+    {
+      "epoch": 0.6261916708479679,
+      "grad_norm": 0.4014279228736453,
+      "learning_rate": 9.644466048618386e-07,
+      "logits/chosen": -0.6922046542167664,
+      "logits/rejected": -0.4939608573913574,
+      "logps/chosen": -5.006098747253418,
+      "logps/rejected": -5.234212398529053,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -5.006098747253418,
+      "rewards/margins": 0.22811400890350342,
+      "rewards/rejected": -5.234212398529053,
+      "sft_loss": 4.620914459228516,
+      "step": 1170
+    },
+    {
+      "epoch": 0.6288677036293695,
+      "grad_norm": 0.8535156242979435,
+      "learning_rate": 9.63867572297816e-07,
+      "logits/chosen": -0.705421507358551,
+      "logits/rejected": -0.4823324680328369,
+      "logps/chosen": -4.5951361656188965,
+      "logps/rejected": -4.936031818389893,
+      "loss": 0.0543,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.5951361656188965,
+      "rewards/margins": 0.34089553356170654,
+      "rewards/rejected": -4.936031818389893,
+      "sft_loss": 4.360302925109863,
+      "step": 1175
+    },
+    {
+      "epoch": 0.631543736410771,
+      "grad_norm": 0.30103834357953974,
+      "learning_rate": 9.632840392885727e-07,
+      "logits/chosen": -0.7017086744308472,
+      "logits/rejected": -0.4853813648223877,
+      "logps/chosen": -4.485077857971191,
+      "logps/rejected": -4.886071681976318,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.485077857971191,
+      "rewards/margins": 0.4009944498538971,
+      "rewards/rejected": -4.886071681976318,
+      "sft_loss": 4.283857822418213,
+      "step": 1180
+    },
+    {
+      "epoch": 0.6342197691921726,
+      "grad_norm": 0.4847167733676787,
+      "learning_rate": 9.626960114955483e-07,
+      "logits/chosen": -0.6630807518959045,
+      "logits/rejected": -0.46446362137794495,
+      "logps/chosen": -4.6696085929870605,
+      "logps/rejected": -5.147528648376465,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.6696085929870605,
+      "rewards/margins": 0.47792062163352966,
+      "rewards/rejected": -5.147528648376465,
+      "sft_loss": 4.391916751861572,
+      "step": 1185
+    },
+    {
+      "epoch": 0.6368958019735742,
+      "grad_norm": 0.6465226432916064,
+      "learning_rate": 9.621034946237909e-07,
+      "logits/chosen": -0.7046906352043152,
+      "logits/rejected": -0.514801025390625,
+      "logps/chosen": -4.797091007232666,
+      "logps/rejected": -5.1934494972229,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.797091007232666,
+      "rewards/margins": 0.3963584899902344,
+      "rewards/rejected": -5.1934494972229,
+      "sft_loss": 4.548532485961914,
+      "step": 1190
+    },
+    {
+      "epoch": 0.6395718347549757,
+      "grad_norm": 0.6053610691894898,
+      "learning_rate": 9.615064944219021e-07,
+      "logits/chosen": -0.5024127960205078,
+      "logits/rejected": -0.33732375502586365,
+      "logps/chosen": -4.484908103942871,
+      "logps/rejected": -4.776598930358887,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.484908103942871,
+      "rewards/margins": 0.2916913628578186,
+      "rewards/rejected": -4.776598930358887,
+      "sft_loss": 4.1641082763671875,
+      "step": 1195
+    },
+    {
+      "epoch": 0.6422478675363773,
+      "grad_norm": 1.0983117178191424,
+      "learning_rate": 9.609050166819803e-07,
+      "logits/chosen": -0.49139395356178284,
+      "logits/rejected": -0.4269639551639557,
+      "logps/chosen": -4.553410530090332,
+      "logps/rejected": -4.853245735168457,
+      "loss": 0.0545,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.553410530090332,
+      "rewards/margins": 0.29983487725257874,
+      "rewards/rejected": -4.853245735168457,
+      "sft_loss": 4.232519149780273,
+      "step": 1200
+    },
+    {
+      "epoch": 0.6422478675363773,
+      "eval_logits/chosen": -0.025698307901620865,
+      "eval_logits/rejected": 0.09142318367958069,
+      "eval_logps/chosen": -4.657962799072266,
+      "eval_logps/rejected": -5.0485663414001465,
+      "eval_loss": 0.05232247710227966,
+      "eval_rewards/accuracies": 0.6350148320198059,
+      "eval_rewards/chosen": -4.657962799072266,
+      "eval_rewards/margins": 0.390603631734848,
+      "eval_rewards/rejected": -5.0485663414001465,
+      "eval_runtime": 43.3819,
+      "eval_samples_per_second": 31.004,
+      "eval_sft_loss": 4.285591125488281,
+      "eval_steps_per_second": 7.768,
+      "step": 1200
+    },
+    {
+      "epoch": 0.6449239003177789,
+      "grad_norm": 0.6598117160164223,
+      "learning_rate": 9.602990672395653e-07,
+      "logits/chosen": -0.6095829010009766,
+      "logits/rejected": -0.38074302673339844,
+      "logps/chosen": -4.750426292419434,
+      "logps/rejected": -5.1771368980407715,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.750426292419434,
+      "rewards/margins": 0.42671045660972595,
+      "rewards/rejected": -5.1771368980407715,
+      "sft_loss": 4.473883628845215,
+      "step": 1205
+    },
+    {
+      "epoch": 0.6475999330991805,
+      "grad_norm": 0.5694728776522204,
+      "learning_rate": 9.59688651973581e-07,
+      "logits/chosen": -0.5579411387443542,
+      "logits/rejected": -0.27562081813812256,
+      "logps/chosen": -4.6332831382751465,
+      "logps/rejected": -5.050353050231934,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.6332831382751465,
+      "rewards/margins": 0.4170694351196289,
+      "rewards/rejected": -5.050353050231934,
+      "sft_loss": 4.405447959899902,
+      "step": 1210
+    },
+    {
+      "epoch": 0.650275965880582,
+      "grad_norm": 1.128531677554023,
+      "learning_rate": 9.590737768062792e-07,
+      "logits/chosen": -0.5781430602073669,
+      "logits/rejected": -0.4014016091823578,
+      "logps/chosen": -4.605533599853516,
+      "logps/rejected": -4.950179100036621,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.605533599853516,
+      "rewards/margins": 0.34464550018310547,
+      "rewards/rejected": -4.950179100036621,
+      "sft_loss": 4.295781135559082,
+      "step": 1215
+    },
+    {
+      "epoch": 0.6529519986619836,
+      "grad_norm": 0.44594614543377137,
+      "learning_rate": 9.584544477031816e-07,
+      "logits/chosen": -0.3928416967391968,
+      "logits/rejected": -0.23103955388069153,
+      "logps/chosen": -4.714247703552246,
+      "logps/rejected": -5.023430347442627,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.714247703552246,
+      "rewards/margins": 0.30918246507644653,
+      "rewards/rejected": -5.023430347442627,
+      "sft_loss": 4.452761650085449,
+      "step": 1220
+    },
+    {
+      "epoch": 0.6556280314433852,
+      "grad_norm": 0.5839455517038359,
+      "learning_rate": 9.578306706730215e-07,
+      "logits/chosen": -0.6410695314407349,
+      "logits/rejected": -0.3730124831199646,
+      "logps/chosen": -4.631039619445801,
+      "logps/rejected": -4.9336748123168945,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.631039619445801,
+      "rewards/margins": 0.3026350438594818,
+      "rewards/rejected": -4.9336748123168945,
+      "sft_loss": 4.3163886070251465,
+      "step": 1225
+    },
+    {
+      "epoch": 0.6583040642247867,
+      "grad_norm": 0.643742164156604,
+      "learning_rate": 9.572024517676865e-07,
+      "logits/chosen": -0.5921844840049744,
+      "logits/rejected": -0.46048134565353394,
+      "logps/chosen": -4.5794901847839355,
+      "logps/rejected": -4.9018235206604,
+      "loss": 0.055,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.5794901847839355,
+      "rewards/margins": 0.3223329186439514,
+      "rewards/rejected": -4.9018235206604,
+      "sft_loss": 4.405655860900879,
+      "step": 1230
+    },
+    {
+      "epoch": 0.6609800970061883,
+      "grad_norm": 0.45905634809888224,
+      "learning_rate": 9.565697970821593e-07,
+      "logits/chosen": -0.6735567450523376,
+      "logits/rejected": -0.4286714196205139,
+      "logps/chosen": -4.814480781555176,
+      "logps/rejected": -5.108050346374512,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.814480781555176,
+      "rewards/margins": 0.2935686707496643,
+      "rewards/rejected": -5.108050346374512,
+      "sft_loss": 4.5338850021362305,
+      "step": 1235
+    },
+    {
+      "epoch": 0.6636561297875899,
+      "grad_norm": 0.3718124611962775,
+      "learning_rate": 9.559327127544585e-07,
+      "logits/chosen": -0.7341528534889221,
+      "logits/rejected": -0.5741795897483826,
+      "logps/chosen": -4.655917167663574,
+      "logps/rejected": -4.946690559387207,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.655917167663574,
+      "rewards/margins": 0.2907727062702179,
+      "rewards/rejected": -4.946690559387207,
+      "sft_loss": 4.393622398376465,
+      "step": 1240
+    },
+    {
+      "epoch": 0.6663321625689914,
+      "grad_norm": 0.47313857382521357,
+      "learning_rate": 9.552912049655789e-07,
+      "logits/chosen": -0.5752710103988647,
+      "logits/rejected": -0.3442533612251282,
+      "logps/chosen": -4.453244209289551,
+      "logps/rejected": -4.790841102600098,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.453244209289551,
+      "rewards/margins": 0.3375973701477051,
+      "rewards/rejected": -4.790841102600098,
+      "sft_loss": 4.221789836883545,
+      "step": 1245
+    },
+    {
+      "epoch": 0.669008195350393,
+      "grad_norm": 0.3080289767371704,
+      "learning_rate": 9.546452799394315e-07,
+      "logits/chosen": -0.6375377178192139,
+      "logits/rejected": -0.35768041014671326,
+      "logps/chosen": -4.7251386642456055,
+      "logps/rejected": -5.06402063369751,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.7251386642456055,
+      "rewards/margins": 0.3388821482658386,
+      "rewards/rejected": -5.06402063369751,
+      "sft_loss": 4.505545616149902,
+      "step": 1250
+    },
+    {
+      "epoch": 0.6716842281317946,
+      "grad_norm": 0.46157795630326126,
+      "learning_rate": 9.539949439427846e-07,
+      "logits/chosen": -0.5481323003768921,
+      "logits/rejected": -0.40522176027297974,
+      "logps/chosen": -4.713761329650879,
+      "logps/rejected": -5.103206157684326,
+      "loss": 0.0543,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -4.713761329650879,
+      "rewards/margins": 0.38944488763809204,
+      "rewards/rejected": -5.103206157684326,
+      "sft_loss": 4.457167148590088,
+      "step": 1255
+    },
+    {
+      "epoch": 0.6743602609131962,
+      "grad_norm": 0.7554972971844888,
+      "learning_rate": 9.533402032852002e-07,
+      "logits/chosen": -0.5324119329452515,
+      "logits/rejected": -0.3285521864891052,
+      "logps/chosen": -4.660325050354004,
+      "logps/rejected": -5.11639404296875,
+      "loss": 0.053,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.660325050354004,
+      "rewards/margins": 0.45606860518455505,
+      "rewards/rejected": -5.11639404296875,
+      "sft_loss": 4.358373641967773,
+      "step": 1260
+    },
+    {
+      "epoch": 0.6770362936945977,
+      "grad_norm": 0.5334990626640641,
+      "learning_rate": 9.526810643189754e-07,
+      "logits/chosen": -0.4682994484901428,
+      "logits/rejected": -0.20002928376197815,
+      "logps/chosen": -4.4835896492004395,
+      "logps/rejected": -4.984886646270752,
+      "loss": 0.053,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.4835896492004395,
+      "rewards/margins": 0.501296877861023,
+      "rewards/rejected": -4.984886646270752,
+      "sft_loss": 4.2979960441589355,
+      "step": 1265
+    },
+    {
+      "epoch": 0.6797123264759993,
+      "grad_norm": 0.43311466275245447,
+      "learning_rate": 9.52017533439079e-07,
+      "logits/chosen": -0.6111350655555725,
+      "logits/rejected": -0.5189183950424194,
+      "logps/chosen": -4.642319679260254,
+      "logps/rejected": -4.935871601104736,
+      "loss": 0.055,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.642319679260254,
+      "rewards/margins": 0.29355183243751526,
+      "rewards/rejected": -4.935871601104736,
+      "sft_loss": 4.443874359130859,
+      "step": 1270
+    },
+    {
+      "epoch": 0.6823883592574009,
+      "grad_norm": 0.4523237714480055,
+      "learning_rate": 9.513496170830909e-07,
+      "logits/chosen": -0.7188066244125366,
+      "logits/rejected": -0.6016985177993774,
+      "logps/chosen": -4.738122463226318,
+      "logps/rejected": -4.9969658851623535,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.738122463226318,
+      "rewards/margins": 0.25884348154067993,
+      "rewards/rejected": -4.9969658851623535,
+      "sft_loss": 4.435435771942139,
+      "step": 1275
+    },
+    {
+      "epoch": 0.6850643920388024,
+      "grad_norm": 0.39885392574273837,
+      "learning_rate": 9.506773217311382e-07,
+      "logits/chosen": -0.6342407464981079,
+      "logits/rejected": -0.4105973243713379,
+      "logps/chosen": -4.633099555969238,
+      "logps/rejected": -4.917786121368408,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.633099555969238,
+      "rewards/margins": 0.2846868932247162,
+      "rewards/rejected": -4.917786121368408,
+      "sft_loss": 4.4025044441223145,
+      "step": 1280
+    },
+    {
+      "epoch": 0.687740424820204,
+      "grad_norm": 0.46859664979499294,
+      "learning_rate": 9.500006539058334e-07,
+      "logits/chosen": -0.5705159902572632,
+      "logits/rejected": -0.32873377203941345,
+      "logps/chosen": -4.4862871170043945,
+      "logps/rejected": -4.728259086608887,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.4862871170043945,
+      "rewards/margins": 0.24197213351726532,
+      "rewards/rejected": -4.728259086608887,
+      "sft_loss": 4.215309143066406,
+      "step": 1285
+    },
+    {
+      "epoch": 0.6904164576016056,
+      "grad_norm": 0.5566937884089165,
+      "learning_rate": 9.493196201722109e-07,
+      "logits/chosen": -0.6300413012504578,
+      "logits/rejected": -0.4329058527946472,
+      "logps/chosen": -4.681478500366211,
+      "logps/rejected": -4.944789886474609,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.681478500366211,
+      "rewards/margins": 0.26331159472465515,
+      "rewards/rejected": -4.944789886474609,
+      "sft_loss": 4.417520999908447,
+      "step": 1290
+    },
+    {
+      "epoch": 0.6930924903830072,
+      "grad_norm": 0.5005083892393065,
+      "learning_rate": 9.486342271376628e-07,
+      "logits/chosen": -0.5651583671569824,
+      "logits/rejected": -0.6070525050163269,
+      "logps/chosen": -4.731893539428711,
+      "logps/rejected": -5.015305519104004,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.731893539428711,
+      "rewards/margins": 0.28341221809387207,
+      "rewards/rejected": -5.015305519104004,
+      "sft_loss": 4.412493705749512,
+      "step": 1295
+    },
+    {
+      "epoch": 0.6957685231644087,
+      "grad_norm": 0.4047781839205759,
+      "learning_rate": 9.479444814518755e-07,
+      "logits/chosen": -0.5466286540031433,
+      "logits/rejected": -0.2215103805065155,
+      "logps/chosen": -4.484042644500732,
+      "logps/rejected": -5.004049301147461,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.484042644500732,
+      "rewards/margins": 0.5200066566467285,
+      "rewards/rejected": -5.004049301147461,
+      "sft_loss": 4.189947605133057,
+      "step": 1300
+    },
+    {
+      "epoch": 0.6984445559458103,
+      "grad_norm": 0.4919221601673572,
+      "learning_rate": 9.472503898067645e-07,
+      "logits/chosen": -0.3866714835166931,
+      "logits/rejected": -0.3418964743614197,
+      "logps/chosen": -4.764806747436523,
+      "logps/rejected": -4.945952892303467,
+      "loss": 0.0548,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -4.764806747436523,
+      "rewards/margins": 0.18114568293094635,
+      "rewards/rejected": -4.945952892303467,
+      "sft_loss": 4.389309883117676,
+      "step": 1305
+    },
+    {
+      "epoch": 0.701120588727212,
+      "grad_norm": 0.6165938817684519,
+      "learning_rate": 9.465519589364099e-07,
+      "logits/chosen": -0.43105238676071167,
+      "logits/rejected": -0.34996071457862854,
+      "logps/chosen": -4.986048698425293,
+      "logps/rejected": -5.196448802947998,
+      "loss": 0.0539,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -4.986048698425293,
+      "rewards/margins": 0.2104000300168991,
+      "rewards/rejected": -5.196448802947998,
+      "sft_loss": 4.601518154144287,
+      "step": 1310
+    },
+    {
+      "epoch": 0.7037966215086134,
+      "grad_norm": 0.7147540339684912,
+      "learning_rate": 9.458491956169914e-07,
+      "logits/chosen": -0.5200138688087463,
+      "logits/rejected": -0.30936622619628906,
+      "logps/chosen": -4.600648403167725,
+      "logps/rejected": -4.895071983337402,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.600648403167725,
+      "rewards/margins": 0.29442328214645386,
+      "rewards/rejected": -4.895071983337402,
+      "sft_loss": 4.186682224273682,
+      "step": 1315
+    },
+    {
+      "epoch": 0.706472654290015,
+      "grad_norm": 0.5222909906022749,
+      "learning_rate": 9.451421066667215e-07,
+      "logits/chosen": -0.6883410215377808,
+      "logits/rejected": -0.45219460129737854,
+      "logps/chosen": -4.589325904846191,
+      "logps/rejected": -5.036540508270264,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.589325904846191,
+      "rewards/margins": 0.4472144544124603,
+      "rewards/rejected": -5.036540508270264,
+      "sft_loss": 4.3194098472595215,
+      "step": 1320
+    },
+    {
+      "epoch": 0.7091486870714167,
+      "grad_norm": 0.4776830565945863,
+      "learning_rate": 9.444306989457805e-07,
+      "logits/chosen": -0.46004819869995117,
+      "logits/rejected": -0.3241944909095764,
+      "logps/chosen": -4.868544101715088,
+      "logps/rejected": -5.153488636016846,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -4.868544101715088,
+      "rewards/margins": 0.2849445641040802,
+      "rewards/rejected": -5.153488636016846,
+      "sft_loss": 4.496047019958496,
+      "step": 1325
+    },
+    {
+      "epoch": 0.7118247198528181,
+      "grad_norm": 0.4294909802110886,
+      "learning_rate": 9.437149793562489e-07,
+      "logits/chosen": -0.5180003046989441,
+      "logits/rejected": -0.3660891652107239,
+      "logps/chosen": -4.651247501373291,
+      "logps/rejected": -4.924403190612793,
+      "loss": 0.054,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.651247501373291,
+      "rewards/margins": 0.2731553614139557,
+      "rewards/rejected": -4.924403190612793,
+      "sft_loss": 4.393563270568848,
+      "step": 1330
+    },
+    {
+      "epoch": 0.7145007526342197,
+      "grad_norm": 0.39095469809878836,
+      "learning_rate": 9.429949548420417e-07,
+      "logits/chosen": -0.5731022357940674,
+      "logits/rejected": -0.4370029866695404,
+      "logps/chosen": -4.695939064025879,
+      "logps/rejected": -4.961706638336182,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.695939064025879,
+      "rewards/margins": 0.2657679617404938,
+      "rewards/rejected": -4.961706638336182,
+      "sft_loss": 4.3769636154174805,
+      "step": 1335
+    },
+    {
+      "epoch": 0.7171767854156214,
+      "grad_norm": 0.430223860508395,
+      "learning_rate": 9.422706323888396e-07,
+      "logits/chosen": -0.5357168912887573,
+      "logits/rejected": -0.49134665727615356,
+      "logps/chosen": -4.6197052001953125,
+      "logps/rejected": -4.928733825683594,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.6197052001953125,
+      "rewards/margins": 0.3090288043022156,
+      "rewards/rejected": -4.928733825683594,
+      "sft_loss": 4.343020439147949,
+      "step": 1340
+    },
+    {
+      "epoch": 0.719852818197023,
+      "grad_norm": 0.5805561587916378,
+      "learning_rate": 9.415420190240225e-07,
+      "logits/chosen": -0.5039713382720947,
+      "logits/rejected": -0.23104509711265564,
+      "logps/chosen": -4.593451976776123,
+      "logps/rejected": -5.125218868255615,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.593451976776123,
+      "rewards/margins": 0.5317668914794922,
+      "rewards/rejected": -5.125218868255615,
+      "sft_loss": 4.35361385345459,
+      "step": 1345
+    },
+    {
+      "epoch": 0.7225288509784245,
+      "grad_norm": 0.794948230843,
+      "learning_rate": 9.408091218166002e-07,
+      "logits/chosen": -0.4368966519832611,
+      "logits/rejected": -0.349046915769577,
+      "logps/chosen": -4.646870136260986,
+      "logps/rejected": -4.846858024597168,
+      "loss": 0.0545,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.646870136260986,
+      "rewards/margins": 0.19998829066753387,
+      "rewards/rejected": -4.846858024597168,
+      "sft_loss": 4.331336498260498,
+      "step": 1350
+    },
+    {
+      "epoch": 0.7252048837598261,
+      "grad_norm": 0.4322649148913923,
+      "learning_rate": 9.400719478771449e-07,
+      "logits/chosen": -0.4928915500640869,
+      "logits/rejected": -0.10693871974945068,
+      "logps/chosen": -4.772175312042236,
+      "logps/rejected": -5.275957107543945,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.772175312042236,
+      "rewards/margins": 0.5037820935249329,
+      "rewards/rejected": -5.275957107543945,
+      "sft_loss": 4.457993507385254,
+      "step": 1355
+    },
+    {
+      "epoch": 0.7278809165412277,
+      "grad_norm": 0.4188247735094768,
+      "learning_rate": 9.393305043577209e-07,
+      "logits/chosen": -0.4668586850166321,
+      "logits/rejected": -0.3280408978462219,
+      "logps/chosen": -4.57827615737915,
+      "logps/rejected": -4.987173080444336,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.57827615737915,
+      "rewards/margins": 0.40889644622802734,
+      "rewards/rejected": -4.987173080444336,
+      "sft_loss": 4.273009300231934,
+      "step": 1360
+    },
+    {
+      "epoch": 0.7305569493226292,
+      "grad_norm": 0.43986121858771965,
+      "learning_rate": 9.38584798451817e-07,
+      "logits/chosen": -0.5263036489486694,
+      "logits/rejected": -0.32945355772972107,
+      "logps/chosen": -4.534844398498535,
+      "logps/rejected": -4.883469581604004,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.534844398498535,
+      "rewards/margins": 0.3486255705356598,
+      "rewards/rejected": -4.883469581604004,
+      "sft_loss": 4.275891304016113,
+      "step": 1365
+    },
+    {
+      "epoch": 0.7332329821040308,
+      "grad_norm": 0.5027182729571974,
+      "learning_rate": 9.37834837394275e-07,
+      "logits/chosen": -0.41481298208236694,
+      "logits/rejected": -0.24069428443908691,
+      "logps/chosen": -4.659480094909668,
+      "logps/rejected": -5.198307514190674,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.659480094909668,
+      "rewards/margins": 0.5388270616531372,
+      "rewards/rejected": -5.198307514190674,
+      "sft_loss": 4.420654296875,
+      "step": 1370
+    },
+    {
+      "epoch": 0.7359090148854324,
+      "grad_norm": 0.5596253221586619,
+      "learning_rate": 9.370806284612203e-07,
+      "logits/chosen": -0.4726240038871765,
+      "logits/rejected": -0.2989794909954071,
+      "logps/chosen": -4.519133567810059,
+      "logps/rejected": -5.032240867614746,
+      "loss": 0.053,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.519133567810059,
+      "rewards/margins": 0.5131076574325562,
+      "rewards/rejected": -5.032240867614746,
+      "sft_loss": 4.242257118225098,
+      "step": 1375
+    },
+    {
+      "epoch": 0.738585047666834,
+      "grad_norm": 0.639067801858824,
+      "learning_rate": 9.363221789699912e-07,
+      "logits/chosen": -0.5229755640029907,
+      "logits/rejected": -0.3458749055862427,
+      "logps/chosen": -4.645009517669678,
+      "logps/rejected": -5.047579765319824,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.645009517669678,
+      "rewards/margins": 0.4025697112083435,
+      "rewards/rejected": -5.047579765319824,
+      "sft_loss": 4.33743953704834,
+      "step": 1380
+    },
+    {
+      "epoch": 0.7412610804482355,
+      "grad_norm": 0.45340914139326294,
+      "learning_rate": 9.355594962790682e-07,
+      "logits/chosen": -0.5707454681396484,
+      "logits/rejected": -0.39737915992736816,
+      "logps/chosen": -4.724990367889404,
+      "logps/rejected": -5.140081405639648,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.724990367889404,
+      "rewards/margins": 0.4150908589363098,
+      "rewards/rejected": -5.140081405639648,
+      "sft_loss": 4.456721305847168,
+      "step": 1385
+    },
+    {
+      "epoch": 0.7439371132296371,
+      "grad_norm": 0.7726334686248777,
+      "learning_rate": 9.34792587788002e-07,
+      "logits/chosen": -0.41615360975265503,
+      "logits/rejected": -0.25168564915657043,
+      "logps/chosen": -4.589061260223389,
+      "logps/rejected": -4.944598197937012,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.589061260223389,
+      "rewards/margins": 0.3555375933647156,
+      "rewards/rejected": -4.944598197937012,
+      "sft_loss": 4.252989292144775,
+      "step": 1390
+    },
+    {
+      "epoch": 0.7466131460110387,
+      "grad_norm": 0.9713420234440161,
+      "learning_rate": 9.34021460937342e-07,
+      "logits/chosen": -0.54308021068573,
+      "logits/rejected": -0.46706587076187134,
+      "logps/chosen": -4.68400764465332,
+      "logps/rejected": -4.97832727432251,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.68400764465332,
+      "rewards/margins": 0.2943192422389984,
+      "rewards/rejected": -4.97832727432251,
+      "sft_loss": 4.438172817230225,
+      "step": 1395
+    },
+    {
+      "epoch": 0.7492891787924402,
+      "grad_norm": 0.7646700933687028,
+      "learning_rate": 9.332461232085646e-07,
+      "logits/chosen": -0.7934912443161011,
+      "logits/rejected": -0.5852801203727722,
+      "logps/chosen": -4.717600345611572,
+      "logps/rejected": -4.994270324707031,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.717600345611572,
+      "rewards/margins": 0.27667081356048584,
+      "rewards/rejected": -4.994270324707031,
+      "sft_loss": 4.516864776611328,
+      "step": 1400
+    },
+    {
+      "epoch": 0.7519652115738418,
+      "grad_norm": 1.1584729658858994,
+      "learning_rate": 9.324665821239998e-07,
+      "logits/chosen": -0.7201007604598999,
+      "logits/rejected": -0.4515800476074219,
+      "logps/chosen": -4.434556484222412,
+      "logps/rejected": -4.892245292663574,
+      "loss": 0.0539,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.434556484222412,
+      "rewards/margins": 0.45768898725509644,
+      "rewards/rejected": -4.892245292663574,
+      "sft_loss": 4.247842788696289,
+      "step": 1405
+    },
+    {
+      "epoch": 0.7546412443552434,
+      "grad_norm": 0.6140739611213539,
+      "learning_rate": 9.316828452467583e-07,
+      "logits/chosen": -0.6372144222259521,
+      "logits/rejected": -0.42637553811073303,
+      "logps/chosen": -4.58826208114624,
+      "logps/rejected": -5.025907516479492,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.58826208114624,
+      "rewards/margins": 0.4376456141471863,
+      "rewards/rejected": -5.025907516479492,
+      "sft_loss": 4.415618896484375,
+      "step": 1410
+    },
+    {
+      "epoch": 0.7573172771366449,
+      "grad_norm": 0.6626048352879617,
+      "learning_rate": 9.30894920180659e-07,
+      "logits/chosen": -0.47773098945617676,
+      "logits/rejected": -0.34003114700317383,
+      "logps/chosen": -4.645096778869629,
+      "logps/rejected": -4.930912971496582,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.645096778869629,
+      "rewards/margins": 0.2858158051967621,
+      "rewards/rejected": -4.930912971496582,
+      "sft_loss": 4.292731285095215,
+      "step": 1415
+    },
+    {
+      "epoch": 0.7599933099180465,
+      "grad_norm": 0.46790800138866595,
+      "learning_rate": 9.301028145701543e-07,
+      "logits/chosen": -0.4381941854953766,
+      "logits/rejected": -0.23344704508781433,
+      "logps/chosen": -4.556414604187012,
+      "logps/rejected": -5.132447242736816,
+      "loss": 0.053,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.556414604187012,
+      "rewards/margins": 0.5760326385498047,
+      "rewards/rejected": -5.132447242736816,
+      "sft_loss": 4.284694671630859,
+      "step": 1420
+    },
+    {
+      "epoch": 0.7626693426994481,
+      "grad_norm": 0.4538761637030669,
+      "learning_rate": 9.293065361002563e-07,
+      "logits/chosen": -0.46119099855422974,
+      "logits/rejected": -0.28453510999679565,
+      "logps/chosen": -4.576390266418457,
+      "logps/rejected": -5.065820217132568,
+      "loss": 0.055,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.576390266418457,
+      "rewards/margins": 0.4894295632839203,
+      "rewards/rejected": -5.065820217132568,
+      "sft_loss": 4.269384860992432,
+      "step": 1425
+    },
+    {
+      "epoch": 0.7653453754808497,
+      "grad_norm": 0.5262247782905854,
+      "learning_rate": 9.285060924964622e-07,
+      "logits/chosen": -0.5923141241073608,
+      "logits/rejected": -0.43897438049316406,
+      "logps/chosen": -4.758378028869629,
+      "logps/rejected": -5.130233287811279,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.758378028869629,
+      "rewards/margins": 0.3718549311161041,
+      "rewards/rejected": -5.130233287811279,
+      "sft_loss": 4.387657165527344,
+      "step": 1430
+    },
+    {
+      "epoch": 0.7680214082622512,
+      "grad_norm": 0.8738310108131122,
+      "learning_rate": 9.277014915246792e-07,
+      "logits/chosen": -0.4979740083217621,
+      "logits/rejected": -0.4129610061645508,
+      "logps/chosen": -4.627547740936279,
+      "logps/rejected": -5.047521114349365,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.627547740936279,
+      "rewards/margins": 0.41997361183166504,
+      "rewards/rejected": -5.047521114349365,
+      "sft_loss": 4.41188907623291,
+      "step": 1435
+    },
+    {
+      "epoch": 0.7706974410436528,
+      "grad_norm": 0.3665660050498204,
+      "learning_rate": 9.268927409911498e-07,
+      "logits/chosen": -0.6547388434410095,
+      "logits/rejected": -0.5283665060997009,
+      "logps/chosen": -4.4866533279418945,
+      "logps/rejected": -4.788092136383057,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.4866533279418945,
+      "rewards/margins": 0.3014386296272278,
+      "rewards/rejected": -4.788092136383057,
+      "sft_loss": 4.192168235778809,
+      "step": 1440
+    },
+    {
+      "epoch": 0.7733734738250544,
+      "grad_norm": 0.5777401513346766,
+      "learning_rate": 9.260798487423749e-07,
+      "logits/chosen": -0.8416692614555359,
+      "logits/rejected": -0.5481675863265991,
+      "logps/chosen": -4.734822750091553,
+      "logps/rejected": -5.159518241882324,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.734822750091553,
+      "rewards/margins": 0.42469555139541626,
+      "rewards/rejected": -5.159518241882324,
+      "sft_loss": 4.497926235198975,
+      "step": 1445
+    },
+    {
+      "epoch": 0.7760495066064559,
+      "grad_norm": 0.31040149478224743,
+      "learning_rate": 9.252628226650389e-07,
+      "logits/chosen": -0.6390944719314575,
+      "logits/rejected": -0.5294175744056702,
+      "logps/chosen": -4.691042423248291,
+      "logps/rejected": -4.936129093170166,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.691042423248291,
+      "rewards/margins": 0.24508705735206604,
+      "rewards/rejected": -4.936129093170166,
+      "sft_loss": 4.376203536987305,
+      "step": 1450
+    },
+    {
+      "epoch": 0.7787255393878575,
+      "grad_norm": 0.8324745102838857,
+      "learning_rate": 9.244416706859321e-07,
+      "logits/chosen": -0.6490924954414368,
+      "logits/rejected": -0.43776410818099976,
+      "logps/chosen": -4.428481578826904,
+      "logps/rejected": -4.945733070373535,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.428481578826904,
+      "rewards/margins": 0.5172508955001831,
+      "rewards/rejected": -4.945733070373535,
+      "sft_loss": 4.210099220275879,
+      "step": 1455
+    },
+    {
+      "epoch": 0.7814015721692591,
+      "grad_norm": 0.4524607634156159,
+      "learning_rate": 9.23616400771875e-07,
+      "logits/chosen": -0.6316944360733032,
+      "logits/rejected": -0.37926986813545227,
+      "logps/chosen": -4.5279340744018555,
+      "logps/rejected": -4.91841459274292,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.5279340744018555,
+      "rewards/margins": 0.39048075675964355,
+      "rewards/rejected": -4.91841459274292,
+      "sft_loss": 4.214154243469238,
+      "step": 1460
+    },
+    {
+      "epoch": 0.7840776049506607,
+      "grad_norm": 0.45011613171946563,
+      "learning_rate": 9.227870209296395e-07,
+      "logits/chosen": -0.5129834413528442,
+      "logits/rejected": -0.3363080322742462,
+      "logps/chosen": -4.668064117431641,
+      "logps/rejected": -5.000769138336182,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.668064117431641,
+      "rewards/margins": 0.3327048420906067,
+      "rewards/rejected": -5.000769138336182,
+      "sft_loss": 4.383709907531738,
+      "step": 1465
+    },
+    {
+      "epoch": 0.7867536377320622,
+      "grad_norm": 0.7776345772417559,
+      "learning_rate": 9.219535392058728e-07,
+      "logits/chosen": -0.5440296530723572,
+      "logits/rejected": -0.5481287837028503,
+      "logps/chosen": -4.869925022125244,
+      "logps/rejected": -5.206021308898926,
+      "loss": 0.054,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.869925022125244,
+      "rewards/margins": 0.33609622716903687,
+      "rewards/rejected": -5.206021308898926,
+      "sft_loss": 4.53743314743042,
+      "step": 1470
+    },
+    {
+      "epoch": 0.7894296705134638,
+      "grad_norm": 0.72464037163324,
+      "learning_rate": 9.211159636870181e-07,
+      "logits/chosen": -0.5129449367523193,
+      "logits/rejected": -0.3094898760318756,
+      "logps/chosen": -4.535016059875488,
+      "logps/rejected": -4.995938777923584,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.535016059875488,
+      "rewards/margins": 0.46092361211776733,
+      "rewards/rejected": -4.995938777923584,
+      "sft_loss": 4.218871116638184,
+      "step": 1475
+    },
+    {
+      "epoch": 0.7921057032948654,
+      "grad_norm": 0.43282860953379676,
+      "learning_rate": 9.202743024992367e-07,
+      "logits/chosen": -0.29366233944892883,
+      "logits/rejected": -0.19437307119369507,
+      "logps/chosen": -4.601809978485107,
+      "logps/rejected": -4.955982208251953,
+      "loss": 0.054,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -4.601809978485107,
+      "rewards/margins": 0.35417240858078003,
+      "rewards/rejected": -4.955982208251953,
+      "sft_loss": 4.2811689376831055,
+      "step": 1480
+    },
+    {
+      "epoch": 0.7947817360762669,
+      "grad_norm": 0.6048286155475231,
+      "learning_rate": 9.194285638083293e-07,
+      "logits/chosen": -0.4645848870277405,
+      "logits/rejected": -0.2716544270515442,
+      "logps/chosen": -4.692347049713135,
+      "logps/rejected": -5.111114501953125,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.692347049713135,
+      "rewards/margins": 0.4187680780887604,
+      "rewards/rejected": -5.111114501953125,
+      "sft_loss": 4.267924785614014,
+      "step": 1485
+    },
+    {
+      "epoch": 0.7974577688576685,
+      "grad_norm": 0.7623333749730745,
+      "learning_rate": 9.185787558196562e-07,
+      "logits/chosen": -0.5028216242790222,
+      "logits/rejected": -0.36704540252685547,
+      "logps/chosen": -4.721834182739258,
+      "logps/rejected": -5.188471794128418,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.721834182739258,
+      "rewards/margins": 0.466637521982193,
+      "rewards/rejected": -5.188471794128418,
+      "sft_loss": 4.410996437072754,
+      "step": 1490
+    },
+    {
+      "epoch": 0.8001338016390701,
+      "grad_norm": 0.5986005442913178,
+      "learning_rate": 9.177248867780583e-07,
+      "logits/chosen": -0.38116052746772766,
+      "logits/rejected": -0.2935437262058258,
+      "logps/chosen": -4.649445533752441,
+      "logps/rejected": -4.961583137512207,
+      "loss": 0.0549,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.649445533752441,
+      "rewards/margins": 0.3121368885040283,
+      "rewards/rejected": -4.961583137512207,
+      "sft_loss": 4.2997026443481445,
+      "step": 1495
+    },
+    {
+      "epoch": 0.8028098344204716,
+      "grad_norm": 0.5435666495008531,
+      "learning_rate": 9.168669649677769e-07,
+      "logits/chosen": -0.6052119135856628,
+      "logits/rejected": -0.4620954394340515,
+      "logps/chosen": -4.813775539398193,
+      "logps/rejected": -5.131553649902344,
+      "loss": 0.0548,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -4.813775539398193,
+      "rewards/margins": 0.31777825951576233,
+      "rewards/rejected": -5.131553649902344,
+      "sft_loss": 4.5045881271362305,
+      "step": 1500
+    },
+    {
+      "epoch": 0.8054858672018732,
+      "grad_norm": 0.47022443877330955,
+      "learning_rate": 9.16004998712373e-07,
+      "logits/chosen": -0.5376960039138794,
+      "logits/rejected": -0.4372090697288513,
+      "logps/chosen": -4.671207904815674,
+      "logps/rejected": -4.986814498901367,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.671207904815674,
+      "rewards/margins": 0.3156066834926605,
+      "rewards/rejected": -4.986814498901367,
+      "sft_loss": 4.38083553314209,
+      "step": 1505
+    },
+    {
+      "epoch": 0.8081618999832748,
+      "grad_norm": 0.6199400338313545,
+      "learning_rate": 9.151389963746472e-07,
+      "logits/chosen": -0.5899304747581482,
+      "logits/rejected": -0.18980534374713898,
+      "logps/chosen": -4.4528584480285645,
+      "logps/rejected": -5.0324931144714355,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -4.4528584480285645,
+      "rewards/margins": 0.579634964466095,
+      "rewards/rejected": -5.0324931144714355,
+      "sft_loss": 4.2412800788879395,
+      "step": 1510
+    },
+    {
+      "epoch": 0.8108379327646764,
+      "grad_norm": 0.5555948687982465,
+      "learning_rate": 9.142689663565577e-07,
+      "logits/chosen": -0.5152315497398376,
+      "logits/rejected": -0.436471164226532,
+      "logps/chosen": -4.5293426513671875,
+      "logps/rejected": -4.902256965637207,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.5293426513671875,
+      "rewards/margins": 0.3729146420955658,
+      "rewards/rejected": -4.902256965637207,
+      "sft_loss": 4.266984462738037,
+      "step": 1515
+    },
+    {
+      "epoch": 0.8135139655460779,
+      "grad_norm": 0.45142612803831084,
+      "learning_rate": 9.133949170991397e-07,
+      "logits/chosen": -0.53922039270401,
+      "logits/rejected": -0.4263296127319336,
+      "logps/chosen": -4.764470100402832,
+      "logps/rejected": -5.035752296447754,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.764470100402832,
+      "rewards/margins": 0.27128297090530396,
+      "rewards/rejected": -5.035752296447754,
+      "sft_loss": 4.539823055267334,
+      "step": 1520
+    },
+    {
+      "epoch": 0.8161899983274795,
+      "grad_norm": 0.41254383948397666,
+      "learning_rate": 9.125168570824231e-07,
+      "logits/chosen": -0.6185752153396606,
+      "logits/rejected": -0.3851194977760315,
+      "logps/chosen": -4.496891021728516,
+      "logps/rejected": -4.882938861846924,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.496891021728516,
+      "rewards/margins": 0.3860477805137634,
+      "rewards/rejected": -4.882938861846924,
+      "sft_loss": 4.258194923400879,
+      "step": 1525
+    },
+    {
+      "epoch": 0.8188660311088811,
+      "grad_norm": 0.7151117766432287,
+      "learning_rate": 9.116347948253496e-07,
+      "logits/chosen": -0.5224160552024841,
+      "logits/rejected": -0.3366536498069763,
+      "logps/chosen": -4.449323654174805,
+      "logps/rejected": -4.829797267913818,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.449323654174805,
+      "rewards/margins": 0.3804740905761719,
+      "rewards/rejected": -4.829797267913818,
+      "sft_loss": 4.185049533843994,
+      "step": 1530
+    },
+    {
+      "epoch": 0.8215420638902826,
+      "grad_norm": 0.583199273167788,
+      "learning_rate": 9.107487388856916e-07,
+      "logits/chosen": -0.5745779275894165,
+      "logits/rejected": -0.3310237228870392,
+      "logps/chosen": -4.638999938964844,
+      "logps/rejected": -5.003554344177246,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.638999938964844,
+      "rewards/margins": 0.3645547330379486,
+      "rewards/rejected": -5.003554344177246,
+      "sft_loss": 4.2992844581604,
+      "step": 1535
+    },
+    {
+      "epoch": 0.8242180966716842,
+      "grad_norm": 1.0175227991306681,
+      "learning_rate": 9.098586978599673e-07,
+      "logits/chosen": -0.4655645489692688,
+      "logits/rejected": -0.27722448110580444,
+      "logps/chosen": -4.643935680389404,
+      "logps/rejected": -5.168910503387451,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.643935680389404,
+      "rewards/margins": 0.524974524974823,
+      "rewards/rejected": -5.168910503387451,
+      "sft_loss": 4.284567832946777,
+      "step": 1540
+    },
+    {
+      "epoch": 0.8268941294530858,
+      "grad_norm": 1.0535335408279394,
+      "learning_rate": 9.089646803833588e-07,
+      "logits/chosen": -0.48223942518234253,
+      "logits/rejected": -0.29267364740371704,
+      "logps/chosen": -4.699334621429443,
+      "logps/rejected": -5.175503730773926,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.699334621429443,
+      "rewards/margins": 0.47616925835609436,
+      "rewards/rejected": -5.175503730773926,
+      "sft_loss": 4.3690996170043945,
+      "step": 1545
+    },
+    {
+      "epoch": 0.8295701622344873,
+      "grad_norm": 1.225203498147348,
+      "learning_rate": 9.080666951296276e-07,
+      "logits/chosen": -0.5685257315635681,
+      "logits/rejected": -0.23230977356433868,
+      "logps/chosen": -4.532471656799316,
+      "logps/rejected": -5.177938938140869,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.532471656799316,
+      "rewards/margins": 0.6454674601554871,
+      "rewards/rejected": -5.177938938140869,
+      "sft_loss": 4.264753818511963,
+      "step": 1550
+    },
+    {
+      "epoch": 0.8322461950158889,
+      "grad_norm": 0.3571080799880975,
+      "learning_rate": 9.071647508110305e-07,
+      "logits/chosen": -0.5444567203521729,
+      "logits/rejected": -0.20953519642353058,
+      "logps/chosen": -4.405203819274902,
+      "logps/rejected": -5.111395359039307,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.405203819274902,
+      "rewards/margins": 0.706192135810852,
+      "rewards/rejected": -5.111395359039307,
+      "sft_loss": 4.1048126220703125,
+      "step": 1555
+    },
+    {
+      "epoch": 0.8349222277972905,
+      "grad_norm": 0.4647677566083696,
+      "learning_rate": 9.062588561782354e-07,
+      "logits/chosen": -0.3920247256755829,
+      "logits/rejected": -0.31407880783081055,
+      "logps/chosen": -4.9274492263793945,
+      "logps/rejected": -5.254195690155029,
+      "loss": 0.0543,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.9274492263793945,
+      "rewards/margins": 0.32674673199653625,
+      "rewards/rejected": -5.254195690155029,
+      "sft_loss": 4.650223731994629,
+      "step": 1560
+    },
+    {
+      "epoch": 0.8375982605786921,
+      "grad_norm": 0.33147523191362094,
+      "learning_rate": 9.053490200202358e-07,
+      "logits/chosen": -0.4448007047176361,
+      "logits/rejected": -0.37099045515060425,
+      "logps/chosen": -4.541591167449951,
+      "logps/rejected": -4.890256404876709,
+      "loss": 0.0546,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.541591167449951,
+      "rewards/margins": 0.3486659526824951,
+      "rewards/rejected": -4.890256404876709,
+      "sft_loss": 4.340108871459961,
+      "step": 1565
+    },
+    {
+      "epoch": 0.8402742933600936,
+      "grad_norm": 0.666464713418103,
+      "learning_rate": 9.044352511642661e-07,
+      "logits/chosen": -0.4679701328277588,
+      "logits/rejected": -0.3994170129299164,
+      "logps/chosen": -4.796227931976318,
+      "logps/rejected": -5.057922840118408,
+      "loss": 0.0545,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.796227931976318,
+      "rewards/margins": 0.2616943120956421,
+      "rewards/rejected": -5.057922840118408,
+      "sft_loss": 4.517434597015381,
+      "step": 1570
+    },
+    {
+      "epoch": 0.8429503261414952,
+      "grad_norm": 0.6293748246790352,
+      "learning_rate": 9.03517558475716e-07,
+      "logits/chosen": -0.5948684811592102,
+      "logits/rejected": -0.4503151774406433,
+      "logps/chosen": -4.529299736022949,
+      "logps/rejected": -4.76159143447876,
+      "loss": 0.056,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.529299736022949,
+      "rewards/margins": 0.23229138553142548,
+      "rewards/rejected": -4.76159143447876,
+      "sft_loss": 4.296181678771973,
+      "step": 1575
+    },
+    {
+      "epoch": 0.8456263589228968,
+      "grad_norm": 0.4250320609206886,
+      "learning_rate": 9.025959508580436e-07,
+      "logits/chosen": -0.6207831501960754,
+      "logits/rejected": -0.3034132719039917,
+      "logps/chosen": -4.6496171951293945,
+      "logps/rejected": -5.029486656188965,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.6496171951293945,
+      "rewards/margins": 0.3798690736293793,
+      "rewards/rejected": -5.029486656188965,
+      "sft_loss": 4.412766456604004,
+      "step": 1580
+    },
+    {
+      "epoch": 0.8483023917042983,
+      "grad_norm": 0.3411551452075428,
+      "learning_rate": 9.016704372526905e-07,
+      "logits/chosen": -0.6650777459144592,
+      "logits/rejected": -0.4226778447628021,
+      "logps/chosen": -4.5687479972839355,
+      "logps/rejected": -5.018956661224365,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.5687479972839355,
+      "rewards/margins": 0.4502086639404297,
+      "rewards/rejected": -5.018956661224365,
+      "sft_loss": 4.3290324211120605,
+      "step": 1585
+    },
+    {
+      "epoch": 0.8509784244856999,
+      "grad_norm": 0.3485149666875214,
+      "learning_rate": 9.007410266389934e-07,
+      "logits/chosen": -0.6740955114364624,
+      "logits/rejected": -0.6200405359268188,
+      "logps/chosen": -4.6393561363220215,
+      "logps/rejected": -4.842759132385254,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -4.6393561363220215,
+      "rewards/margins": 0.20340339839458466,
+      "rewards/rejected": -4.842759132385254,
+      "sft_loss": 4.307949066162109,
+      "step": 1590
+    },
+    {
+      "epoch": 0.8536544572671015,
+      "grad_norm": 0.6993713590336478,
+      "learning_rate": 8.998077280340981e-07,
+      "logits/chosen": -0.5477308034896851,
+      "logits/rejected": -0.4848629832267761,
+      "logps/chosen": -4.8374176025390625,
+      "logps/rejected": -5.147733688354492,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.8374176025390625,
+      "rewards/margins": 0.31031590700149536,
+      "rewards/rejected": -5.147733688354492,
+      "sft_loss": 4.451597690582275,
+      "step": 1595
+    },
+    {
+      "epoch": 0.8563304900485031,
+      "grad_norm": 0.5857090427241004,
+      "learning_rate": 8.988705504928722e-07,
+      "logits/chosen": -0.593254029750824,
+      "logits/rejected": -0.3634767532348633,
+      "logps/chosen": -4.616876125335693,
+      "logps/rejected": -5.206143856048584,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.616876125335693,
+      "rewards/margins": 0.5892679691314697,
+      "rewards/rejected": -5.206143856048584,
+      "sft_loss": 4.2967424392700195,
+      "step": 1600
+    },
+    {
+      "epoch": 0.8563304900485031,
+      "eval_logits/chosen": -0.02904585748910904,
+      "eval_logits/rejected": 0.07823801785707474,
+      "eval_logps/chosen": -4.5006561279296875,
+      "eval_logps/rejected": -4.917550086975098,
+      "eval_loss": 0.051895011216402054,
+      "eval_rewards/accuracies": 0.6313056349754333,
+      "eval_rewards/chosen": -4.5006561279296875,
+      "eval_rewards/margins": 0.41689401865005493,
+      "eval_rewards/rejected": -4.917550086975098,
+      "eval_runtime": 43.4471,
+      "eval_samples_per_second": 30.957,
+      "eval_sft_loss": 4.063570499420166,
+      "eval_steps_per_second": 7.757,
+      "step": 1600
+    },
+    {
+      "epoch": 0.8590065228299046,
+      "grad_norm": 0.4116631557826821,
+      "learning_rate": 8.979295031078157e-07,
+      "logits/chosen": -0.5330547094345093,
+      "logits/rejected": -0.25229379534721375,
+      "logps/chosen": -4.346199035644531,
+      "logps/rejected": -4.8608598709106445,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.346199035644531,
+      "rewards/margins": 0.5146608948707581,
+      "rewards/rejected": -4.8608598709106445,
+      "sft_loss": 4.030482292175293,
+      "step": 1605
+    },
+    {
+      "epoch": 0.8616825556113062,
+      "grad_norm": 0.38214100986360633,
+      "learning_rate": 8.969845950089751e-07,
+      "logits/chosen": -0.5902666449546814,
+      "logits/rejected": -0.3709181249141693,
+      "logps/chosen": -4.7403693199157715,
+      "logps/rejected": -5.202174186706543,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.7403693199157715,
+      "rewards/margins": 0.4618045389652252,
+      "rewards/rejected": -5.202174186706543,
+      "sft_loss": 4.481407165527344,
+      "step": 1610
+    },
+    {
+      "epoch": 0.8643585883927078,
+      "grad_norm": 0.41539489108804767,
+      "learning_rate": 8.960358353638526e-07,
+      "logits/chosen": -0.4401358664035797,
+      "logits/rejected": -0.2931682765483856,
+      "logps/chosen": -4.8379106521606445,
+      "logps/rejected": -5.323012828826904,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.8379106521606445,
+      "rewards/margins": 0.48510226607322693,
+      "rewards/rejected": -5.323012828826904,
+      "sft_loss": 4.449693202972412,
+      "step": 1615
+    },
+    {
+      "epoch": 0.8670346211741093,
+      "grad_norm": 0.9036399070841549,
+      "learning_rate": 8.950832333773184e-07,
+      "logits/chosen": -0.45871132612228394,
+      "logits/rejected": -0.25480368733406067,
+      "logps/chosen": -4.593859672546387,
+      "logps/rejected": -4.8868608474731445,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.593859672546387,
+      "rewards/margins": 0.2930009961128235,
+      "rewards/rejected": -4.8868608474731445,
+      "sft_loss": 4.267394065856934,
+      "step": 1620
+    },
+    {
+      "epoch": 0.869710653955511,
+      "grad_norm": 0.4427581928743918,
+      "learning_rate": 8.941267982915213e-07,
+      "logits/chosen": -0.45720523595809937,
+      "logits/rejected": -0.39475345611572266,
+      "logps/chosen": -4.652650356292725,
+      "logps/rejected": -4.871710777282715,
+      "loss": 0.0551,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.652650356292725,
+      "rewards/margins": 0.2190600335597992,
+      "rewards/rejected": -4.871710777282715,
+      "sft_loss": 4.429585933685303,
+      "step": 1625
+    },
+    {
+      "epoch": 0.8723866867369126,
+      "grad_norm": 0.6495835714729182,
+      "learning_rate": 8.931665393857983e-07,
+      "logits/chosen": -0.5988067388534546,
+      "logits/rejected": -0.41781100630760193,
+      "logps/chosen": -4.665750026702881,
+      "logps/rejected": -5.207381725311279,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.665750026702881,
+      "rewards/margins": 0.5416315793991089,
+      "rewards/rejected": -5.207381725311279,
+      "sft_loss": 4.4793195724487305,
+      "step": 1630
+    },
+    {
+      "epoch": 0.875062719518314,
+      "grad_norm": 0.41623413128943515,
+      "learning_rate": 8.922024659765861e-07,
+      "logits/chosen": -0.7399147748947144,
+      "logits/rejected": -0.5550050139427185,
+      "logps/chosen": -4.498051643371582,
+      "logps/rejected": -4.938418865203857,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.498051643371582,
+      "rewards/margins": 0.44036778807640076,
+      "rewards/rejected": -4.938418865203857,
+      "sft_loss": 4.29925012588501,
+      "step": 1635
+    },
+    {
+      "epoch": 0.8777387522997157,
+      "grad_norm": 0.5313489161329956,
+      "learning_rate": 8.912345874173288e-07,
+      "logits/chosen": -0.7936776876449585,
+      "logits/rejected": -0.5909181237220764,
+      "logps/chosen": -4.6866841316223145,
+      "logps/rejected": -5.111433506011963,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.6866841316223145,
+      "rewards/margins": 0.4247500002384186,
+      "rewards/rejected": -5.111433506011963,
+      "sft_loss": 4.379754543304443,
+      "step": 1640
+    },
+    {
+      "epoch": 0.8804147850811173,
+      "grad_norm": 0.6359924799554241,
+      "learning_rate": 8.902629130983885e-07,
+      "logits/chosen": -0.7347515821456909,
+      "logits/rejected": -0.6840382814407349,
+      "logps/chosen": -4.671685695648193,
+      "logps/rejected": -4.900129795074463,
+      "loss": 0.0551,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -4.671685695648193,
+      "rewards/margins": 0.22844386100769043,
+      "rewards/rejected": -4.900129795074463,
+      "sft_loss": 4.440484046936035,
+      "step": 1645
+    },
+    {
+      "epoch": 0.8830908178625189,
+      "grad_norm": 0.5805478038553051,
+      "learning_rate": 8.892874524469537e-07,
+      "logits/chosen": -0.5966772437095642,
+      "logits/rejected": -0.5708019137382507,
+      "logps/chosen": -4.531647682189941,
+      "logps/rejected": -4.784287452697754,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.531647682189941,
+      "rewards/margins": 0.25264012813568115,
+      "rewards/rejected": -4.784287452697754,
+      "sft_loss": 4.222924709320068,
+      "step": 1650
+    },
+    {
+      "epoch": 0.8857668506439204,
+      "grad_norm": 0.4154036618062291,
+      "learning_rate": 8.883082149269478e-07,
+      "logits/chosen": -0.7492375373840332,
+      "logits/rejected": -0.625057578086853,
+      "logps/chosen": -4.6985764503479,
+      "logps/rejected": -5.03641414642334,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.6985764503479,
+      "rewards/margins": 0.33783769607543945,
+      "rewards/rejected": -5.03641414642334,
+      "sft_loss": 4.449755668640137,
+      "step": 1655
+    },
+    {
+      "epoch": 0.888442883425322,
+      "grad_norm": 0.6942420678365312,
+      "learning_rate": 8.873252100389377e-07,
+      "logits/chosen": -0.6865184307098389,
+      "logits/rejected": -0.6350366473197937,
+      "logps/chosen": -4.639318466186523,
+      "logps/rejected": -4.9327192306518555,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.639318466186523,
+      "rewards/margins": 0.2934008240699768,
+      "rewards/rejected": -4.9327192306518555,
+      "sft_loss": 4.407034873962402,
+      "step": 1660
+    },
+    {
+      "epoch": 0.8911189162067236,
+      "grad_norm": 0.8556379469701134,
+      "learning_rate": 8.863384473200411e-07,
+      "logits/chosen": -0.5269213914871216,
+      "logits/rejected": -0.47171592712402344,
+      "logps/chosen": -4.581943511962891,
+      "logps/rejected": -4.845519065856934,
+      "loss": 0.054,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.581943511962891,
+      "rewards/margins": 0.2635752558708191,
+      "rewards/rejected": -4.845519065856934,
+      "sft_loss": 4.290720462799072,
+      "step": 1665
+    },
+    {
+      "epoch": 0.8937949489881251,
+      "grad_norm": 0.38175959218553795,
+      "learning_rate": 8.853479363438342e-07,
+      "logits/chosen": -0.5232313871383667,
+      "logits/rejected": -0.3149745464324951,
+      "logps/chosen": -4.757014751434326,
+      "logps/rejected": -5.126891136169434,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.757014751434326,
+      "rewards/margins": 0.3698762059211731,
+      "rewards/rejected": -5.126891136169434,
+      "sft_loss": 4.466257572174072,
+      "step": 1670
+    },
+    {
+      "epoch": 0.8964709817695267,
+      "grad_norm": 0.7736549454767151,
+      "learning_rate": 8.843536867202588e-07,
+      "logits/chosen": -0.5337976217269897,
+      "logits/rejected": -0.30675259232521057,
+      "logps/chosen": -4.592353343963623,
+      "logps/rejected": -5.095428943634033,
+      "loss": 0.0539,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.592353343963623,
+      "rewards/margins": 0.5030753016471863,
+      "rewards/rejected": -5.095428943634033,
+      "sft_loss": 4.437192440032959,
+      "step": 1675
+    },
+    {
+      "epoch": 0.8991470145509283,
+      "grad_norm": 0.5225751200668408,
+      "learning_rate": 8.833557080955292e-07,
+      "logits/chosen": -0.668506383895874,
+      "logits/rejected": -0.5236512422561646,
+      "logps/chosen": -4.579380035400391,
+      "logps/rejected": -4.8987932205200195,
+      "loss": 0.0548,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.579380035400391,
+      "rewards/margins": 0.3194130063056946,
+      "rewards/rejected": -4.8987932205200195,
+      "sft_loss": 4.3848490715026855,
+      "step": 1680
+    },
+    {
+      "epoch": 0.9018230473323299,
+      "grad_norm": 0.3776357485013768,
+      "learning_rate": 8.823540101520381e-07,
+      "logits/chosen": -0.6779472231864929,
+      "logits/rejected": -0.41766971349716187,
+      "logps/chosen": -4.687167644500732,
+      "logps/rejected": -5.149572372436523,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.687167644500732,
+      "rewards/margins": 0.4624043107032776,
+      "rewards/rejected": -5.149572372436523,
+      "sft_loss": 4.447117805480957,
+      "step": 1685
+    },
+    {
+      "epoch": 0.9044990801137314,
+      "grad_norm": 0.40393689961090434,
+      "learning_rate": 8.813486026082637e-07,
+      "logits/chosen": -0.6170944571495056,
+      "logits/rejected": -0.3785988688468933,
+      "logps/chosen": -4.538905143737793,
+      "logps/rejected": -5.0261921882629395,
+      "loss": 0.052,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.538905143737793,
+      "rewards/margins": 0.48728686571121216,
+      "rewards/rejected": -5.0261921882629395,
+      "sft_loss": 4.21726131439209,
+      "step": 1690
+    },
+    {
+      "epoch": 0.907175112895133,
+      "grad_norm": 0.6594051353912328,
+      "learning_rate": 8.803394952186742e-07,
+      "logits/chosen": -0.6432759165763855,
+      "logits/rejected": -0.46522051095962524,
+      "logps/chosen": -4.532651424407959,
+      "logps/rejected": -4.9046783447265625,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.532651424407959,
+      "rewards/margins": 0.37202686071395874,
+      "rewards/rejected": -4.9046783447265625,
+      "sft_loss": 4.1833367347717285,
+      "step": 1695
+    },
+    {
+      "epoch": 0.9098511456765346,
+      "grad_norm": 0.7064046912842677,
+      "learning_rate": 8.793266977736342e-07,
+      "logits/chosen": -0.5009998083114624,
+      "logits/rejected": -0.6162468791007996,
+      "logps/chosen": -4.827506065368652,
+      "logps/rejected": -4.991827964782715,
+      "loss": 0.0557,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -4.827506065368652,
+      "rewards/margins": 0.16432145237922668,
+      "rewards/rejected": -4.991827964782715,
+      "sft_loss": 4.603185653686523,
+      "step": 1700
+    },
+    {
+      "epoch": 0.9125271784579361,
+      "grad_norm": 0.5206622644102082,
+      "learning_rate": 8.783102200993085e-07,
+      "logits/chosen": -0.5576699376106262,
+      "logits/rejected": -0.43255311250686646,
+      "logps/chosen": -4.724583625793457,
+      "logps/rejected": -5.145751953125,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.724583625793457,
+      "rewards/margins": 0.42116886377334595,
+      "rewards/rejected": -5.145751953125,
+      "sft_loss": 4.429846286773682,
+      "step": 1705
+    },
+    {
+      "epoch": 0.9152032112393377,
+      "grad_norm": 0.44476556944134527,
+      "learning_rate": 8.772900720575683e-07,
+      "logits/chosen": -0.5629488229751587,
+      "logits/rejected": -0.43112772703170776,
+      "logps/chosen": -4.643448829650879,
+      "logps/rejected": -4.956890106201172,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.643448829650879,
+      "rewards/margins": 0.3134412169456482,
+      "rewards/rejected": -4.956890106201172,
+      "sft_loss": 4.35292911529541,
+      "step": 1710
+    },
+    {
+      "epoch": 0.9178792440207393,
+      "grad_norm": 0.3480510172874056,
+      "learning_rate": 8.762662635458944e-07,
+      "logits/chosen": -0.5707327127456665,
+      "logits/rejected": -0.3803301453590393,
+      "logps/chosen": -4.5035834312438965,
+      "logps/rejected": -4.909355640411377,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.5035834312438965,
+      "rewards/margins": 0.4057716429233551,
+      "rewards/rejected": -4.909355640411377,
+      "sft_loss": 4.233164310455322,
+      "step": 1715
+    },
+    {
+      "epoch": 0.9205552768021408,
+      "grad_norm": 0.5164062533786418,
+      "learning_rate": 8.752388044972811e-07,
+      "logits/chosen": -0.484900563955307,
+      "logits/rejected": -0.4002462923526764,
+      "logps/chosen": -4.537683963775635,
+      "logps/rejected": -4.98577356338501,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.537683963775635,
+      "rewards/margins": 0.448089599609375,
+      "rewards/rejected": -4.98577356338501,
+      "sft_loss": 4.265603542327881,
+      "step": 1720
+    },
+    {
+      "epoch": 0.9232313095835424,
+      "grad_norm": 1.3750125559284796,
+      "learning_rate": 8.74207704880141e-07,
+      "logits/chosen": -0.38507509231567383,
+      "logits/rejected": -0.2651823163032532,
+      "logps/chosen": -4.553491115570068,
+      "logps/rejected": -5.053616523742676,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.553491115570068,
+      "rewards/margins": 0.5001254081726074,
+      "rewards/rejected": -5.053616523742676,
+      "sft_loss": 4.2198309898376465,
+      "step": 1725
+    },
+    {
+      "epoch": 0.925907342364944,
+      "grad_norm": 0.6479414735451374,
+      "learning_rate": 8.731729746982068e-07,
+      "logits/chosen": -0.4087442457675934,
+      "logits/rejected": -0.3513071537017822,
+      "logps/chosen": -4.680237293243408,
+      "logps/rejected": -5.0000081062316895,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.680237293243408,
+      "rewards/margins": 0.319771409034729,
+      "rewards/rejected": -5.0000081062316895,
+      "sft_loss": 4.4230875968933105,
+      "step": 1730
+    },
+    {
+      "epoch": 0.9285833751463456,
+      "grad_norm": 0.5320410614211661,
+      "learning_rate": 8.721346239904355e-07,
+      "logits/chosen": -0.6621562242507935,
+      "logits/rejected": -0.38414087891578674,
+      "logps/chosen": -4.588827610015869,
+      "logps/rejected": -5.170763969421387,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.588827610015869,
+      "rewards/margins": 0.5819366574287415,
+      "rewards/rejected": -5.170763969421387,
+      "sft_loss": 4.343879699707031,
+      "step": 1735
+    },
+    {
+      "epoch": 0.9312594079277471,
+      "grad_norm": 0.7707100372795159,
+      "learning_rate": 8.710926628309101e-07,
+      "logits/chosen": -0.6921442151069641,
+      "logits/rejected": -0.4576943814754486,
+      "logps/chosen": -4.3977508544921875,
+      "logps/rejected": -4.836879730224609,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.3977508544921875,
+      "rewards/margins": 0.4391290545463562,
+      "rewards/rejected": -4.836879730224609,
+      "sft_loss": 4.144339561462402,
+      "step": 1740
+    },
+    {
+      "epoch": 0.9339354407091487,
+      "grad_norm": 0.40155269984358594,
+      "learning_rate": 8.700471013287424e-07,
+      "logits/chosen": -0.5236523747444153,
+      "logits/rejected": -0.5326138734817505,
+      "logps/chosen": -4.680853366851807,
+      "logps/rejected": -4.947844982147217,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.680853366851807,
+      "rewards/margins": 0.266991525888443,
+      "rewards/rejected": -4.947844982147217,
+      "sft_loss": 4.386083126068115,
+      "step": 1745
+    },
+    {
+      "epoch": 0.9366114734905503,
+      "grad_norm": 0.5819037718837893,
+      "learning_rate": 8.689979496279746e-07,
+      "logits/chosen": -0.7221859693527222,
+      "logits/rejected": -0.6695482730865479,
+      "logps/chosen": -4.943053722381592,
+      "logps/rejected": -5.197215557098389,
+      "loss": 0.0556,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.943053722381592,
+      "rewards/margins": 0.2541615962982178,
+      "rewards/rejected": -5.197215557098389,
+      "sft_loss": 4.715271949768066,
+      "step": 1750
+    },
+    {
+      "epoch": 0.9392875062719518,
+      "grad_norm": 0.4677346992925345,
+      "learning_rate": 8.679452179074811e-07,
+      "logits/chosen": -0.7230926752090454,
+      "logits/rejected": -0.5767534375190735,
+      "logps/chosen": -4.456429481506348,
+      "logps/rejected": -4.846767425537109,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.456429481506348,
+      "rewards/margins": 0.39033815264701843,
+      "rewards/rejected": -4.846767425537109,
+      "sft_loss": 4.245125770568848,
+      "step": 1755
+    },
+    {
+      "epoch": 0.9419635390533534,
+      "grad_norm": 0.9766948042968379,
+      "learning_rate": 8.668889163808698e-07,
+      "logits/chosen": -0.6952082514762878,
+      "logits/rejected": -0.5020134449005127,
+      "logps/chosen": -4.320244312286377,
+      "logps/rejected": -4.711088180541992,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.320244312286377,
+      "rewards/margins": 0.3908434510231018,
+      "rewards/rejected": -4.711088180541992,
+      "sft_loss": 4.1057281494140625,
+      "step": 1760
+    },
+    {
+      "epoch": 0.944639571834755,
+      "grad_norm": 0.8113564262746916,
+      "learning_rate": 8.658290552963827e-07,
+      "logits/chosen": -0.6360453963279724,
+      "logits/rejected": -0.5571495294570923,
+      "logps/chosen": -4.789154529571533,
+      "logps/rejected": -5.147796154022217,
+      "loss": 0.0547,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.789154529571533,
+      "rewards/margins": 0.35864168405532837,
+      "rewards/rejected": -5.147796154022217,
+      "sft_loss": 4.56979513168335,
+      "step": 1765
+    },
+    {
+      "epoch": 0.9473156046161565,
+      "grad_norm": 0.40121347821380127,
+      "learning_rate": 8.647656449367966e-07,
+      "logits/chosen": -0.6600741147994995,
+      "logits/rejected": -0.48249635100364685,
+      "logps/chosen": -4.793465614318848,
+      "logps/rejected": -5.12413215637207,
+      "loss": 0.0543,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.793465614318848,
+      "rewards/margins": 0.33066678047180176,
+      "rewards/rejected": -5.12413215637207,
+      "sft_loss": 4.610315799713135,
+      "step": 1770
+    },
+    {
+      "epoch": 0.9499916373975581,
+      "grad_norm": 0.4609934021865174,
+      "learning_rate": 8.636986956193235e-07,
+      "logits/chosen": -0.7527320981025696,
+      "logits/rejected": -0.6049268841743469,
+      "logps/chosen": -4.484477519989014,
+      "logps/rejected": -4.966238975524902,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.484477519989014,
+      "rewards/margins": 0.4817616045475006,
+      "rewards/rejected": -4.966238975524902,
+      "sft_loss": 4.206968307495117,
+      "step": 1775
+    },
+    {
+      "epoch": 0.9526676701789597,
+      "grad_norm": 0.5990607543889642,
+      "learning_rate": 8.626282176955104e-07,
+      "logits/chosen": -0.7125250697135925,
+      "logits/rejected": -0.5920495390892029,
+      "logps/chosen": -4.532000541687012,
+      "logps/rejected": -4.994868278503418,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.532000541687012,
+      "rewards/margins": 0.46286827325820923,
+      "rewards/rejected": -4.994868278503418,
+      "sft_loss": 4.308735370635986,
+      "step": 1780
+    },
+    {
+      "epoch": 0.9553437029603613,
+      "grad_norm": 0.6096852450418404,
+      "learning_rate": 8.615542215511389e-07,
+      "logits/chosen": -0.6793350577354431,
+      "logits/rejected": -0.6178755760192871,
+      "logps/chosen": -4.636038303375244,
+      "logps/rejected": -4.855711460113525,
+      "loss": 0.0548,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.636038303375244,
+      "rewards/margins": 0.21967339515686035,
+      "rewards/rejected": -4.855711460113525,
+      "sft_loss": 4.38653564453125,
+      "step": 1785
+    },
+    {
+      "epoch": 0.9580197357417628,
+      "grad_norm": 0.37934684130563323,
+      "learning_rate": 8.604767176061241e-07,
+      "logits/chosen": -0.6598840951919556,
+      "logits/rejected": -0.5501774549484253,
+      "logps/chosen": -4.648791313171387,
+      "logps/rejected": -5.084364414215088,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.648791313171387,
+      "rewards/margins": 0.4355725347995758,
+      "rewards/rejected": -5.084364414215088,
+      "sft_loss": 4.3693437576293945,
+      "step": 1790
+    },
+    {
+      "epoch": 0.9606957685231644,
+      "grad_norm": 0.3543703405808491,
+      "learning_rate": 8.593957163144141e-07,
+      "logits/chosen": -0.8177486658096313,
+      "logits/rejected": -0.6528640985488892,
+      "logps/chosen": -4.524102210998535,
+      "logps/rejected": -5.0155134201049805,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.524102210998535,
+      "rewards/margins": 0.4914116859436035,
+      "rewards/rejected": -5.0155134201049805,
+      "sft_loss": 4.297591686248779,
+      "step": 1795
+    },
+    {
+      "epoch": 0.963371801304566,
+      "grad_norm": 0.587837761567623,
+      "learning_rate": 8.58311228163888e-07,
+      "logits/chosen": -0.7575830817222595,
+      "logits/rejected": -0.6973570585250854,
+      "logps/chosen": -4.548133850097656,
+      "logps/rejected": -4.882990837097168,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.548133850097656,
+      "rewards/margins": 0.3348572254180908,
+      "rewards/rejected": -4.882990837097168,
+      "sft_loss": 4.285576343536377,
+      "step": 1800
+    },
+    {
+      "epoch": 0.9660478340859675,
+      "grad_norm": 0.7831749014712782,
+      "learning_rate": 8.57223263676255e-07,
+      "logits/chosen": -0.827855110168457,
+      "logits/rejected": -0.6802206039428711,
+      "logps/chosen": -4.526045799255371,
+      "logps/rejected": -5.129607200622559,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.526045799255371,
+      "rewards/margins": 0.603561282157898,
+      "rewards/rejected": -5.129607200622559,
+      "sft_loss": 4.310399055480957,
+      "step": 1805
+    },
+    {
+      "epoch": 0.9687238668673691,
+      "grad_norm": 0.4646618216970684,
+      "learning_rate": 8.561318334069511e-07,
+      "logits/chosen": -0.695126473903656,
+      "logits/rejected": -0.5704531073570251,
+      "logps/chosen": -4.636171340942383,
+      "logps/rejected": -5.058096408843994,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.636171340942383,
+      "rewards/margins": 0.42192497849464417,
+      "rewards/rejected": -5.058096408843994,
+      "sft_loss": 4.393563747406006,
+      "step": 1810
+    },
+    {
+      "epoch": 0.9713998996487707,
+      "grad_norm": 0.4712524042717296,
+      "learning_rate": 8.550369479450375e-07,
+      "logits/chosen": -0.5622086524963379,
+      "logits/rejected": -0.40890851616859436,
+      "logps/chosen": -4.376356601715088,
+      "logps/rejected": -4.874255657196045,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.376356601715088,
+      "rewards/margins": 0.4978991448879242,
+      "rewards/rejected": -4.874255657196045,
+      "sft_loss": 4.152787685394287,
+      "step": 1815
+    },
+    {
+      "epoch": 0.9740759324301723,
+      "grad_norm": 0.34063083871666683,
+      "learning_rate": 8.539386179130977e-07,
+      "logits/chosen": -0.49509549140930176,
+      "logits/rejected": -0.4550551474094391,
+      "logps/chosen": -4.642989158630371,
+      "logps/rejected": -5.026967525482178,
+      "loss": 0.053,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.642989158630371,
+      "rewards/margins": 0.3839784264564514,
+      "rewards/rejected": -5.026967525482178,
+      "sft_loss": 4.345905303955078,
+      "step": 1820
+    },
+    {
+      "epoch": 0.9767519652115738,
+      "grad_norm": 0.3872496478526205,
+      "learning_rate": 8.528368539671347e-07,
+      "logits/chosen": -0.5788191556930542,
+      "logits/rejected": -0.3689228594303131,
+      "logps/chosen": -4.652213096618652,
+      "logps/rejected": -5.309985160827637,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.652213096618652,
+      "rewards/margins": 0.6577725410461426,
+      "rewards/rejected": -5.309985160827637,
+      "sft_loss": 4.4433393478393555,
+      "step": 1825
+    },
+    {
+      "epoch": 0.9794279979929754,
+      "grad_norm": 0.9287089525439055,
+      "learning_rate": 8.51731666796467e-07,
+      "logits/chosen": -0.3933059573173523,
+      "logits/rejected": -0.328712522983551,
+      "logps/chosen": -4.452856540679932,
+      "logps/rejected": -4.835246562957764,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.452856540679932,
+      "rewards/margins": 0.38238975405693054,
+      "rewards/rejected": -4.835246562957764,
+      "sft_loss": 4.1753621101379395,
+      "step": 1830
+    },
+    {
+      "epoch": 0.982104030774377,
+      "grad_norm": 0.5941262114362317,
+      "learning_rate": 8.506230671236254e-07,
+      "logits/chosen": -0.5021569132804871,
+      "logits/rejected": -0.39401504397392273,
+      "logps/chosen": -4.731034278869629,
+      "logps/rejected": -5.092568397521973,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.731034278869629,
+      "rewards/margins": 0.3615338206291199,
+      "rewards/rejected": -5.092568397521973,
+      "sft_loss": 4.504293441772461,
+      "step": 1835
+    },
+    {
+      "epoch": 0.9847800635557785,
+      "grad_norm": 0.34993440604791326,
+      "learning_rate": 8.495110657042488e-07,
+      "logits/chosen": -0.47997450828552246,
+      "logits/rejected": -0.2725691795349121,
+      "logps/chosen": -4.54381799697876,
+      "logps/rejected": -5.032081127166748,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.54381799697876,
+      "rewards/margins": 0.4882632791996002,
+      "rewards/rejected": -5.032081127166748,
+      "sft_loss": 4.36675500869751,
+      "step": 1840
+    },
+    {
+      "epoch": 0.9874560963371801,
+      "grad_norm": 0.4308703258389709,
+      "learning_rate": 8.483956733269799e-07,
+      "logits/chosen": -0.4384092688560486,
+      "logits/rejected": -0.3426581919193268,
+      "logps/chosen": -4.591732978820801,
+      "logps/rejected": -4.909899711608887,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.591732978820801,
+      "rewards/margins": 0.31816643476486206,
+      "rewards/rejected": -4.909899711608887,
+      "sft_loss": 4.256570816040039,
+      "step": 1845
+    },
+    {
+      "epoch": 0.9901321291185817,
+      "grad_norm": 0.3473658903289487,
+      "learning_rate": 8.472769008133602e-07,
+      "logits/chosen": -0.6959007978439331,
+      "logits/rejected": -0.5554211735725403,
+      "logps/chosen": -4.5757036209106445,
+      "logps/rejected": -4.93710994720459,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.5757036209106445,
+      "rewards/margins": 0.3614066243171692,
+      "rewards/rejected": -4.93710994720459,
+      "sft_loss": 4.2838358879089355,
+      "step": 1850
+    },
+    {
+      "epoch": 0.9928081618999832,
+      "grad_norm": 0.4555695972184036,
+      "learning_rate": 8.461547590177259e-07,
+      "logits/chosen": -0.557471513748169,
+      "logits/rejected": -0.37859243154525757,
+      "logps/chosen": -4.733495712280273,
+      "logps/rejected": -5.202332973480225,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.733495712280273,
+      "rewards/margins": 0.4688374996185303,
+      "rewards/rejected": -5.202332973480225,
+      "sft_loss": 4.25360631942749,
+      "step": 1855
+    },
+    {
+      "epoch": 0.9954841946813848,
+      "grad_norm": 0.43279035633127416,
+      "learning_rate": 8.450292588271014e-07,
+      "logits/chosen": -0.604584276676178,
+      "logits/rejected": -0.4833999276161194,
+      "logps/chosen": -4.606520652770996,
+      "logps/rejected": -5.053445339202881,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.606520652770996,
+      "rewards/margins": 0.44692516326904297,
+      "rewards/rejected": -5.053445339202881,
+      "sft_loss": 4.216244220733643,
+      "step": 1860
+    },
+    {
+      "epoch": 0.9981602274627864,
+      "grad_norm": 0.5329839703813494,
+      "learning_rate": 8.439004111610945e-07,
+      "logits/chosen": -0.5827174782752991,
+      "logits/rejected": -0.5101505517959595,
+      "logps/chosen": -4.813737869262695,
+      "logps/rejected": -5.12241268157959,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.813737869262695,
+      "rewards/margins": 0.30867481231689453,
+      "rewards/rejected": -5.12241268157959,
+      "sft_loss": 4.392908573150635,
+      "step": 1865
+    },
+    {
+      "epoch": 1.000836260244188,
+      "grad_norm": 0.3367694694066443,
+      "learning_rate": 8.427682269717901e-07,
+      "logits/chosen": -0.6478679776191711,
+      "logits/rejected": -0.4914635717868805,
+      "logps/chosen": -4.561735153198242,
+      "logps/rejected": -5.053101539611816,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.561735153198242,
+      "rewards/margins": 0.49136656522750854,
+      "rewards/rejected": -5.053101539611816,
+      "sft_loss": 4.242537498474121,
+      "step": 1870
+    },
+    {
+      "epoch": 1.0035122930255895,
+      "grad_norm": 0.5834097301481967,
+      "learning_rate": 8.416327172436446e-07,
+      "logits/chosen": -0.8005746603012085,
+      "logits/rejected": -0.6261088848114014,
+      "logps/chosen": -4.756998538970947,
+      "logps/rejected": -5.0415496826171875,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.756998538970947,
+      "rewards/margins": 0.28455111384391785,
+      "rewards/rejected": -5.0415496826171875,
+      "sft_loss": 4.461302757263184,
+      "step": 1875
+    },
+    {
+      "epoch": 1.0061883258069912,
+      "grad_norm": 0.3973798220767986,
+      "learning_rate": 8.404938929933778e-07,
+      "logits/chosen": -0.6104685664176941,
+      "logits/rejected": -0.4688517153263092,
+      "logps/chosen": -4.621617794036865,
+      "logps/rejected": -5.099360466003418,
+      "loss": 0.052,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.621617794036865,
+      "rewards/margins": 0.47774267196655273,
+      "rewards/rejected": -5.099360466003418,
+      "sft_loss": 4.235894203186035,
+      "step": 1880
+    },
+    {
+      "epoch": 1.0088643585883927,
+      "grad_norm": 0.3540909470534527,
+      "learning_rate": 8.39351765269868e-07,
+      "logits/chosen": -0.6760979890823364,
+      "logits/rejected": -0.609889805316925,
+      "logps/chosen": -4.65677547454834,
+      "logps/rejected": -5.0093488693237305,
+      "loss": 0.0539,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.65677547454834,
+      "rewards/margins": 0.3525732159614563,
+      "rewards/rejected": -5.0093488693237305,
+      "sft_loss": 4.3292412757873535,
+      "step": 1885
+    },
+    {
+      "epoch": 1.0115403913697942,
+      "grad_norm": 0.7267303197089038,
+      "learning_rate": 8.382063451540431e-07,
+      "logits/chosen": -0.7155488729476929,
+      "logits/rejected": -0.45979467034339905,
+      "logps/chosen": -4.453604698181152,
+      "logps/rejected": -4.879714488983154,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.453604698181152,
+      "rewards/margins": 0.4261098802089691,
+      "rewards/rejected": -4.879714488983154,
+      "sft_loss": 4.265443801879883,
+      "step": 1890
+    },
+    {
+      "epoch": 1.014216424151196,
+      "grad_norm": 0.4939611442573637,
+      "learning_rate": 8.370576437587742e-07,
+      "logits/chosen": -0.6435521841049194,
+      "logits/rejected": -0.6209043264389038,
+      "logps/chosen": -4.716995716094971,
+      "logps/rejected": -5.006557464599609,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.716995716094971,
+      "rewards/margins": 0.28956204652786255,
+      "rewards/rejected": -5.006557464599609,
+      "sft_loss": 4.35626220703125,
+      "step": 1895
+    },
+    {
+      "epoch": 1.0168924569325974,
+      "grad_norm": 0.41701206009092195,
+      "learning_rate": 8.359056722287674e-07,
+      "logits/chosen": -0.7903778553009033,
+      "logits/rejected": -0.4564805030822754,
+      "logps/chosen": -4.639558792114258,
+      "logps/rejected": -5.135783672332764,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.639558792114258,
+      "rewards/margins": 0.49622488021850586,
+      "rewards/rejected": -5.135783672332764,
+      "sft_loss": 4.399038791656494,
+      "step": 1900
+    },
+    {
+      "epoch": 1.019568489713999,
+      "grad_norm": 0.7172297640459512,
+      "learning_rate": 8.347504417404553e-07,
+      "logits/chosen": -0.5581150054931641,
+      "logits/rejected": -0.4032842218875885,
+      "logps/chosen": -4.544151306152344,
+      "logps/rejected": -4.86648416519165,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.544151306152344,
+      "rewards/margins": 0.32233288884162903,
+      "rewards/rejected": -4.86648416519165,
+      "sft_loss": 4.188483238220215,
+      "step": 1905
+    },
+    {
+      "epoch": 1.0222445224954007,
+      "grad_norm": 0.5879181967847867,
+      "learning_rate": 8.335919635018893e-07,
+      "logits/chosen": -0.7893685102462769,
+      "logits/rejected": -0.6554333567619324,
+      "logps/chosen": -4.7575788497924805,
+      "logps/rejected": -5.040469169616699,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.7575788497924805,
+      "rewards/margins": 0.2828896641731262,
+      "rewards/rejected": -5.040469169616699,
+      "sft_loss": 4.464777946472168,
+      "step": 1910
+    },
+    {
+      "epoch": 1.0249205552768021,
+      "grad_norm": 0.43231433946425346,
+      "learning_rate": 8.324302487526303e-07,
+      "logits/chosen": -0.7919416427612305,
+      "logits/rejected": -0.6639026403427124,
+      "logps/chosen": -4.676050186157227,
+      "logps/rejected": -5.0417985916137695,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.676050186157227,
+      "rewards/margins": 0.36574864387512207,
+      "rewards/rejected": -5.0417985916137695,
+      "sft_loss": 4.378249168395996,
+      "step": 1915
+    },
+    {
+      "epoch": 1.0275965880582036,
+      "grad_norm": 0.37404157282655504,
+      "learning_rate": 8.312653087636398e-07,
+      "logits/chosen": -0.7558452486991882,
+      "logits/rejected": -0.6526741981506348,
+      "logps/chosen": -4.579874515533447,
+      "logps/rejected": -4.968623161315918,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.579874515533447,
+      "rewards/margins": 0.38874801993370056,
+      "rewards/rejected": -4.968623161315918,
+      "sft_loss": 4.276757717132568,
+      "step": 1920
+    },
+    {
+      "epoch": 1.0302726208396054,
+      "grad_norm": 0.36303918679572256,
+      "learning_rate": 8.300971548371711e-07,
+      "logits/chosen": -0.883111834526062,
+      "logits/rejected": -0.602270781993866,
+      "logps/chosen": -4.452870845794678,
+      "logps/rejected": -4.911247730255127,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.452870845794678,
+      "rewards/margins": 0.45837679505348206,
+      "rewards/rejected": -4.911247730255127,
+      "sft_loss": 4.131048202514648,
+      "step": 1925
+    },
+    {
+      "epoch": 1.0329486536210069,
+      "grad_norm": 0.490560671457728,
+      "learning_rate": 8.289257983066582e-07,
+      "logits/chosen": -0.7516878843307495,
+      "logits/rejected": -0.5508066415786743,
+      "logps/chosen": -4.64019775390625,
+      "logps/rejected": -5.095065593719482,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.64019775390625,
+      "rewards/margins": 0.4548683166503906,
+      "rewards/rejected": -5.095065593719482,
+      "sft_loss": 4.342806339263916,
+      "step": 1930
+    },
+    {
+      "epoch": 1.0356246864024083,
+      "grad_norm": 0.2893276344274073,
+      "learning_rate": 8.277512505366077e-07,
+      "logits/chosen": -0.8057788014411926,
+      "logits/rejected": -0.5403832197189331,
+      "logps/chosen": -4.649588108062744,
+      "logps/rejected": -5.086615085601807,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.649588108062744,
+      "rewards/margins": 0.4370269179344177,
+      "rewards/rejected": -5.086615085601807,
+      "sft_loss": 4.376861095428467,
+      "step": 1935
+    },
+    {
+      "epoch": 1.03830071918381,
+      "grad_norm": 0.7350489462981791,
+      "learning_rate": 8.265735229224868e-07,
+      "logits/chosen": -0.6775953769683838,
+      "logits/rejected": -0.5745862722396851,
+      "logps/chosen": -4.5659379959106445,
+      "logps/rejected": -5.098782062530518,
+      "loss": 0.052,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.5659379959106445,
+      "rewards/margins": 0.5328438878059387,
+      "rewards/rejected": -5.098782062530518,
+      "sft_loss": 4.174136161804199,
+      "step": 1940
+    },
+    {
+      "epoch": 1.0409767519652116,
+      "grad_norm": 0.4516735988201357,
+      "learning_rate": 8.253926268906144e-07,
+      "logits/chosen": -0.7164157629013062,
+      "logits/rejected": -0.5405601859092712,
+      "logps/chosen": -4.749704837799072,
+      "logps/rejected": -5.371539115905762,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.749704837799072,
+      "rewards/margins": 0.6218348145484924,
+      "rewards/rejected": -5.371539115905762,
+      "sft_loss": 4.391979694366455,
+      "step": 1945
+    },
+    {
+      "epoch": 1.043652784746613,
+      "grad_norm": 0.3510630604415655,
+      "learning_rate": 8.242085738980487e-07,
+      "logits/chosen": -0.5420058965682983,
+      "logits/rejected": -0.2874351739883423,
+      "logps/chosen": -4.528346061706543,
+      "logps/rejected": -5.030544281005859,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.528346061706543,
+      "rewards/margins": 0.5021986961364746,
+      "rewards/rejected": -5.030544281005859,
+      "sft_loss": 4.249731540679932,
+      "step": 1950
+    },
+    {
+      "epoch": 1.0463288175280148,
+      "grad_norm": 0.5538265414870637,
+      "learning_rate": 8.230213754324772e-07,
+      "logits/chosen": -0.6377596259117126,
+      "logits/rejected": -0.5630909204483032,
+      "logps/chosen": -4.5644450187683105,
+      "logps/rejected": -4.965079307556152,
+      "loss": 0.0539,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.5644450187683105,
+      "rewards/margins": 0.40063467621803284,
+      "rewards/rejected": -4.965079307556152,
+      "sft_loss": 4.360657215118408,
+      "step": 1955
+    },
+    {
+      "epoch": 1.0490048503094163,
+      "grad_norm": 0.2949614545223873,
+      "learning_rate": 8.218310430121045e-07,
+      "logits/chosen": -0.5675156712532043,
+      "logits/rejected": -0.5481168031692505,
+      "logps/chosen": -4.707152366638184,
+      "logps/rejected": -4.992379188537598,
+      "loss": 0.0539,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -4.707152366638184,
+      "rewards/margins": 0.2852264642715454,
+      "rewards/rejected": -4.992379188537598,
+      "sft_loss": 4.478783130645752,
+      "step": 1960
+    },
+    {
+      "epoch": 1.051680883090818,
+      "grad_norm": 0.5543113807704609,
+      "learning_rate": 8.20637588185541e-07,
+      "logits/chosen": -0.5608028173446655,
+      "logits/rejected": -0.46988582611083984,
+      "logps/chosen": -4.442027568817139,
+      "logps/rejected": -4.943470478057861,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.442027568817139,
+      "rewards/margins": 0.5014427900314331,
+      "rewards/rejected": -4.943470478057861,
+      "sft_loss": 4.245416164398193,
+      "step": 1965
+    },
+    {
+      "epoch": 1.0543569158722195,
+      "grad_norm": 0.2866117529311254,
+      "learning_rate": 8.194410225316906e-07,
+      "logits/chosen": -0.7090498208999634,
+      "logits/rejected": -0.5142877101898193,
+      "logps/chosen": -4.616601467132568,
+      "logps/rejected": -5.014645576477051,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.616601467132568,
+      "rewards/margins": 0.39804330468177795,
+      "rewards/rejected": -5.014645576477051,
+      "sft_loss": 4.298870086669922,
+      "step": 1970
+    },
+    {
+      "epoch": 1.057032948653621,
+      "grad_norm": 0.35295190473633326,
+      "learning_rate": 8.182413576596385e-07,
+      "logits/chosen": -0.5405132174491882,
+      "logits/rejected": -0.48180103302001953,
+      "logps/chosen": -4.607327938079834,
+      "logps/rejected": -5.084053039550781,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.607327938079834,
+      "rewards/margins": 0.4767250418663025,
+      "rewards/rejected": -5.084053039550781,
+      "sft_loss": 4.384486675262451,
+      "step": 1975
+    },
+    {
+      "epoch": 1.0597089814350227,
+      "grad_norm": 0.3776636814624771,
+      "learning_rate": 8.170386052085389e-07,
+      "logits/chosen": -0.5723873376846313,
+      "logits/rejected": -0.44267210364341736,
+      "logps/chosen": -4.660275459289551,
+      "logps/rejected": -5.051526069641113,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.660275459289551,
+      "rewards/margins": 0.39125025272369385,
+      "rewards/rejected": -5.051526069641113,
+      "sft_loss": 4.37464714050293,
+      "step": 1980
+    },
+    {
+      "epoch": 1.0623850142164242,
+      "grad_norm": 0.5159615230619127,
+      "learning_rate": 8.158327768475008e-07,
+      "logits/chosen": -0.6213208436965942,
+      "logits/rejected": -0.4670190215110779,
+      "logps/chosen": -4.665583610534668,
+      "logps/rejected": -4.967374801635742,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.665583610534668,
+      "rewards/margins": 0.30179107189178467,
+      "rewards/rejected": -4.967374801635742,
+      "sft_loss": 4.264622211456299,
+      "step": 1985
+    },
+    {
+      "epoch": 1.0650610469978257,
+      "grad_norm": 0.44400302792011925,
+      "learning_rate": 8.146238842754767e-07,
+      "logits/chosen": -0.6752325296401978,
+      "logits/rejected": -0.5508004426956177,
+      "logps/chosen": -4.6823554039001465,
+      "logps/rejected": -5.140440940856934,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.6823554039001465,
+      "rewards/margins": 0.45808520913124084,
+      "rewards/rejected": -5.140440940856934,
+      "sft_loss": 4.321152210235596,
+      "step": 1990
+    },
+    {
+      "epoch": 1.0677370797792274,
+      "grad_norm": 0.7057418884353844,
+      "learning_rate": 8.134119392211476e-07,
+      "logits/chosen": -0.5761233568191528,
+      "logits/rejected": -0.38899824023246765,
+      "logps/chosen": -4.757379055023193,
+      "logps/rejected": -5.293605804443359,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.757379055023193,
+      "rewards/margins": 0.5362268686294556,
+      "rewards/rejected": -5.293605804443359,
+      "sft_loss": 4.403469085693359,
+      "step": 1995
+    },
+    {
+      "epoch": 1.0704131125606289,
+      "grad_norm": 0.49240902562783034,
+      "learning_rate": 8.121969534428094e-07,
+      "logits/chosen": -0.7272932529449463,
+      "logits/rejected": -0.5523107051849365,
+      "logps/chosen": -4.743283271789551,
+      "logps/rejected": -5.234428882598877,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.743283271789551,
+      "rewards/margins": 0.4911455512046814,
+      "rewards/rejected": -5.234428882598877,
+      "sft_loss": 4.387631416320801,
+      "step": 2000
+    },
+    {
+      "epoch": 1.0704131125606289,
+      "eval_logits/chosen": -0.23999834060668945,
+      "eval_logits/rejected": -0.15501756966114044,
+      "eval_logps/chosen": -4.426951885223389,
+      "eval_logps/rejected": -4.892354488372803,
+      "eval_loss": 0.051702603697776794,
+      "eval_rewards/accuracies": 0.6468842625617981,
+      "eval_rewards/chosen": -4.426951885223389,
+      "eval_rewards/margins": 0.4654025435447693,
+      "eval_rewards/rejected": -4.892354488372803,
+      "eval_runtime": 43.2906,
+      "eval_samples_per_second": 31.069,
+      "eval_sft_loss": 3.9661672115325928,
+      "eval_steps_per_second": 7.785,
+      "step": 2000
+    },
+    {
+      "epoch": 1.0730891453420304,
+      "grad_norm": 0.4600706793047822,
+      "learning_rate": 8.109789387282599e-07,
+      "logits/chosen": -0.6717931032180786,
+      "logits/rejected": -0.6304608583450317,
+      "logps/chosen": -4.575514793395996,
+      "logps/rejected": -4.920340538024902,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.575514793395996,
+      "rewards/margins": 0.3448256552219391,
+      "rewards/rejected": -4.920340538024902,
+      "sft_loss": 4.244610786437988,
+      "step": 2005
+    },
+    {
+      "epoch": 1.075765178123432,
+      "grad_norm": 0.3970379766667623,
+      "learning_rate": 8.097579068946827e-07,
+      "logits/chosen": -0.7990378141403198,
+      "logits/rejected": -0.6704200506210327,
+      "logps/chosen": -4.71152400970459,
+      "logps/rejected": -5.189643859863281,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.71152400970459,
+      "rewards/margins": 0.4781200885772705,
+      "rewards/rejected": -5.189643859863281,
+      "sft_loss": 4.398639678955078,
+      "step": 2010
+    },
+    {
+      "epoch": 1.0784412109048336,
+      "grad_norm": 0.5944728609383647,
+      "learning_rate": 8.085338697885344e-07,
+      "logits/chosen": -0.7603497505187988,
+      "logits/rejected": -0.6357791423797607,
+      "logps/chosen": -4.704463481903076,
+      "logps/rejected": -5.05355978012085,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.704463481903076,
+      "rewards/margins": 0.3490960896015167,
+      "rewards/rejected": -5.05355978012085,
+      "sft_loss": 4.395869255065918,
+      "step": 2015
+    },
+    {
+      "epoch": 1.081117243686235,
+      "grad_norm": 0.37605343362394694,
+      "learning_rate": 8.073068392854282e-07,
+      "logits/chosen": -0.826943039894104,
+      "logits/rejected": -0.5462093949317932,
+      "logps/chosen": -4.333527565002441,
+      "logps/rejected": -4.886292457580566,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.333527565002441,
+      "rewards/margins": 0.5527652502059937,
+      "rewards/rejected": -4.886292457580566,
+      "sft_loss": 4.111708641052246,
+      "step": 2020
+    },
+    {
+      "epoch": 1.0837932764676368,
+      "grad_norm": 0.7179662433757041,
+      "learning_rate": 8.060768272900193e-07,
+      "logits/chosen": -0.6332185864448547,
+      "logits/rejected": -0.4485379755496979,
+      "logps/chosen": -4.430613040924072,
+      "logps/rejected": -4.984936237335205,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.430613040924072,
+      "rewards/margins": 0.5543233156204224,
+      "rewards/rejected": -4.984936237335205,
+      "sft_loss": 4.166750907897949,
+      "step": 2025
+    },
+    {
+      "epoch": 1.0864693092490383,
+      "grad_norm": 0.35942591017215597,
+      "learning_rate": 8.0484384573589e-07,
+      "logits/chosen": -0.6678962111473083,
+      "logits/rejected": -0.6808444857597351,
+      "logps/chosen": -4.590447425842285,
+      "logps/rejected": -4.973687648773193,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.590447425842285,
+      "rewards/margins": 0.3832399249076843,
+      "rewards/rejected": -4.973687648773193,
+      "sft_loss": 4.327738285064697,
+      "step": 2030
+    },
+    {
+      "epoch": 1.0891453420304398,
+      "grad_norm": 0.3819578692381291,
+      "learning_rate": 8.03607906585432e-07,
+      "logits/chosen": -0.7403281331062317,
+      "logits/rejected": -0.5643646121025085,
+      "logps/chosen": -4.80985164642334,
+      "logps/rejected": -5.229228496551514,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.80985164642334,
+      "rewards/margins": 0.4193764328956604,
+      "rewards/rejected": -5.229228496551514,
+      "sft_loss": 4.544407844543457,
+      "step": 2035
+    },
+    {
+      "epoch": 1.0918213748118415,
+      "grad_norm": 0.4776251797363461,
+      "learning_rate": 8.023690218297329e-07,
+      "logits/chosen": -0.6995172500610352,
+      "logits/rejected": -0.6782066226005554,
+      "logps/chosen": -4.589086532592773,
+      "logps/rejected": -4.949246406555176,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.589086532592773,
+      "rewards/margins": 0.36015990376472473,
+      "rewards/rejected": -4.949246406555176,
+      "sft_loss": 4.224146842956543,
+      "step": 2040
+    },
+    {
+      "epoch": 1.094497407593243,
+      "grad_norm": 0.7479474801927108,
+      "learning_rate": 8.01127203488458e-07,
+      "logits/chosen": -0.5383197665214539,
+      "logits/rejected": -0.48996859788894653,
+      "logps/chosen": -4.540228366851807,
+      "logps/rejected": -5.0145792961120605,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.540228366851807,
+      "rewards/margins": 0.47435110807418823,
+      "rewards/rejected": -5.0145792961120605,
+      "sft_loss": 4.176971435546875,
+      "step": 2045
+    },
+    {
+      "epoch": 1.0971734403746445,
+      "grad_norm": 0.4776403747608541,
+      "learning_rate": 7.998824636097339e-07,
+      "logits/chosen": -0.6831840872764587,
+      "logits/rejected": -0.5318336486816406,
+      "logps/chosen": -4.693176746368408,
+      "logps/rejected": -5.142681121826172,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.693176746368408,
+      "rewards/margins": 0.44950443506240845,
+      "rewards/rejected": -5.142681121826172,
+      "sft_loss": 4.48852014541626,
+      "step": 2050
+    },
+    {
+      "epoch": 1.0998494731560462,
+      "grad_norm": 0.5140746201417399,
+      "learning_rate": 7.986348142700328e-07,
+      "logits/chosen": -0.7236698269844055,
+      "logits/rejected": -0.5705364346504211,
+      "logps/chosen": -4.719154357910156,
+      "logps/rejected": -5.148931980133057,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.719154357910156,
+      "rewards/margins": 0.42977744340896606,
+      "rewards/rejected": -5.148931980133057,
+      "sft_loss": 4.52241325378418,
+      "step": 2055
+    },
+    {
+      "epoch": 1.1025255059374477,
+      "grad_norm": 0.4179464858196893,
+      "learning_rate": 7.973842675740539e-07,
+      "logits/chosen": -0.609150767326355,
+      "logits/rejected": -0.5410366058349609,
+      "logps/chosen": -4.482192039489746,
+      "logps/rejected": -4.9732208251953125,
+      "loss": 0.052,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.482192039489746,
+      "rewards/margins": 0.49102896451950073,
+      "rewards/rejected": -4.9732208251953125,
+      "sft_loss": 4.188296318054199,
+      "step": 2060
+    },
+    {
+      "epoch": 1.1052015387188494,
+      "grad_norm": 0.5180053361561886,
+      "learning_rate": 7.961308356546066e-07,
+      "logits/chosen": -0.7119373083114624,
+      "logits/rejected": -0.5679913759231567,
+      "logps/chosen": -4.229228973388672,
+      "logps/rejected": -4.735593318939209,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.229228973388672,
+      "rewards/margins": 0.5063642263412476,
+      "rewards/rejected": -4.735593318939209,
+      "sft_loss": 3.9967262744903564,
+      "step": 2065
+    },
+    {
+      "epoch": 1.107877571500251,
+      "grad_norm": 0.46718871726056116,
+      "learning_rate": 7.948745306724931e-07,
+      "logits/chosen": -0.7739778757095337,
+      "logits/rejected": -0.6170490980148315,
+      "logps/chosen": -4.635968208312988,
+      "logps/rejected": -5.1852192878723145,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.635968208312988,
+      "rewards/margins": 0.549250602722168,
+      "rewards/rejected": -5.1852192878723145,
+      "sft_loss": 4.369393825531006,
+      "step": 2070
+    },
+    {
+      "epoch": 1.1105536042816524,
+      "grad_norm": 0.3984587230629792,
+      "learning_rate": 7.936153648163897e-07,
+      "logits/chosen": -0.8065184354782104,
+      "logits/rejected": -0.7064257860183716,
+      "logps/chosen": -4.752967834472656,
+      "logps/rejected": -5.092961311340332,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.752967834472656,
+      "rewards/margins": 0.33999359607696533,
+      "rewards/rejected": -5.092961311340332,
+      "sft_loss": 4.481800079345703,
+      "step": 2075
+    },
+    {
+      "epoch": 1.1132296370630541,
+      "grad_norm": 0.5695032759468597,
+      "learning_rate": 7.92353350302729e-07,
+      "logits/chosen": -0.7206646800041199,
+      "logits/rejected": -0.548992931842804,
+      "logps/chosen": -4.335890293121338,
+      "logps/rejected": -4.820664882659912,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.335890293121338,
+      "rewards/margins": 0.48477450013160706,
+      "rewards/rejected": -4.820664882659912,
+      "sft_loss": 4.050368785858154,
+      "step": 2080
+    },
+    {
+      "epoch": 1.1159056698444556,
+      "grad_norm": 0.5829877306762072,
+      "learning_rate": 7.910884993755816e-07,
+      "logits/chosen": -0.7600020170211792,
+      "logits/rejected": -0.6812421679496765,
+      "logps/chosen": -4.553534984588623,
+      "logps/rejected": -5.294806480407715,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.553534984588623,
+      "rewards/margins": 0.741270899772644,
+      "rewards/rejected": -5.294806480407715,
+      "sft_loss": 4.326718807220459,
+      "step": 2085
+    },
+    {
+      "epoch": 1.118581702625857,
+      "grad_norm": 0.8262505274696786,
+      "learning_rate": 7.898208243065367e-07,
+      "logits/chosen": -0.7206606864929199,
+      "logits/rejected": -0.7447828054428101,
+      "logps/chosen": -4.787585258483887,
+      "logps/rejected": -5.069046497344971,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.787585258483887,
+      "rewards/margins": 0.2814616560935974,
+      "rewards/rejected": -5.069046497344971,
+      "sft_loss": 4.394356727600098,
+      "step": 2090
+    },
+    {
+      "epoch": 1.1212577354072588,
+      "grad_norm": 0.5172306289125376,
+      "learning_rate": 7.88550337394583e-07,
+      "logits/chosen": -0.8299128413200378,
+      "logits/rejected": -0.6698902249336243,
+      "logps/chosen": -4.428835391998291,
+      "logps/rejected": -4.9404144287109375,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.428835391998291,
+      "rewards/margins": 0.5115790367126465,
+      "rewards/rejected": -4.9404144287109375,
+      "sft_loss": 4.22437047958374,
+      "step": 2095
+    },
+    {
+      "epoch": 1.1239337681886603,
+      "grad_norm": 0.3443635655516794,
+      "learning_rate": 7.872770509659905e-07,
+      "logits/chosen": -0.6373104453086853,
+      "logits/rejected": -0.5988988876342773,
+      "logps/chosen": -4.619525909423828,
+      "logps/rejected": -4.976244926452637,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.619525909423828,
+      "rewards/margins": 0.3567189574241638,
+      "rewards/rejected": -4.976244926452637,
+      "sft_loss": 4.2509589195251465,
+      "step": 2100
+    },
+    {
+      "epoch": 1.1266098009700618,
+      "grad_norm": 0.3976039731943874,
+      "learning_rate": 7.860009773741896e-07,
+      "logits/chosen": -0.5825362801551819,
+      "logits/rejected": -0.4224637448787689,
+      "logps/chosen": -4.629540920257568,
+      "logps/rejected": -5.204381465911865,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.629540920257568,
+      "rewards/margins": 0.5748408436775208,
+      "rewards/rejected": -5.204381465911865,
+      "sft_loss": 4.34817361831665,
+      "step": 2105
+    },
+    {
+      "epoch": 1.1292858337514635,
+      "grad_norm": 0.39212041229796935,
+      "learning_rate": 7.84722128999652e-07,
+      "logits/chosen": -0.6080501675605774,
+      "logits/rejected": -0.45994147658348083,
+      "logps/chosen": -4.713656425476074,
+      "logps/rejected": -5.265693187713623,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.713656425476074,
+      "rewards/margins": 0.5520361661911011,
+      "rewards/rejected": -5.265693187713623,
+      "sft_loss": 4.4130964279174805,
+      "step": 2110
+    },
+    {
+      "epoch": 1.131961866532865,
+      "grad_norm": 1.2673650048653755,
+      "learning_rate": 7.834405182497699e-07,
+      "logits/chosen": -0.49251309037208557,
+      "logits/rejected": -0.4597319960594177,
+      "logps/chosen": -4.524658203125,
+      "logps/rejected": -4.95531702041626,
+      "loss": 0.0544,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.524658203125,
+      "rewards/margins": 0.43065857887268066,
+      "rewards/rejected": -4.95531702041626,
+      "sft_loss": 4.2953691482543945,
+      "step": 2115
+    },
+    {
+      "epoch": 1.1346378993142665,
+      "grad_norm": 0.5472533304467926,
+      "learning_rate": 7.821561575587368e-07,
+      "logits/chosen": -0.632826566696167,
+      "logits/rejected": -0.6065362095832825,
+      "logps/chosen": -4.617916584014893,
+      "logps/rejected": -4.967799663543701,
+      "loss": 0.0539,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.617916584014893,
+      "rewards/margins": 0.34988293051719666,
+      "rewards/rejected": -4.967799663543701,
+      "sft_loss": 4.453722953796387,
+      "step": 2120
+    },
+    {
+      "epoch": 1.1373139320956682,
+      "grad_norm": 0.277069241699628,
+      "learning_rate": 7.808690593874254e-07,
+      "logits/chosen": -0.7417432069778442,
+      "logits/rejected": -0.6385709047317505,
+      "logps/chosen": -4.6788010597229,
+      "logps/rejected": -5.170589923858643,
+      "loss": 0.0548,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.6788010597229,
+      "rewards/margins": 0.491788774728775,
+      "rewards/rejected": -5.170589923858643,
+      "sft_loss": 4.516565322875977,
+      "step": 2125
+    },
+    {
+      "epoch": 1.1399899648770697,
+      "grad_norm": 0.47122826813421975,
+      "learning_rate": 7.79579236223268e-07,
+      "logits/chosen": -0.6222350001335144,
+      "logits/rejected": -0.3234289288520813,
+      "logps/chosen": -4.440484523773193,
+      "logps/rejected": -4.997524261474609,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.440484523773193,
+      "rewards/margins": 0.5570399165153503,
+      "rewards/rejected": -4.997524261474609,
+      "sft_loss": 4.1651482582092285,
+      "step": 2130
+    },
+    {
+      "epoch": 1.1426659976584714,
+      "grad_norm": 0.44261863106415245,
+      "learning_rate": 7.782867005801346e-07,
+      "logits/chosen": -0.6503061056137085,
+      "logits/rejected": -0.42080339789390564,
+      "logps/chosen": -4.290419578552246,
+      "logps/rejected": -4.850780963897705,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.290419578552246,
+      "rewards/margins": 0.560361385345459,
+      "rewards/rejected": -4.850780963897705,
+      "sft_loss": 4.068234443664551,
+      "step": 2135
+    },
+    {
+      "epoch": 1.145342030439873,
+      "grad_norm": 0.6257258204652094,
+      "learning_rate": 7.769914649982117e-07,
+      "logits/chosen": -0.6751757264137268,
+      "logits/rejected": -0.5070086717605591,
+      "logps/chosen": -4.67188024520874,
+      "logps/rejected": -5.1316142082214355,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.67188024520874,
+      "rewards/margins": 0.4597338140010834,
+      "rewards/rejected": -5.1316142082214355,
+      "sft_loss": 4.425201416015625,
+      "step": 2140
+    },
+    {
+      "epoch": 1.1480180632212744,
+      "grad_norm": 0.5302938713487861,
+      "learning_rate": 7.756935420438803e-07,
+      "logits/chosen": -0.6334782838821411,
+      "logits/rejected": -0.5502496957778931,
+      "logps/chosen": -4.623457908630371,
+      "logps/rejected": -5.174169540405273,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.623457908630371,
+      "rewards/margins": 0.5507121086120605,
+      "rewards/rejected": -5.174169540405273,
+      "sft_loss": 4.292878150939941,
+      "step": 2145
+    },
+    {
+      "epoch": 1.1506940960026761,
+      "grad_norm": 0.6070596328759367,
+      "learning_rate": 7.743929443095951e-07,
+      "logits/chosen": -0.6257916688919067,
+      "logits/rejected": -0.5801526308059692,
+      "logps/chosen": -4.365163326263428,
+      "logps/rejected": -4.9333815574646,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.365163326263428,
+      "rewards/margins": 0.568219006061554,
+      "rewards/rejected": -4.9333815574646,
+      "sft_loss": 4.0889482498168945,
+      "step": 2150
+    },
+    {
+      "epoch": 1.1533701287840776,
+      "grad_norm": 0.6839084267850211,
+      "learning_rate": 7.730896844137609e-07,
+      "logits/chosen": -0.6672028303146362,
+      "logits/rejected": -0.5766544342041016,
+      "logps/chosen": -4.858401775360107,
+      "logps/rejected": -5.171161651611328,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -4.858401775360107,
+      "rewards/margins": 0.3127599358558655,
+      "rewards/rejected": -5.171161651611328,
+      "sft_loss": 4.545053005218506,
+      "step": 2155
+    },
+    {
+      "epoch": 1.1560461615654791,
+      "grad_norm": 0.3568908807824978,
+      "learning_rate": 7.717837750006106e-07,
+      "logits/chosen": -0.7871010303497314,
+      "logits/rejected": -0.7272329330444336,
+      "logps/chosen": -4.782454490661621,
+      "logps/rejected": -5.240635871887207,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.782454490661621,
+      "rewards/margins": 0.45818084478378296,
+      "rewards/rejected": -5.240635871887207,
+      "sft_loss": 4.542524337768555,
+      "step": 2160
+    },
+    {
+      "epoch": 1.1587221943468808,
+      "grad_norm": 0.5990662498663287,
+      "learning_rate": 7.704752287400832e-07,
+      "logits/chosen": -0.73150235414505,
+      "logits/rejected": -0.5180791020393372,
+      "logps/chosen": -4.3465166091918945,
+      "logps/rejected": -4.793034076690674,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.3465166091918945,
+      "rewards/margins": 0.44651785492897034,
+      "rewards/rejected": -4.793034076690674,
+      "sft_loss": 4.138575553894043,
+      "step": 2165
+    },
+    {
+      "epoch": 1.1613982271282823,
+      "grad_norm": 0.27564186929080403,
+      "learning_rate": 7.691640583277004e-07,
+      "logits/chosen": -0.6673997640609741,
+      "logits/rejected": -0.48911604285240173,
+      "logps/chosen": -4.416393756866455,
+      "logps/rejected": -4.9265360832214355,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.416393756866455,
+      "rewards/margins": 0.5101426839828491,
+      "rewards/rejected": -4.9265360832214355,
+      "sft_loss": 4.156350135803223,
+      "step": 2170
+    },
+    {
+      "epoch": 1.1640742599096838,
+      "grad_norm": 0.4813794132035719,
+      "learning_rate": 7.678502764844433e-07,
+      "logits/chosen": -0.7979347705841064,
+      "logits/rejected": -0.5554312467575073,
+      "logps/chosen": -4.576869964599609,
+      "logps/rejected": -4.963016986846924,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.576869964599609,
+      "rewards/margins": 0.3861469328403473,
+      "rewards/rejected": -4.963016986846924,
+      "sft_loss": 4.301800727844238,
+      "step": 2175
+    },
+    {
+      "epoch": 1.1667502926910855,
+      "grad_norm": 0.3189556223539962,
+      "learning_rate": 7.665338959566288e-07,
+      "logits/chosen": -0.68720543384552,
+      "logits/rejected": -0.6039665937423706,
+      "logps/chosen": -4.594615459442139,
+      "logps/rejected": -5.138156414031982,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.594615459442139,
+      "rewards/margins": 0.5435405969619751,
+      "rewards/rejected": -5.138156414031982,
+      "sft_loss": 4.311945915222168,
+      "step": 2180
+    },
+    {
+      "epoch": 1.169426325472487,
+      "grad_norm": 0.7522816530936208,
+      "learning_rate": 7.652149295157868e-07,
+      "logits/chosen": -0.5866094827651978,
+      "logits/rejected": -0.40514469146728516,
+      "logps/chosen": -4.510667324066162,
+      "logps/rejected": -4.878499984741211,
+      "loss": 0.053,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.510667324066162,
+      "rewards/margins": 0.3678319752216339,
+      "rewards/rejected": -4.878499984741211,
+      "sft_loss": 4.197674751281738,
+      "step": 2185
+    },
+    {
+      "epoch": 1.1721023582538885,
+      "grad_norm": 0.4463112427262321,
+      "learning_rate": 7.638933899585354e-07,
+      "logits/chosen": -0.44559282064437866,
+      "logits/rejected": -0.45199769735336304,
+      "logps/chosen": -4.532196998596191,
+      "logps/rejected": -5.0780205726623535,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.532196998596191,
+      "rewards/margins": 0.5458240509033203,
+      "rewards/rejected": -5.0780205726623535,
+      "sft_loss": 4.232436180114746,
+      "step": 2190
+    },
+    {
+      "epoch": 1.1747783910352902,
+      "grad_norm": 0.457643428777109,
+      "learning_rate": 7.625692901064573e-07,
+      "logits/chosen": -0.6123597025871277,
+      "logits/rejected": -0.5176479816436768,
+      "logps/chosen": -4.659605026245117,
+      "logps/rejected": -5.072868347167969,
+      "loss": 0.0551,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -4.659605026245117,
+      "rewards/margins": 0.41326403617858887,
+      "rewards/rejected": -5.072868347167969,
+      "sft_loss": 4.42981481552124,
+      "step": 2195
+    },
+    {
+      "epoch": 1.1774544238166917,
+      "grad_norm": 0.39494702241247864,
+      "learning_rate": 7.61242642805975e-07,
+      "logits/chosen": -0.7008353471755981,
+      "logits/rejected": -0.7373430132865906,
+      "logps/chosen": -4.8912858963012695,
+      "logps/rejected": -5.152310848236084,
+      "loss": 0.0543,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -4.8912858963012695,
+      "rewards/margins": 0.2610251009464264,
+      "rewards/rejected": -5.152310848236084,
+      "sft_loss": 4.551990509033203,
+      "step": 2200
+    },
+    {
+      "epoch": 1.1801304565980932,
+      "grad_norm": 0.3579528644218798,
+      "learning_rate": 7.599134609282266e-07,
+      "logits/chosen": -0.8311487436294556,
+      "logits/rejected": -0.5965268015861511,
+      "logps/chosen": -4.442378044128418,
+      "logps/rejected": -4.958862781524658,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.442378044128418,
+      "rewards/margins": 0.516484797000885,
+      "rewards/rejected": -4.958862781524658,
+      "sft_loss": 4.260416030883789,
+      "step": 2205
+    },
+    {
+      "epoch": 1.182806489379495,
+      "grad_norm": 0.44236643382684815,
+      "learning_rate": 7.585817573689402e-07,
+      "logits/chosen": -0.7978760600090027,
+      "logits/rejected": -0.6854046583175659,
+      "logps/chosen": -4.419394016265869,
+      "logps/rejected": -5.014368057250977,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.419394016265869,
+      "rewards/margins": 0.5949746370315552,
+      "rewards/rejected": -5.014368057250977,
+      "sft_loss": 4.203858852386475,
+      "step": 2210
+    },
+    {
+      "epoch": 1.1854825221608964,
+      "grad_norm": 0.4060038941710105,
+      "learning_rate": 7.572475450483098e-07,
+      "logits/chosen": -0.704419732093811,
+      "logits/rejected": -0.5912919044494629,
+      "logps/chosen": -4.481686115264893,
+      "logps/rejected": -5.0178141593933105,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.481686115264893,
+      "rewards/margins": 0.5361284017562866,
+      "rewards/rejected": -5.0178141593933105,
+      "sft_loss": 4.2278852462768555,
+      "step": 2215
+    },
+    {
+      "epoch": 1.188158554942298,
+      "grad_norm": 0.5070512938760626,
+      "learning_rate": 7.559108369108689e-07,
+      "logits/chosen": -0.7109116315841675,
+      "logits/rejected": -0.5894996523857117,
+      "logps/chosen": -4.390748500823975,
+      "logps/rejected": -4.943523406982422,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.390748500823975,
+      "rewards/margins": 0.5527750253677368,
+      "rewards/rejected": -4.943523406982422,
+      "sft_loss": 4.148125648498535,
+      "step": 2220
+    },
+    {
+      "epoch": 1.1908345877236997,
+      "grad_norm": 0.5159637234340495,
+      "learning_rate": 7.54571645925366e-07,
+      "logits/chosen": -0.8002594709396362,
+      "logits/rejected": -0.5314058661460876,
+      "logps/chosen": -4.7372260093688965,
+      "logps/rejected": -5.4383015632629395,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.7372260093688965,
+      "rewards/margins": 0.7010757327079773,
+      "rewards/rejected": -5.4383015632629395,
+      "sft_loss": 4.354250907897949,
+      "step": 2225
+    },
+    {
+      "epoch": 1.1935106205051011,
+      "grad_norm": 0.45824704280534556,
+      "learning_rate": 7.532299850846378e-07,
+      "logits/chosen": -0.7148901224136353,
+      "logits/rejected": -0.5222693681716919,
+      "logps/chosen": -4.647156715393066,
+      "logps/rejected": -5.444609642028809,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.647156715393066,
+      "rewards/margins": 0.7974528074264526,
+      "rewards/rejected": -5.444609642028809,
+      "sft_loss": 4.288125038146973,
+      "step": 2230
+    },
+    {
+      "epoch": 1.1961866532865026,
+      "grad_norm": 0.6411370012537118,
+      "learning_rate": 7.518858674054838e-07,
+      "logits/chosen": -0.658902645111084,
+      "logits/rejected": -0.44083452224731445,
+      "logps/chosen": -4.602587699890137,
+      "logps/rejected": -5.086310386657715,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.602587699890137,
+      "rewards/margins": 0.48372262716293335,
+      "rewards/rejected": -5.086310386657715,
+      "sft_loss": 4.2280964851379395,
+      "step": 2235
+    },
+    {
+      "epoch": 1.1988626860679044,
+      "grad_norm": 0.4630768053585857,
+      "learning_rate": 7.505393059285394e-07,
+      "logits/chosen": -0.7535982728004456,
+      "logits/rejected": -0.5658230781555176,
+      "logps/chosen": -4.64790153503418,
+      "logps/rejected": -5.012650966644287,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.64790153503418,
+      "rewards/margins": 0.36474916338920593,
+      "rewards/rejected": -5.012650966644287,
+      "sft_loss": 4.332772254943848,
+      "step": 2240
+    },
+    {
+      "epoch": 1.2015387188493059,
+      "grad_norm": 0.47993843494898725,
+      "learning_rate": 7.491903137181501e-07,
+      "logits/chosen": -0.7029468417167664,
+      "logits/rejected": -0.6749275326728821,
+      "logps/chosen": -4.604931831359863,
+      "logps/rejected": -4.9890336990356445,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.604931831359863,
+      "rewards/margins": 0.3841017186641693,
+      "rewards/rejected": -4.9890336990356445,
+      "sft_loss": 4.398990631103516,
+      "step": 2245
+    },
+    {
+      "epoch": 1.2042147516307076,
+      "grad_norm": 0.41159973312127757,
+      "learning_rate": 7.478389038622441e-07,
+      "logits/chosen": -0.6323720216751099,
+      "logits/rejected": -0.6110953092575073,
+      "logps/chosen": -4.512185096740723,
+      "logps/rejected": -4.970728874206543,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.512185096740723,
+      "rewards/margins": 0.45854368805885315,
+      "rewards/rejected": -4.970728874206543,
+      "sft_loss": 4.247210502624512,
+      "step": 2250
+    },
+    {
+      "epoch": 1.206890784412109,
+      "grad_norm": 0.4682841990616749,
+      "learning_rate": 7.46485089472206e-07,
+      "logits/chosen": -0.7587865591049194,
+      "logits/rejected": -0.7158206701278687,
+      "logps/chosen": -4.485775947570801,
+      "logps/rejected": -4.8737568855285645,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.485775947570801,
+      "rewards/margins": 0.3879804015159607,
+      "rewards/rejected": -4.8737568855285645,
+      "sft_loss": 4.248816013336182,
+      "step": 2255
+    },
+    {
+      "epoch": 1.2095668171935106,
+      "grad_norm": 0.44708077693442627,
+      "learning_rate": 7.451288836827487e-07,
+      "logits/chosen": -0.7529920339584351,
+      "logits/rejected": -0.7776705622673035,
+      "logps/chosen": -4.722033500671387,
+      "logps/rejected": -5.016490459442139,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.722033500671387,
+      "rewards/margins": 0.2944570481777191,
+      "rewards/rejected": -5.016490459442139,
+      "sft_loss": 4.490199089050293,
+      "step": 2260
+    },
+    {
+      "epoch": 1.2122428499749123,
+      "grad_norm": 0.5230816480612343,
+      "learning_rate": 7.437702996517869e-07,
+      "logits/chosen": -0.80247563123703,
+      "logits/rejected": -0.7050091028213501,
+      "logps/chosen": -4.602765083312988,
+      "logps/rejected": -4.979498863220215,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.602765083312988,
+      "rewards/margins": 0.37673383951187134,
+      "rewards/rejected": -4.979498863220215,
+      "sft_loss": 4.365364074707031,
+      "step": 2265
+    },
+    {
+      "epoch": 1.2149188827563138,
+      "grad_norm": 0.4773651019537365,
+      "learning_rate": 7.424093505603087e-07,
+      "logits/chosen": -0.8925487399101257,
+      "logits/rejected": -0.6971911191940308,
+      "logps/chosen": -4.3991804122924805,
+      "logps/rejected": -4.9190897941589355,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.3991804122924805,
+      "rewards/margins": 0.5199095010757446,
+      "rewards/rejected": -4.9190897941589355,
+      "sft_loss": 4.129302024841309,
+      "step": 2270
+    },
+    {
+      "epoch": 1.2175949155377153,
+      "grad_norm": 0.5743721034439623,
+      "learning_rate": 7.410460496122482e-07,
+      "logits/chosen": -0.7015866637229919,
+      "logits/rejected": -0.5496960878372192,
+      "logps/chosen": -4.501893043518066,
+      "logps/rejected": -5.030624866485596,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.501893043518066,
+      "rewards/margins": 0.5287320017814636,
+      "rewards/rejected": -5.030624866485596,
+      "sft_loss": 4.109658718109131,
+      "step": 2275
+    },
+    {
+      "epoch": 1.220270948319117,
+      "grad_norm": 0.5284106323606012,
+      "learning_rate": 7.396804100343572e-07,
+      "logits/chosen": -0.8011897206306458,
+      "logits/rejected": -0.5881227254867554,
+      "logps/chosen": -4.559887409210205,
+      "logps/rejected": -5.120944023132324,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.559887409210205,
+      "rewards/margins": 0.5610562562942505,
+      "rewards/rejected": -5.120944023132324,
+      "sft_loss": 4.201657295227051,
+      "step": 2280
+    },
+    {
+      "epoch": 1.2229469811005185,
+      "grad_norm": 0.4225976306052054,
+      "learning_rate": 7.383124450760768e-07,
+      "logits/chosen": -0.7284170985221863,
+      "logits/rejected": -0.5309914946556091,
+      "logps/chosen": -4.601205348968506,
+      "logps/rejected": -5.269949913024902,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.601205348968506,
+      "rewards/margins": 0.668745219707489,
+      "rewards/rejected": -5.269949913024902,
+      "sft_loss": 4.3044843673706055,
+      "step": 2285
+    },
+    {
+      "epoch": 1.22562301388192,
+      "grad_norm": 0.7224182412559759,
+      "learning_rate": 7.369421680094091e-07,
+      "logits/chosen": -0.7828829884529114,
+      "logits/rejected": -0.6176687479019165,
+      "logps/chosen": -4.525793552398682,
+      "logps/rejected": -5.050825119018555,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.525793552398682,
+      "rewards/margins": 0.5250317454338074,
+      "rewards/rejected": -5.050825119018555,
+      "sft_loss": 4.251994609832764,
+      "step": 2290
+    },
+    {
+      "epoch": 1.2282990466633217,
+      "grad_norm": 0.629047772424287,
+      "learning_rate": 7.355695921287881e-07,
+      "logits/chosen": -0.7749170064926147,
+      "logits/rejected": -0.6841250061988831,
+      "logps/chosen": -4.591757297515869,
+      "logps/rejected": -5.037539482116699,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.591757297515869,
+      "rewards/margins": 0.4457823634147644,
+      "rewards/rejected": -5.037539482116699,
+      "sft_loss": 4.263556480407715,
+      "step": 2295
+    },
+    {
+      "epoch": 1.2309750794447232,
+      "grad_norm": 0.33053420351210816,
+      "learning_rate": 7.341947307509513e-07,
+      "logits/chosen": -0.7545934915542603,
+      "logits/rejected": -0.6486998796463013,
+      "logps/chosen": -4.742500305175781,
+      "logps/rejected": -5.1753973960876465,
+      "loss": 0.053,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.742500305175781,
+      "rewards/margins": 0.4328971803188324,
+      "rewards/rejected": -5.1753973960876465,
+      "sft_loss": 4.4836931228637695,
+      "step": 2300
+    },
+    {
+      "epoch": 1.233651112226125,
+      "grad_norm": 0.4071489993833283,
+      "learning_rate": 7.328175972148094e-07,
+      "logits/chosen": -0.7530182600021362,
+      "logits/rejected": -0.6168416142463684,
+      "logps/chosen": -4.746242523193359,
+      "logps/rejected": -5.2826247215271,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.746242523193359,
+      "rewards/margins": 0.5363827347755432,
+      "rewards/rejected": -5.2826247215271,
+      "sft_loss": 4.459273815155029,
+      "step": 2305
+    },
+    {
+      "epoch": 1.2363271450075264,
+      "grad_norm": 0.5441597534623708,
+      "learning_rate": 7.314382048813185e-07,
+      "logits/chosen": -0.7168510556221008,
+      "logits/rejected": -0.43174856901168823,
+      "logps/chosen": -4.272377967834473,
+      "logps/rejected": -4.978228569030762,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.272377967834473,
+      "rewards/margins": 0.7058510780334473,
+      "rewards/rejected": -4.978228569030762,
+      "sft_loss": 4.000433444976807,
+      "step": 2310
+    },
+    {
+      "epoch": 1.2390031777889279,
+      "grad_norm": 0.6252127343519807,
+      "learning_rate": 7.300565671333486e-07,
+      "logits/chosen": -0.8265350461006165,
+      "logits/rejected": -0.6041692495346069,
+      "logps/chosen": -4.5154876708984375,
+      "logps/rejected": -5.120383262634277,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.5154876708984375,
+      "rewards/margins": 0.6048959493637085,
+      "rewards/rejected": -5.120383262634277,
+      "sft_loss": 4.2072625160217285,
+      "step": 2315
+    },
+    {
+      "epoch": 1.2416792105703296,
+      "grad_norm": 0.4222806528582792,
+      "learning_rate": 7.286726973755554e-07,
+      "logits/chosen": -0.7032849192619324,
+      "logits/rejected": -0.6845365762710571,
+      "logps/chosen": -4.55859375,
+      "logps/rejected": -5.039181709289551,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.55859375,
+      "rewards/margins": 0.4805881083011627,
+      "rewards/rejected": -5.039181709289551,
+      "sft_loss": 4.184603691101074,
+      "step": 2320
+    },
+    {
+      "epoch": 1.244355243351731,
+      "grad_norm": 0.5106826580742952,
+      "learning_rate": 7.272866090342493e-07,
+      "logits/chosen": -0.5765025615692139,
+      "logits/rejected": -0.5348777770996094,
+      "logps/chosen": -4.575560569763184,
+      "logps/rejected": -5.200422286987305,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.575560569763184,
+      "rewards/margins": 0.6248610615730286,
+      "rewards/rejected": -5.200422286987305,
+      "sft_loss": 4.18874979019165,
+      "step": 2325
+    },
+    {
+      "epoch": 1.2470312761331326,
+      "grad_norm": 0.36219022238136406,
+      "learning_rate": 7.258983155572656e-07,
+      "logits/chosen": -0.7583931684494019,
+      "logits/rejected": -0.6806284189224243,
+      "logps/chosen": -4.756449222564697,
+      "logps/rejected": -5.231799125671387,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.756449222564697,
+      "rewards/margins": 0.4753497540950775,
+      "rewards/rejected": -5.231799125671387,
+      "sft_loss": 4.347174644470215,
+      "step": 2330
+    },
+    {
+      "epoch": 1.2497073089145343,
+      "grad_norm": 0.7028939653486833,
+      "learning_rate": 7.245078304138335e-07,
+      "logits/chosen": -0.676438570022583,
+      "logits/rejected": -0.617641270160675,
+      "logps/chosen": -4.50102424621582,
+      "logps/rejected": -5.152174472808838,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.50102424621582,
+      "rewards/margins": 0.6511501669883728,
+      "rewards/rejected": -5.152174472808838,
+      "sft_loss": 4.230152606964111,
+      "step": 2335
+    },
+    {
+      "epoch": 1.2523833416959358,
+      "grad_norm": 0.34441628678949693,
+      "learning_rate": 7.231151670944462e-07,
+      "logits/chosen": -0.9201499223709106,
+      "logits/rejected": -0.6691521406173706,
+      "logps/chosen": -4.548752307891846,
+      "logps/rejected": -5.115777969360352,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.548752307891846,
+      "rewards/margins": 0.5670259594917297,
+      "rewards/rejected": -5.115777969360352,
+      "sft_loss": 4.256083965301514,
+      "step": 2340
+    },
+    {
+      "epoch": 1.2550593744773373,
+      "grad_norm": 0.36028138123982145,
+      "learning_rate": 7.217203391107291e-07,
+      "logits/chosen": -0.8337070345878601,
+      "logits/rejected": -0.6574937105178833,
+      "logps/chosen": -4.54742431640625,
+      "logps/rejected": -5.139768600463867,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.54742431640625,
+      "rewards/margins": 0.5923444032669067,
+      "rewards/rejected": -5.139768600463867,
+      "sft_loss": 4.214510440826416,
+      "step": 2345
+    },
+    {
+      "epoch": 1.257735407258739,
+      "grad_norm": 0.5578957686131796,
+      "learning_rate": 7.203233599953096e-07,
+      "logits/chosen": -0.8540999293327332,
+      "logits/rejected": -0.6997717618942261,
+      "logps/chosen": -4.4829535484313965,
+      "logps/rejected": -5.033291816711426,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.4829535484313965,
+      "rewards/margins": 0.5503381490707397,
+      "rewards/rejected": -5.033291816711426,
+      "sft_loss": 4.220416069030762,
+      "step": 2350
+    },
+    {
+      "epoch": 1.2604114400401405,
+      "grad_norm": 0.4751570065353926,
+      "learning_rate": 7.189242433016852e-07,
+      "logits/chosen": -0.778272271156311,
+      "logits/rejected": -0.6525672674179077,
+      "logps/chosen": -4.632129669189453,
+      "logps/rejected": -5.244833946228027,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.632129669189453,
+      "rewards/margins": 0.6127038598060608,
+      "rewards/rejected": -5.244833946228027,
+      "sft_loss": 4.3556694984436035,
+      "step": 2355
+    },
+    {
+      "epoch": 1.263087472821542,
+      "grad_norm": 0.49060819947586853,
+      "learning_rate": 7.17523002604092e-07,
+      "logits/chosen": -0.8550931811332703,
+      "logits/rejected": -0.6443161368370056,
+      "logps/chosen": -4.523386001586914,
+      "logps/rejected": -5.068552017211914,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.523386001586914,
+      "rewards/margins": 0.5451655983924866,
+      "rewards/rejected": -5.068552017211914,
+      "sft_loss": 4.249871253967285,
+      "step": 2360
+    },
+    {
+      "epoch": 1.2657635056029437,
+      "grad_norm": 0.8862204290910091,
+      "learning_rate": 7.161196514973734e-07,
+      "logits/chosen": -0.7608314752578735,
+      "logits/rejected": -0.5959222912788391,
+      "logps/chosen": -4.416409969329834,
+      "logps/rejected": -5.110037326812744,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -4.416409969329834,
+      "rewards/margins": 0.6936279535293579,
+      "rewards/rejected": -5.110037326812744,
+      "sft_loss": 4.177985191345215,
+      "step": 2365
+    },
+    {
+      "epoch": 1.2684395383843452,
+      "grad_norm": 0.36966512161661286,
+      "learning_rate": 7.147142035968483e-07,
+      "logits/chosen": -0.7292311787605286,
+      "logits/rejected": -0.5311886668205261,
+      "logps/chosen": -4.622040748596191,
+      "logps/rejected": -5.120987415313721,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.622040748596191,
+      "rewards/margins": 0.49894601106643677,
+      "rewards/rejected": -5.120987415313721,
+      "sft_loss": 4.340848445892334,
+      "step": 2370
+    },
+    {
+      "epoch": 1.2711155711657467,
+      "grad_norm": 0.6284592603398197,
+      "learning_rate": 7.133066725381781e-07,
+      "logits/chosen": -0.9135788679122925,
+      "logits/rejected": -0.701241135597229,
+      "logps/chosen": -4.508200645446777,
+      "logps/rejected": -5.073288917541504,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.508200645446777,
+      "rewards/margins": 0.565088152885437,
+      "rewards/rejected": -5.073288917541504,
+      "sft_loss": 4.304009437561035,
+      "step": 2375
+    },
+    {
+      "epoch": 1.2737916039471484,
+      "grad_norm": 0.4830072872353554,
+      "learning_rate": 7.118970719772354e-07,
+      "logits/chosen": -0.8243740200996399,
+      "logits/rejected": -0.6196537017822266,
+      "logps/chosen": -4.614374160766602,
+      "logps/rejected": -5.266146659851074,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.614374160766602,
+      "rewards/margins": 0.651772141456604,
+      "rewards/rejected": -5.266146659851074,
+      "sft_loss": 4.351268768310547,
+      "step": 2380
+    },
+    {
+      "epoch": 1.27646763672855,
+      "grad_norm": 0.4899922503825756,
+      "learning_rate": 7.104854155899711e-07,
+      "logits/chosen": -0.676455020904541,
+      "logits/rejected": -0.5914028286933899,
+      "logps/chosen": -4.441977024078369,
+      "logps/rejected": -4.935439109802246,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.441977024078369,
+      "rewards/margins": 0.4934620261192322,
+      "rewards/rejected": -4.935439109802246,
+      "sft_loss": 4.1658124923706055,
+      "step": 2385
+    },
+    {
+      "epoch": 1.2791436695099514,
+      "grad_norm": 0.4339700877673433,
+      "learning_rate": 7.090717170722817e-07,
+      "logits/chosen": -0.6420737504959106,
+      "logits/rejected": -0.6251760721206665,
+      "logps/chosen": -4.5333662033081055,
+      "logps/rejected": -5.1688055992126465,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.5333662033081055,
+      "rewards/margins": 0.6354392766952515,
+      "rewards/rejected": -5.1688055992126465,
+      "sft_loss": 4.251795768737793,
+      "step": 2390
+    },
+    {
+      "epoch": 1.2818197022913531,
+      "grad_norm": 0.5061167164386356,
+      "learning_rate": 7.076559901398762e-07,
+      "logits/chosen": -0.7426393628120422,
+      "logits/rejected": -0.6046496629714966,
+      "logps/chosen": -4.450405120849609,
+      "logps/rejected": -4.918197154998779,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.450405120849609,
+      "rewards/margins": 0.46779197454452515,
+      "rewards/rejected": -4.918197154998779,
+      "sft_loss": 4.215243339538574,
+      "step": 2395
+    },
+    {
+      "epoch": 1.2844957350727546,
+      "grad_norm": 0.5037524653154726,
+      "learning_rate": 7.062382485281436e-07,
+      "logits/chosen": -0.64927077293396,
+      "logits/rejected": -0.513473391532898,
+      "logps/chosen": -4.409026145935059,
+      "logps/rejected": -4.936314105987549,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.409026145935059,
+      "rewards/margins": 0.5272881388664246,
+      "rewards/rejected": -4.936314105987549,
+      "sft_loss": 4.165338039398193,
+      "step": 2400
+    },
+    {
+      "epoch": 1.2844957350727546,
+      "eval_logits/chosen": -0.2459660917520523,
+      "eval_logits/rejected": -0.15563316643238068,
+      "eval_logps/chosen": -4.822916507720947,
+      "eval_logps/rejected": -5.425734519958496,
+      "eval_loss": 0.051439397037029266,
+      "eval_rewards/accuracies": 0.6632047295570374,
+      "eval_rewards/chosen": -4.822916507720947,
+      "eval_rewards/margins": 0.6028181910514832,
+      "eval_rewards/rejected": -5.425734519958496,
+      "eval_runtime": 43.403,
+      "eval_samples_per_second": 30.989,
+      "eval_sft_loss": 4.406880855560303,
+      "eval_steps_per_second": 7.764,
+      "step": 2400
+    },
+    {
+      "epoch": 1.287171767854156,
+      "grad_norm": 0.7512180519376704,
+      "learning_rate": 7.048185059920193e-07,
+      "logits/chosen": -0.6440736651420593,
+      "logits/rejected": -0.5108321905136108,
+      "logps/chosen": -4.895911693572998,
+      "logps/rejected": -5.490865230560303,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.895911693572998,
+      "rewards/margins": 0.5949534773826599,
+      "rewards/rejected": -5.490865230560303,
+      "sft_loss": 4.582104206085205,
+      "step": 2405
+    },
+    {
+      "epoch": 1.2898478006355578,
+      "grad_norm": 1.0070888677024856,
+      "learning_rate": 7.033967763058516e-07,
+      "logits/chosen": -0.722671389579773,
+      "logits/rejected": -0.5478376150131226,
+      "logps/chosen": -4.615888595581055,
+      "logps/rejected": -5.002371788024902,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.615888595581055,
+      "rewards/margins": 0.386482298374176,
+      "rewards/rejected": -5.002371788024902,
+      "sft_loss": 4.239572048187256,
+      "step": 2410
+    },
+    {
+      "epoch": 1.2925238334169593,
+      "grad_norm": 0.4258529079691424,
+      "learning_rate": 7.019730732632681e-07,
+      "logits/chosen": -0.5446482300758362,
+      "logits/rejected": -0.43018198013305664,
+      "logps/chosen": -4.422608375549316,
+      "logps/rejected": -5.125778675079346,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.422608375549316,
+      "rewards/margins": 0.7031702995300293,
+      "rewards/rejected": -5.125778675079346,
+      "sft_loss": 4.148711681365967,
+      "step": 2415
+    },
+    {
+      "epoch": 1.2951998661983608,
+      "grad_norm": 0.35467819540066825,
+      "learning_rate": 7.005474106770418e-07,
+      "logits/chosen": -0.6596757173538208,
+      "logits/rejected": -0.5504968166351318,
+      "logps/chosen": -4.54642391204834,
+      "logps/rejected": -5.045216083526611,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.54642391204834,
+      "rewards/margins": 0.4987919330596924,
+      "rewards/rejected": -5.045216083526611,
+      "sft_loss": 4.238245964050293,
+      "step": 2420
+    },
+    {
+      "epoch": 1.2978758989797625,
+      "grad_norm": 1.3599351436307947,
+      "learning_rate": 6.991198023789577e-07,
+      "logits/chosen": -0.5905810594558716,
+      "logits/rejected": -0.5072682499885559,
+      "logps/chosen": -4.447530269622803,
+      "logps/rejected": -4.882772445678711,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.447530269622803,
+      "rewards/margins": 0.43524178862571716,
+      "rewards/rejected": -4.882772445678711,
+      "sft_loss": 4.207924842834473,
+      "step": 2425
+    },
+    {
+      "epoch": 1.300551931761164,
+      "grad_norm": 0.5025417009492776,
+      "learning_rate": 6.976902622196776e-07,
+      "logits/chosen": -0.6522349119186401,
+      "logits/rejected": -0.5629934072494507,
+      "logps/chosen": -4.689788818359375,
+      "logps/rejected": -5.103816509246826,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.689788818359375,
+      "rewards/margins": 0.41402775049209595,
+      "rewards/rejected": -5.103816509246826,
+      "sft_loss": 4.374016761779785,
+      "step": 2430
+    },
+    {
+      "epoch": 1.3032279645425655,
+      "grad_norm": 0.33461264496064647,
+      "learning_rate": 6.962588040686064e-07,
+      "logits/chosen": -0.7623521089553833,
+      "logits/rejected": -0.5751533508300781,
+      "logps/chosen": -4.745035171508789,
+      "logps/rejected": -5.1734185218811035,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.745035171508789,
+      "rewards/margins": 0.42838358879089355,
+      "rewards/rejected": -5.1734185218811035,
+      "sft_loss": 4.583779811859131,
+      "step": 2435
+    },
+    {
+      "epoch": 1.3059039973239672,
+      "grad_norm": 0.3503261781496817,
+      "learning_rate": 6.948254418137573e-07,
+      "logits/chosen": -0.712530255317688,
+      "logits/rejected": -0.5704913139343262,
+      "logps/chosen": -4.597674369812012,
+      "logps/rejected": -5.1268134117126465,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.597674369812012,
+      "rewards/margins": 0.5291392803192139,
+      "rewards/rejected": -5.1268134117126465,
+      "sft_loss": 4.329079627990723,
+      "step": 2440
+    },
+    {
+      "epoch": 1.3085800301053687,
+      "grad_norm": 0.39441341051128054,
+      "learning_rate": 6.933901893616174e-07,
+      "logits/chosen": -0.7677714824676514,
+      "logits/rejected": -0.6126347184181213,
+      "logps/chosen": -4.438683032989502,
+      "logps/rejected": -5.042284965515137,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.438683032989502,
+      "rewards/margins": 0.6036025881767273,
+      "rewards/rejected": -5.042284965515137,
+      "sft_loss": 4.119725227355957,
+      "step": 2445
+    },
+    {
+      "epoch": 1.3112560628867704,
+      "grad_norm": 0.3927956162770441,
+      "learning_rate": 6.919530606370121e-07,
+      "logits/chosen": -0.7254393696784973,
+      "logits/rejected": -0.5506623983383179,
+      "logps/chosen": -4.357143402099609,
+      "logps/rejected": -4.954403877258301,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.357143402099609,
+      "rewards/margins": 0.5972608327865601,
+      "rewards/rejected": -4.954403877258301,
+      "sft_loss": 4.115630149841309,
+      "step": 2450
+    },
+    {
+      "epoch": 1.313932095668172,
+      "grad_norm": 0.3738986864645462,
+      "learning_rate": 6.905140695829706e-07,
+      "logits/chosen": -0.7687052488327026,
+      "logits/rejected": -0.46032652258872986,
+      "logps/chosen": -4.44050407409668,
+      "logps/rejected": -5.113463878631592,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.44050407409668,
+      "rewards/margins": 0.6729599237442017,
+      "rewards/rejected": -5.113463878631592,
+      "sft_loss": 4.17454719543457,
+      "step": 2455
+    },
+    {
+      "epoch": 1.3166081284495736,
+      "grad_norm": 0.5571532095288572,
+      "learning_rate": 6.890732301605904e-07,
+      "logits/chosen": -0.6341904401779175,
+      "logits/rejected": -0.5448669195175171,
+      "logps/chosen": -4.6078996658325195,
+      "logps/rejected": -5.000533103942871,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.6078996658325195,
+      "rewards/margins": 0.3926334083080292,
+      "rewards/rejected": -5.000533103942871,
+      "sft_loss": 4.291760444641113,
+      "step": 2460
+    },
+    {
+      "epoch": 1.3192841612309751,
+      "grad_norm": 0.3892123944114807,
+      "learning_rate": 6.876305563489021e-07,
+      "logits/chosen": -0.6441881060600281,
+      "logits/rejected": -0.5762056112289429,
+      "logps/chosen": -4.400938510894775,
+      "logps/rejected": -5.1529011726379395,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.400938510894775,
+      "rewards/margins": 0.7519630193710327,
+      "rewards/rejected": -5.1529011726379395,
+      "sft_loss": 4.062161445617676,
+      "step": 2465
+    },
+    {
+      "epoch": 1.3219601940123766,
+      "grad_norm": 0.5043097198631741,
+      "learning_rate": 6.861860621447331e-07,
+      "logits/chosen": -0.7821289300918579,
+      "logits/rejected": -0.6575089693069458,
+      "logps/chosen": -4.7076544761657715,
+      "logps/rejected": -5.158883094787598,
+      "loss": 0.0539,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.7076544761657715,
+      "rewards/margins": 0.451228529214859,
+      "rewards/rejected": -5.158883094787598,
+      "sft_loss": 4.450681686401367,
+      "step": 2470
+    },
+    {
+      "epoch": 1.3246362267937783,
+      "grad_norm": 0.5241208894891419,
+      "learning_rate": 6.847397615625725e-07,
+      "logits/chosen": -0.6322156190872192,
+      "logits/rejected": -0.5924761295318604,
+      "logps/chosen": -4.701780796051025,
+      "logps/rejected": -5.140883922576904,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.701780796051025,
+      "rewards/margins": 0.43910354375839233,
+      "rewards/rejected": -5.140883922576904,
+      "sft_loss": 4.368111610412598,
+      "step": 2475
+    },
+    {
+      "epoch": 1.3273122595751798,
+      "grad_norm": 0.5530610503614979,
+      "learning_rate": 6.83291668634435e-07,
+      "logits/chosen": -0.7902230024337769,
+      "logits/rejected": -0.5947554111480713,
+      "logps/chosen": -4.581168174743652,
+      "logps/rejected": -5.159932613372803,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.581168174743652,
+      "rewards/margins": 0.5787646174430847,
+      "rewards/rejected": -5.159932613372803,
+      "sft_loss": 4.308629512786865,
+      "step": 2480
+    },
+    {
+      "epoch": 1.3299882923565813,
+      "grad_norm": 0.5053924424200537,
+      "learning_rate": 6.818417974097246e-07,
+      "logits/chosen": -0.5258646607398987,
+      "logits/rejected": -0.38841742277145386,
+      "logps/chosen": -4.452823638916016,
+      "logps/rejected": -5.15665864944458,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.452823638916016,
+      "rewards/margins": 0.7038346529006958,
+      "rewards/rejected": -5.15665864944458,
+      "sft_loss": 4.177115440368652,
+      "step": 2485
+    },
+    {
+      "epoch": 1.332664325137983,
+      "grad_norm": 0.46316275553685127,
+      "learning_rate": 6.803901619550981e-07,
+      "logits/chosen": -0.6718374490737915,
+      "logits/rejected": -0.6161580085754395,
+      "logps/chosen": -4.422329902648926,
+      "logps/rejected": -4.936065673828125,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.422329902648926,
+      "rewards/margins": 0.5137358903884888,
+      "rewards/rejected": -4.936065673828125,
+      "sft_loss": 4.147505283355713,
+      "step": 2490
+    },
+    {
+      "epoch": 1.3353403579193845,
+      "grad_norm": 0.5381914488734175,
+      "learning_rate": 6.789367763543292e-07,
+      "logits/chosen": -0.6015291213989258,
+      "logits/rejected": -0.5835751295089722,
+      "logps/chosen": -4.570321559906006,
+      "logps/rejected": -5.038084983825684,
+      "loss": 0.0544,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.570321559906006,
+      "rewards/margins": 0.467763751745224,
+      "rewards/rejected": -5.038084983825684,
+      "sft_loss": 4.3525590896606445,
+      "step": 2495
+    },
+    {
+      "epoch": 1.338016390700786,
+      "grad_norm": 0.7920108460285353,
+      "learning_rate": 6.774816547081714e-07,
+      "logits/chosen": -0.6614540815353394,
+      "logits/rejected": -0.4659046232700348,
+      "logps/chosen": -4.600603103637695,
+      "logps/rejected": -5.15115213394165,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -4.600603103637695,
+      "rewards/margins": 0.5505493879318237,
+      "rewards/rejected": -5.15115213394165,
+      "sft_loss": 4.411740303039551,
+      "step": 2500
+    },
+    {
+      "epoch": 1.3406924234821878,
+      "grad_norm": 0.32826827986852447,
+      "learning_rate": 6.760248111342211e-07,
+      "logits/chosen": -0.7377767562866211,
+      "logits/rejected": -0.5531325340270996,
+      "logps/chosen": -4.688912868499756,
+      "logps/rejected": -5.238001823425293,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.688912868499756,
+      "rewards/margins": 0.5490891337394714,
+      "rewards/rejected": -5.238001823425293,
+      "sft_loss": 4.432499885559082,
+      "step": 2505
+    },
+    {
+      "epoch": 1.3433684562635893,
+      "grad_norm": 0.6428193005965454,
+      "learning_rate": 6.745662597667813e-07,
+      "logits/chosen": -0.750898540019989,
+      "logits/rejected": -0.6026118993759155,
+      "logps/chosen": -4.439126014709473,
+      "logps/rejected": -4.934445381164551,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.439126014709473,
+      "rewards/margins": 0.49531856179237366,
+      "rewards/rejected": -4.934445381164551,
+      "sft_loss": 4.148636817932129,
+      "step": 2510
+    },
+    {
+      "epoch": 1.3460444890449907,
+      "grad_norm": 0.38982972965941254,
+      "learning_rate": 6.731060147567236e-07,
+      "logits/chosen": -0.5894494652748108,
+      "logits/rejected": -0.5215369462966919,
+      "logps/chosen": -4.36592960357666,
+      "logps/rejected": -4.934881210327148,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.36592960357666,
+      "rewards/margins": 0.5689516067504883,
+      "rewards/rejected": -4.934881210327148,
+      "sft_loss": 4.115760326385498,
+      "step": 2515
+    },
+    {
+      "epoch": 1.3487205218263925,
+      "grad_norm": 0.611393333794855,
+      "learning_rate": 6.716440902713515e-07,
+      "logits/chosen": -0.6964327096939087,
+      "logits/rejected": -0.6343324184417725,
+      "logps/chosen": -4.4690141677856445,
+      "logps/rejected": -4.8751397132873535,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.4690141677856445,
+      "rewards/margins": 0.406125545501709,
+      "rewards/rejected": -4.8751397132873535,
+      "sft_loss": 4.147188663482666,
+      "step": 2520
+    },
+    {
+      "epoch": 1.351396554607794,
+      "grad_norm": 0.39953792525208676,
+      "learning_rate": 6.701805004942627e-07,
+      "logits/chosen": -0.6524925231933594,
+      "logits/rejected": -0.595403254032135,
+      "logps/chosen": -4.759545803070068,
+      "logps/rejected": -5.341307640075684,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.759545803070068,
+      "rewards/margins": 0.5817619562149048,
+      "rewards/rejected": -5.341307640075684,
+      "sft_loss": 4.543875694274902,
+      "step": 2525
+    },
+    {
+      "epoch": 1.3540725873891954,
+      "grad_norm": 0.5767017487926428,
+      "learning_rate": 6.687152596252119e-07,
+      "logits/chosen": -0.7337282299995422,
+      "logits/rejected": -0.6855202913284302,
+      "logps/chosen": -4.838815212249756,
+      "logps/rejected": -5.192466735839844,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.838815212249756,
+      "rewards/margins": 0.35365158319473267,
+      "rewards/rejected": -5.192466735839844,
+      "sft_loss": 4.54294490814209,
+      "step": 2530
+    },
+    {
+      "epoch": 1.3567486201705972,
+      "grad_norm": 0.4398488876945579,
+      "learning_rate": 6.672483818799722e-07,
+      "logits/chosen": -0.7373114824295044,
+      "logits/rejected": -0.5846437215805054,
+      "logps/chosen": -4.462458610534668,
+      "logps/rejected": -4.956521511077881,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.462458610534668,
+      "rewards/margins": 0.4940629005432129,
+      "rewards/rejected": -4.956521511077881,
+      "sft_loss": 4.213451862335205,
+      "step": 2535
+    },
+    {
+      "epoch": 1.3594246529519987,
+      "grad_norm": 0.3887196561405752,
+      "learning_rate": 6.657798814901978e-07,
+      "logits/chosen": -0.7088289260864258,
+      "logits/rejected": -0.4987329840660095,
+      "logps/chosen": -4.533008098602295,
+      "logps/rejected": -4.962521553039551,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.533008098602295,
+      "rewards/margins": 0.4295133650302887,
+      "rewards/rejected": -4.962521553039551,
+      "sft_loss": 4.149649143218994,
+      "step": 2540
+    },
+    {
+      "epoch": 1.3621006857334002,
+      "grad_norm": 0.51660624133589,
+      "learning_rate": 6.643097727032863e-07,
+      "logits/chosen": -0.7439020872116089,
+      "logits/rejected": -0.5362112522125244,
+      "logps/chosen": -4.448864936828613,
+      "logps/rejected": -5.117484092712402,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.448864936828613,
+      "rewards/margins": 0.6686197519302368,
+      "rewards/rejected": -5.117484092712402,
+      "sft_loss": 4.22825288772583,
+      "step": 2545
+    },
+    {
+      "epoch": 1.3647767185148019,
+      "grad_norm": 0.5837046444529704,
+      "learning_rate": 6.628380697822392e-07,
+      "logits/chosen": -0.7480605244636536,
+      "logits/rejected": -0.5812298059463501,
+      "logps/chosen": -4.5380167961120605,
+      "logps/rejected": -5.00238561630249,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.5380167961120605,
+      "rewards/margins": 0.46436864137649536,
+      "rewards/rejected": -5.00238561630249,
+      "sft_loss": 4.235762119293213,
+      "step": 2550
+    },
+    {
+      "epoch": 1.3674527512962034,
+      "grad_norm": 0.5264631149675114,
+      "learning_rate": 6.61364787005525e-07,
+      "logits/chosen": -0.7261337041854858,
+      "logits/rejected": -0.5831155180931091,
+      "logps/chosen": -4.57515287399292,
+      "logps/rejected": -5.160813331604004,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.57515287399292,
+      "rewards/margins": 0.585660457611084,
+      "rewards/rejected": -5.160813331604004,
+      "sft_loss": 4.378745079040527,
+      "step": 2555
+    },
+    {
+      "epoch": 1.3701287840776049,
+      "grad_norm": 0.4918096353013463,
+      "learning_rate": 6.598899386669395e-07,
+      "logits/chosen": -0.7453008890151978,
+      "logits/rejected": -0.6159285306930542,
+      "logps/chosen": -4.560245513916016,
+      "logps/rejected": -5.077498435974121,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.560245513916016,
+      "rewards/margins": 0.5172520279884338,
+      "rewards/rejected": -5.077498435974121,
+      "sft_loss": 4.279725074768066,
+      "step": 2560
+    },
+    {
+      "epoch": 1.3728048168590066,
+      "grad_norm": 0.44468809524085257,
+      "learning_rate": 6.584135390754679e-07,
+      "logits/chosen": -0.7995618581771851,
+      "logits/rejected": -0.6510564088821411,
+      "logps/chosen": -4.504146099090576,
+      "logps/rejected": -5.2412333488464355,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.504146099090576,
+      "rewards/margins": 0.7370876669883728,
+      "rewards/rejected": -5.2412333488464355,
+      "sft_loss": 4.247662544250488,
+      "step": 2565
+    },
+    {
+      "epoch": 1.375480849640408,
+      "grad_norm": 0.38358992062706415,
+      "learning_rate": 6.569356025551454e-07,
+      "logits/chosen": -0.7912853956222534,
+      "logits/rejected": -0.7308619022369385,
+      "logps/chosen": -4.615150451660156,
+      "logps/rejected": -5.038762092590332,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.615150451660156,
+      "rewards/margins": 0.42361217737197876,
+      "rewards/rejected": -5.038762092590332,
+      "sft_loss": 4.207141399383545,
+      "step": 2570
+    },
+    {
+      "epoch": 1.3781568824218096,
+      "grad_norm": 0.36018657240876495,
+      "learning_rate": 6.554561434449186e-07,
+      "logits/chosen": -0.9121583104133606,
+      "logits/rejected": -0.7354962825775146,
+      "logps/chosen": -4.590145587921143,
+      "logps/rejected": -5.142067909240723,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.590145587921143,
+      "rewards/margins": 0.5519219636917114,
+      "rewards/rejected": -5.142067909240723,
+      "sft_loss": 4.361257076263428,
+      "step": 2575
+    },
+    {
+      "epoch": 1.3808329152032113,
+      "grad_norm": 0.41305806068904577,
+      "learning_rate": 6.539751760985063e-07,
+      "logits/chosen": -0.780017614364624,
+      "logits/rejected": -0.7046865224838257,
+      "logps/chosen": -4.54480504989624,
+      "logps/rejected": -4.859285831451416,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.54480504989624,
+      "rewards/margins": 0.31448012590408325,
+      "rewards/rejected": -4.859285831451416,
+      "sft_loss": 4.202549934387207,
+      "step": 2580
+    },
+    {
+      "epoch": 1.3835089479846128,
+      "grad_norm": 0.3782339057679898,
+      "learning_rate": 6.524927148842602e-07,
+      "logits/chosen": -0.7151792049407959,
+      "logits/rejected": -0.5270150303840637,
+      "logps/chosen": -4.705697059631348,
+      "logps/rejected": -5.182741641998291,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.705697059631348,
+      "rewards/margins": 0.47704464197158813,
+      "rewards/rejected": -5.182741641998291,
+      "sft_loss": 4.334061622619629,
+      "step": 2585
+    },
+    {
+      "epoch": 1.3861849807660143,
+      "grad_norm": 0.5868680617117082,
+      "learning_rate": 6.510087741850254e-07,
+      "logits/chosen": -0.7695601582527161,
+      "logits/rejected": -0.6490595936775208,
+      "logps/chosen": -4.606133460998535,
+      "logps/rejected": -5.070815086364746,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.606133460998535,
+      "rewards/margins": 0.464682012796402,
+      "rewards/rejected": -5.070815086364746,
+      "sft_loss": 4.332033634185791,
+      "step": 2590
+    },
+    {
+      "epoch": 1.388861013547416,
+      "grad_norm": 0.31923571248340193,
+      "learning_rate": 6.495233683980012e-07,
+      "logits/chosen": -0.7917790412902832,
+      "logits/rejected": -0.757839560508728,
+      "logps/chosen": -4.3590850830078125,
+      "logps/rejected": -4.865941047668457,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.3590850830078125,
+      "rewards/margins": 0.5068557858467102,
+      "rewards/rejected": -4.865941047668457,
+      "sft_loss": 4.141876220703125,
+      "step": 2595
+    },
+    {
+      "epoch": 1.3915370463288175,
+      "grad_norm": 0.6110778260580617,
+      "learning_rate": 6.480365119346011e-07,
+      "logits/chosen": -0.7061268091201782,
+      "logits/rejected": -0.5795341730117798,
+      "logps/chosen": -4.5417585372924805,
+      "logps/rejected": -4.974133491516113,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.5417585372924805,
+      "rewards/margins": 0.4323754906654358,
+      "rewards/rejected": -4.974133491516113,
+      "sft_loss": 4.267592906951904,
+      "step": 2600
+    },
+    {
+      "epoch": 1.394213079110219,
+      "grad_norm": 0.3968746308711125,
+      "learning_rate": 6.465482192203129e-07,
+      "logits/chosen": -0.6922782063484192,
+      "logits/rejected": -0.6162427663803101,
+      "logps/chosen": -4.611149787902832,
+      "logps/rejected": -5.119748115539551,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.611149787902832,
+      "rewards/margins": 0.508597731590271,
+      "rewards/rejected": -5.119748115539551,
+      "sft_loss": 4.388543605804443,
+      "step": 2605
+    },
+    {
+      "epoch": 1.3968891118916207,
+      "grad_norm": 0.43719125204764003,
+      "learning_rate": 6.45058504694559e-07,
+      "logits/chosen": -0.6123635768890381,
+      "logits/rejected": -0.5769099593162537,
+      "logps/chosen": -4.499598503112793,
+      "logps/rejected": -5.056746006011963,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.499598503112793,
+      "rewards/margins": 0.557147204875946,
+      "rewards/rejected": -5.056746006011963,
+      "sft_loss": 4.259655952453613,
+      "step": 2610
+    },
+    {
+      "epoch": 1.3995651446730222,
+      "grad_norm": 0.6082587458332754,
+      "learning_rate": 6.435673828105564e-07,
+      "logits/chosen": -0.7298904657363892,
+      "logits/rejected": -0.5628734230995178,
+      "logps/chosen": -4.518126964569092,
+      "logps/rejected": -5.07608699798584,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.518126964569092,
+      "rewards/margins": 0.5579599142074585,
+      "rewards/rejected": -5.07608699798584,
+      "sft_loss": 4.245197296142578,
+      "step": 2615
+    },
+    {
+      "epoch": 1.402241177454424,
+      "grad_norm": 0.8367099282361746,
+      "learning_rate": 6.420748680351763e-07,
+      "logits/chosen": -0.6793943643569946,
+      "logits/rejected": -0.7437562942504883,
+      "logps/chosen": -4.676467418670654,
+      "logps/rejected": -5.091729164123535,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.676467418670654,
+      "rewards/margins": 0.4152621626853943,
+      "rewards/rejected": -5.091729164123535,
+      "sft_loss": 4.483765125274658,
+      "step": 2620
+    },
+    {
+      "epoch": 1.4049172102358254,
+      "grad_norm": 0.3433494848599413,
+      "learning_rate": 6.405809748488032e-07,
+      "logits/chosen": -0.8351935148239136,
+      "logits/rejected": -0.6800636053085327,
+      "logps/chosen": -4.701047420501709,
+      "logps/rejected": -5.2720818519592285,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.701047420501709,
+      "rewards/margins": 0.571033775806427,
+      "rewards/rejected": -5.2720818519592285,
+      "sft_loss": 4.444343566894531,
+      "step": 2625
+    },
+    {
+      "epoch": 1.4075932430172269,
+      "grad_norm": 0.4795161285636052,
+      "learning_rate": 6.390857177451956e-07,
+      "logits/chosen": -0.8992173075675964,
+      "logits/rejected": -0.6891031265258789,
+      "logps/chosen": -4.538784027099609,
+      "logps/rejected": -5.005777835845947,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.538784027099609,
+      "rewards/margins": 0.46699291467666626,
+      "rewards/rejected": -5.005777835845947,
+      "sft_loss": 4.31063985824585,
+      "step": 2630
+    },
+    {
+      "epoch": 1.4102692757986286,
+      "grad_norm": 0.7016521627801586,
+      "learning_rate": 6.375891112313445e-07,
+      "logits/chosen": -0.8801708221435547,
+      "logits/rejected": -0.7778578996658325,
+      "logps/chosen": -4.37764310836792,
+      "logps/rejected": -4.836381912231445,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.37764310836792,
+      "rewards/margins": 0.4587384760379791,
+      "rewards/rejected": -4.836381912231445,
+      "sft_loss": 4.069215774536133,
+      "step": 2635
+    },
+    {
+      "epoch": 1.41294530858003,
+      "grad_norm": 0.4657342923390584,
+      "learning_rate": 6.360911698273326e-07,
+      "logits/chosen": -0.7782545685768127,
+      "logits/rejected": -0.6499952077865601,
+      "logps/chosen": -4.533600807189941,
+      "logps/rejected": -4.894816875457764,
+      "loss": 0.0543,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.533600807189941,
+      "rewards/margins": 0.36121657490730286,
+      "rewards/rejected": -4.894816875457764,
+      "sft_loss": 4.284762382507324,
+      "step": 2640
+    },
+    {
+      "epoch": 1.4156213413614318,
+      "grad_norm": 0.47112647697309784,
+      "learning_rate": 6.345919080661944e-07,
+      "logits/chosen": -0.9207308888435364,
+      "logits/rejected": -0.8575568199157715,
+      "logps/chosen": -4.626215934753418,
+      "logps/rejected": -5.244388580322266,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.626215934753418,
+      "rewards/margins": 0.6181727051734924,
+      "rewards/rejected": -5.244388580322266,
+      "sft_loss": 4.426272392272949,
+      "step": 2645
+    },
+    {
+      "epoch": 1.4182973741428333,
+      "grad_norm": 0.29671780119762087,
+      "learning_rate": 6.330913404937737e-07,
+      "logits/chosen": -0.9336298108100891,
+      "logits/rejected": -0.7710980176925659,
+      "logps/chosen": -4.727460861206055,
+      "logps/rejected": -5.37843132019043,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.727460861206055,
+      "rewards/margins": 0.6509709358215332,
+      "rewards/rejected": -5.37843132019043,
+      "sft_loss": 4.423439979553223,
+      "step": 2650
+    },
+    {
+      "epoch": 1.4209734069242348,
+      "grad_norm": 0.5791699497151932,
+      "learning_rate": 6.315894816685838e-07,
+      "logits/chosen": -0.7895804643630981,
+      "logits/rejected": -0.6132604479789734,
+      "logps/chosen": -4.567627906799316,
+      "logps/rejected": -4.8765106201171875,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.567627906799316,
+      "rewards/margins": 0.3088833689689636,
+      "rewards/rejected": -4.8765106201171875,
+      "sft_loss": 4.217142105102539,
+      "step": 2655
+    },
+    {
+      "epoch": 1.4236494397056365,
+      "grad_norm": 0.5073327462343239,
+      "learning_rate": 6.300863461616657e-07,
+      "logits/chosen": -0.7717041969299316,
+      "logits/rejected": -0.6730908155441284,
+      "logps/chosen": -4.412945747375488,
+      "logps/rejected": -4.921000003814697,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.412945747375488,
+      "rewards/margins": 0.5080535411834717,
+      "rewards/rejected": -4.921000003814697,
+      "sft_loss": 4.149726390838623,
+      "step": 2660
+    },
+    {
+      "epoch": 1.426325472487038,
+      "grad_norm": 0.3239165109984895,
+      "learning_rate": 6.285819485564465e-07,
+      "logits/chosen": -0.9396502375602722,
+      "logits/rejected": -0.7643145322799683,
+      "logps/chosen": -4.451190948486328,
+      "logps/rejected": -5.017029762268066,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.451190948486328,
+      "rewards/margins": 0.5658388733863831,
+      "rewards/rejected": -5.017029762268066,
+      "sft_loss": 4.155117511749268,
+      "step": 2665
+    },
+    {
+      "epoch": 1.4290015052684395,
+      "grad_norm": 0.47727707504839423,
+      "learning_rate": 6.270763034485986e-07,
+      "logits/chosen": -0.8083940744400024,
+      "logits/rejected": -0.6736945509910583,
+      "logps/chosen": -4.661080360412598,
+      "logps/rejected": -5.173447132110596,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.661080360412598,
+      "rewards/margins": 0.5123669505119324,
+      "rewards/rejected": -5.173447132110596,
+      "sft_loss": 4.300747871398926,
+      "step": 2670
+    },
+    {
+      "epoch": 1.4316775380498412,
+      "grad_norm": 0.49975394521225786,
+      "learning_rate": 6.255694254458972e-07,
+      "logits/chosen": -0.9003822207450867,
+      "logits/rejected": -0.7234520316123962,
+      "logps/chosen": -4.687695503234863,
+      "logps/rejected": -5.162412166595459,
+      "loss": 0.0545,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.687695503234863,
+      "rewards/margins": 0.474717378616333,
+      "rewards/rejected": -5.162412166595459,
+      "sft_loss": 4.253655433654785,
+      "step": 2675
+    },
+    {
+      "epoch": 1.4343535708312427,
+      "grad_norm": 0.7970279083312815,
+      "learning_rate": 6.240613291680795e-07,
+      "logits/chosen": -0.8419411778450012,
+      "logits/rejected": -0.653814435005188,
+      "logps/chosen": -4.679329872131348,
+      "logps/rejected": -5.142355918884277,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.679329872131348,
+      "rewards/margins": 0.4630259871482849,
+      "rewards/rejected": -5.142355918884277,
+      "sft_loss": 4.350041389465332,
+      "step": 2680
+    },
+    {
+      "epoch": 1.4370296036126442,
+      "grad_norm": 0.44742716716600295,
+      "learning_rate": 6.225520292467021e-07,
+      "logits/chosen": -0.8692100644111633,
+      "logits/rejected": -0.6362851858139038,
+      "logps/chosen": -4.339302062988281,
+      "logps/rejected": -5.107569217681885,
+      "loss": 0.05,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.339302062988281,
+      "rewards/margins": 0.7682672739028931,
+      "rewards/rejected": -5.107569217681885,
+      "sft_loss": 4.058346748352051,
+      "step": 2685
+    },
+    {
+      "epoch": 1.439705636394046,
+      "grad_norm": 0.5956502684124177,
+      "learning_rate": 6.210415403249993e-07,
+      "logits/chosen": -0.9499366879463196,
+      "logits/rejected": -0.6553173065185547,
+      "logps/chosen": -4.524402141571045,
+      "logps/rejected": -5.320879936218262,
+      "loss": 0.051,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.524402141571045,
+      "rewards/margins": 0.7964780330657959,
+      "rewards/rejected": -5.320879936218262,
+      "sft_loss": 4.101487159729004,
+      "step": 2690
+    },
+    {
+      "epoch": 1.4423816691754474,
+      "grad_norm": 0.6211044159448419,
+      "learning_rate": 6.195298770577415e-07,
+      "logits/chosen": -0.7495101094245911,
+      "logits/rejected": -0.7657794952392578,
+      "logps/chosen": -4.553832054138184,
+      "logps/rejected": -5.104920864105225,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.553832054138184,
+      "rewards/margins": 0.551088809967041,
+      "rewards/rejected": -5.104920864105225,
+      "sft_loss": 4.29389762878418,
+      "step": 2695
+    },
+    {
+      "epoch": 1.445057701956849,
+      "grad_norm": 0.3685359376502068,
+      "learning_rate": 6.180170541110923e-07,
+      "logits/chosen": -0.82825767993927,
+      "logits/rejected": -0.6391871571540833,
+      "logps/chosen": -4.653820514678955,
+      "logps/rejected": -5.102269649505615,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.653820514678955,
+      "rewards/margins": 0.4484497010707855,
+      "rewards/rejected": -5.102269649505615,
+      "sft_loss": 4.338795185089111,
+      "step": 2700
+    },
+    {
+      "epoch": 1.4477337347382506,
+      "grad_norm": 0.36879136698007464,
+      "learning_rate": 6.165030861624663e-07,
+      "logits/chosen": -1.0215765237808228,
+      "logits/rejected": -0.704619288444519,
+      "logps/chosen": -4.500130653381348,
+      "logps/rejected": -5.3099188804626465,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.500130653381348,
+      "rewards/margins": 0.809788703918457,
+      "rewards/rejected": -5.3099188804626465,
+      "sft_loss": 4.320158004760742,
+      "step": 2705
+    },
+    {
+      "epoch": 1.4504097675196521,
+      "grad_norm": 0.35921794005610297,
+      "learning_rate": 6.149879879003876e-07,
+      "logits/chosen": -0.736519992351532,
+      "logits/rejected": -0.7717832326889038,
+      "logps/chosen": -4.527623176574707,
+      "logps/rejected": -4.958807945251465,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.527623176574707,
+      "rewards/margins": 0.431184858083725,
+      "rewards/rejected": -4.958807945251465,
+      "sft_loss": 4.252358436584473,
+      "step": 2710
+    },
+    {
+      "epoch": 1.4530858003010536,
+      "grad_norm": 0.4029998204084026,
+      "learning_rate": 6.13471774024346e-07,
+      "logits/chosen": -0.9023638963699341,
+      "logits/rejected": -0.7861912250518799,
+      "logps/chosen": -4.328760623931885,
+      "logps/rejected": -4.899123191833496,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.328760623931885,
+      "rewards/margins": 0.5703624486923218,
+      "rewards/rejected": -4.899123191833496,
+      "sft_loss": 4.081745624542236,
+      "step": 2715
+    },
+    {
+      "epoch": 1.4557618330824553,
+      "grad_norm": 0.3181867162536053,
+      "learning_rate": 6.119544592446551e-07,
+      "logits/chosen": -0.8461063504219055,
+      "logits/rejected": -0.7370610237121582,
+      "logps/chosen": -4.367936611175537,
+      "logps/rejected": -4.853856086730957,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.367936611175537,
+      "rewards/margins": 0.48591962456703186,
+      "rewards/rejected": -4.853856086730957,
+      "sft_loss": 4.085618019104004,
+      "step": 2720
+    },
+    {
+      "epoch": 1.4584378658638568,
+      "grad_norm": 0.4417312535342543,
+      "learning_rate": 6.104360582823096e-07,
+      "logits/chosen": -0.853473961353302,
+      "logits/rejected": -0.7386851906776428,
+      "logps/chosen": -4.7194013595581055,
+      "logps/rejected": -5.207268714904785,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.7194013595581055,
+      "rewards/margins": 0.4878672659397125,
+      "rewards/rejected": -5.207268714904785,
+      "sft_loss": 4.425844192504883,
+      "step": 2725
+    },
+    {
+      "epoch": 1.4611138986452583,
+      "grad_norm": 0.4267754873095855,
+      "learning_rate": 6.089165858688423e-07,
+      "logits/chosen": -0.7712498903274536,
+      "logits/rejected": -0.5837336778640747,
+      "logps/chosen": -4.546807289123535,
+      "logps/rejected": -5.284149169921875,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.546807289123535,
+      "rewards/margins": 0.7373424768447876,
+      "rewards/rejected": -5.284149169921875,
+      "sft_loss": 4.248849391937256,
+      "step": 2730
+    },
+    {
+      "epoch": 1.46378993142666,
+      "grad_norm": 0.5243832632135247,
+      "learning_rate": 6.073960567461811e-07,
+      "logits/chosen": -0.7328991889953613,
+      "logits/rejected": -0.5446587800979614,
+      "logps/chosen": -4.3400068283081055,
+      "logps/rejected": -5.033965110778809,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.3400068283081055,
+      "rewards/margins": 0.6939582824707031,
+      "rewards/rejected": -5.033965110778809,
+      "sft_loss": 4.018763542175293,
+      "step": 2735
+    },
+    {
+      "epoch": 1.4664659642080615,
+      "grad_norm": 0.43775113695751916,
+      "learning_rate": 6.058744856665065e-07,
+      "logits/chosen": -0.7534220218658447,
+      "logits/rejected": -0.6736117601394653,
+      "logps/chosen": -4.569155693054199,
+      "logps/rejected": -5.47993278503418,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.569155693054199,
+      "rewards/margins": 0.9107775688171387,
+      "rewards/rejected": -5.47993278503418,
+      "sft_loss": 4.218738555908203,
+      "step": 2740
+    },
+    {
+      "epoch": 1.469141996989463,
+      "grad_norm": 0.4717035016463936,
+      "learning_rate": 6.043518873921074e-07,
+      "logits/chosen": -0.7917760610580444,
+      "logits/rejected": -0.657507061958313,
+      "logps/chosen": -4.352883338928223,
+      "logps/rejected": -4.968850135803223,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.352883338928223,
+      "rewards/margins": 0.6159666776657104,
+      "rewards/rejected": -4.968850135803223,
+      "sft_loss": 3.9638657569885254,
+      "step": 2745
+    },
+    {
+      "epoch": 1.4718180297708647,
+      "grad_norm": 0.45414652768336783,
+      "learning_rate": 6.028282766952393e-07,
+      "logits/chosen": -0.7075755596160889,
+      "logits/rejected": -0.6383055448532104,
+      "logps/chosen": -4.6790876388549805,
+      "logps/rejected": -5.391210556030273,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.6790876388549805,
+      "rewards/margins": 0.7121232151985168,
+      "rewards/rejected": -5.391210556030273,
+      "sft_loss": 4.254571914672852,
+      "step": 2750
+    },
+    {
+      "epoch": 1.4744940625522662,
+      "grad_norm": 0.5878041839400147,
+      "learning_rate": 6.013036683579798e-07,
+      "logits/chosen": -0.6836889386177063,
+      "logits/rejected": -0.5880746841430664,
+      "logps/chosen": -4.447044849395752,
+      "logps/rejected": -5.132805824279785,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.447044849395752,
+      "rewards/margins": 0.6857603788375854,
+      "rewards/rejected": -5.132805824279785,
+      "sft_loss": 4.128294944763184,
+      "step": 2755
+    },
+    {
+      "epoch": 1.4771700953336677,
+      "grad_norm": 0.44709587630970354,
+      "learning_rate": 5.997780771720854e-07,
+      "logits/chosen": -0.8651409149169922,
+      "logits/rejected": -0.6353691816329956,
+      "logps/chosen": -4.558709144592285,
+      "logps/rejected": -5.283412933349609,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.558709144592285,
+      "rewards/margins": 0.7247046232223511,
+      "rewards/rejected": -5.283412933349609,
+      "sft_loss": 4.31949520111084,
+      "step": 2760
+    },
+    {
+      "epoch": 1.4798461281150694,
+      "grad_norm": 0.4967270396229397,
+      "learning_rate": 5.982515179388486e-07,
+      "logits/chosen": -0.6980944871902466,
+      "logits/rejected": -0.5798202753067017,
+      "logps/chosen": -4.641325950622559,
+      "logps/rejected": -5.210552215576172,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.641325950622559,
+      "rewards/margins": 0.5692263841629028,
+      "rewards/rejected": -5.210552215576172,
+      "sft_loss": 4.369772911071777,
+      "step": 2765
+    },
+    {
+      "epoch": 1.482522160896471,
+      "grad_norm": 0.414039866021628,
+      "learning_rate": 5.967240054689541e-07,
+      "logits/chosen": -0.8741037249565125,
+      "logits/rejected": -0.8427600860595703,
+      "logps/chosen": -4.335209846496582,
+      "logps/rejected": -4.899285316467285,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.335209846496582,
+      "rewards/margins": 0.5640758872032166,
+      "rewards/rejected": -4.899285316467285,
+      "sft_loss": 4.151376247406006,
+      "step": 2770
+    },
+    {
+      "epoch": 1.4851981936778724,
+      "grad_norm": 0.32893641730688494,
+      "learning_rate": 5.951955545823342e-07,
+      "logits/chosen": -0.8423898816108704,
+      "logits/rejected": -0.7706656455993652,
+      "logps/chosen": -4.534244537353516,
+      "logps/rejected": -5.130573272705078,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.534244537353516,
+      "rewards/margins": 0.5963292717933655,
+      "rewards/rejected": -5.130573272705078,
+      "sft_loss": 4.373254776000977,
+      "step": 2775
+    },
+    {
+      "epoch": 1.4878742264592741,
+      "grad_norm": 0.2899564595789967,
+      "learning_rate": 5.936661801080263e-07,
+      "logits/chosen": -0.8321715593338013,
+      "logits/rejected": -0.7527654767036438,
+      "logps/chosen": -4.561938285827637,
+      "logps/rejected": -5.114630222320557,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.561938285827637,
+      "rewards/margins": 0.5526921153068542,
+      "rewards/rejected": -5.114630222320557,
+      "sft_loss": 4.300882339477539,
+      "step": 2780
+    },
+    {
+      "epoch": 1.4905502592406756,
+      "grad_norm": 0.39218345035640906,
+      "learning_rate": 5.92135896884028e-07,
+      "logits/chosen": -0.8997092247009277,
+      "logits/rejected": -0.7471407055854797,
+      "logps/chosen": -4.617938041687012,
+      "logps/rejected": -5.369228839874268,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.617938041687012,
+      "rewards/margins": 0.7512913346290588,
+      "rewards/rejected": -5.369228839874268,
+      "sft_loss": 4.363840579986572,
+      "step": 2785
+    },
+    {
+      "epoch": 1.4932262920220774,
+      "grad_norm": 0.5000524915500977,
+      "learning_rate": 5.906047197571541e-07,
+      "logits/chosen": -0.7601592540740967,
+      "logits/rejected": -0.8226611018180847,
+      "logps/chosen": -4.56928014755249,
+      "logps/rejected": -4.973417282104492,
+      "loss": 0.0548,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.56928014755249,
+      "rewards/margins": 0.4041372835636139,
+      "rewards/rejected": -4.973417282104492,
+      "sft_loss": 4.3844685554504395,
+      "step": 2790
+    },
+    {
+      "epoch": 1.4959023248034788,
+      "grad_norm": 0.4501939041101977,
+      "learning_rate": 5.890726635828919e-07,
+      "logits/chosen": -0.6789978742599487,
+      "logits/rejected": -0.7200356721878052,
+      "logps/chosen": -4.450923919677734,
+      "logps/rejected": -4.913413047790527,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.450923919677734,
+      "rewards/margins": 0.46248936653137207,
+      "rewards/rejected": -4.913413047790527,
+      "sft_loss": 4.257745265960693,
+      "step": 2795
+    },
+    {
+      "epoch": 1.4985783575848803,
+      "grad_norm": 0.4064590009353923,
+      "learning_rate": 5.875397432252569e-07,
+      "logits/chosen": -0.8166916966438293,
+      "logits/rejected": -0.7958462834358215,
+      "logps/chosen": -4.582040786743164,
+      "logps/rejected": -5.0602827072143555,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.582040786743164,
+      "rewards/margins": 0.47824162244796753,
+      "rewards/rejected": -5.0602827072143555,
+      "sft_loss": 4.298692226409912,
+      "step": 2800
+    },
+    {
+      "epoch": 1.4985783575848803,
+      "eval_logits/chosen": -0.38490617275238037,
+      "eval_logits/rejected": -0.29844847321510315,
+      "eval_logps/chosen": -4.544572830200195,
+      "eval_logps/rejected": -5.137356758117676,
+      "eval_loss": 0.051060404628515244,
+      "eval_rewards/accuracies": 0.6802670359611511,
+      "eval_rewards/chosen": -4.544572830200195,
+      "eval_rewards/margins": 0.5927836298942566,
+      "eval_rewards/rejected": -5.137356758117676,
+      "eval_runtime": 43.3873,
+      "eval_samples_per_second": 31.0,
+      "eval_sft_loss": 4.224380970001221,
+      "eval_steps_per_second": 7.767,
+      "step": 2800
+    },
+    {
+      "epoch": 1.5012543903662818,
+      "grad_norm": 0.5857737743279292,
+      "learning_rate": 5.860059735566491e-07,
+      "logits/chosen": -0.986019492149353,
+      "logits/rejected": -0.7938674688339233,
+      "logps/chosen": -4.5381388664245605,
+      "logps/rejected": -5.097907066345215,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.5381388664245605,
+      "rewards/margins": 0.5597677230834961,
+      "rewards/rejected": -5.097907066345215,
+      "sft_loss": 4.319216728210449,
+      "step": 2805
+    },
+    {
+      "epoch": 1.5039304231476835,
+      "grad_norm": 0.5663653190019956,
+      "learning_rate": 5.844713694577087e-07,
+      "logits/chosen": -0.7099121809005737,
+      "logits/rejected": -0.6400435566902161,
+      "logps/chosen": -4.539187431335449,
+      "logps/rejected": -5.13986873626709,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.539187431335449,
+      "rewards/margins": 0.6006811857223511,
+      "rewards/rejected": -5.13986873626709,
+      "sft_loss": 4.334919452667236,
+      "step": 2810
+    },
+    {
+      "epoch": 1.5066064559290853,
+      "grad_norm": 0.8834082325413514,
+      "learning_rate": 5.829359458171714e-07,
+      "logits/chosen": -0.6759532690048218,
+      "logits/rejected": -0.5866900682449341,
+      "logps/chosen": -4.316249847412109,
+      "logps/rejected": -5.01406192779541,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.316249847412109,
+      "rewards/margins": 0.6978114247322083,
+      "rewards/rejected": -5.01406192779541,
+      "sft_loss": 4.148660659790039,
+      "step": 2815
+    },
+    {
+      "epoch": 1.5092824887104868,
+      "grad_norm": 0.7366505641829346,
+      "learning_rate": 5.81399717531724e-07,
+      "logits/chosen": -0.7597302198410034,
+      "logits/rejected": -0.5598629713058472,
+      "logps/chosen": -4.290135860443115,
+      "logps/rejected": -4.8505539894104,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.290135860443115,
+      "rewards/margins": 0.5604175329208374,
+      "rewards/rejected": -4.8505539894104,
+      "sft_loss": 4.115383148193359,
+      "step": 2820
+    },
+    {
+      "epoch": 1.5119585214918883,
+      "grad_norm": 0.4483292065178676,
+      "learning_rate": 5.798626995058602e-07,
+      "logits/chosen": -0.9454687833786011,
+      "logits/rejected": -0.7220587730407715,
+      "logps/chosen": -4.53515100479126,
+      "logps/rejected": -5.261549949645996,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.53515100479126,
+      "rewards/margins": 0.7263993620872498,
+      "rewards/rejected": -5.261549949645996,
+      "sft_loss": 4.332110404968262,
+      "step": 2825
+    },
+    {
+      "epoch": 1.51463455427329,
+      "grad_norm": 0.5592988396281663,
+      "learning_rate": 5.783249066517354e-07,
+      "logits/chosen": -0.8145958781242371,
+      "logits/rejected": -0.7614253163337708,
+      "logps/chosen": -4.5657124519348145,
+      "logps/rejected": -5.182963848114014,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.5657124519348145,
+      "rewards/margins": 0.6172509789466858,
+      "rewards/rejected": -5.182963848114014,
+      "sft_loss": 4.231114387512207,
+      "step": 2830
+    },
+    {
+      "epoch": 1.5173105870546915,
+      "grad_norm": 0.632303409364965,
+      "learning_rate": 5.767863538890228e-07,
+      "logits/chosen": -0.8183963894844055,
+      "logits/rejected": -0.6835195422172546,
+      "logps/chosen": -4.389924049377441,
+      "logps/rejected": -5.1250081062316895,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.389924049377441,
+      "rewards/margins": 0.7350836992263794,
+      "rewards/rejected": -5.1250081062316895,
+      "sft_loss": 4.124279499053955,
+      "step": 2835
+    },
+    {
+      "epoch": 1.519986619836093,
+      "grad_norm": 0.541651076628437,
+      "learning_rate": 5.75247056144768e-07,
+      "logits/chosen": -0.8295953869819641,
+      "logits/rejected": -0.8045207858085632,
+      "logps/chosen": -4.505274772644043,
+      "logps/rejected": -4.960772514343262,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.505274772644043,
+      "rewards/margins": 0.4554976522922516,
+      "rewards/rejected": -4.960772514343262,
+      "sft_loss": 4.231078147888184,
+      "step": 2840
+    },
+    {
+      "epoch": 1.5226626526174947,
+      "grad_norm": 0.40328763918897,
+      "learning_rate": 5.737070283532444e-07,
+      "logits/chosen": -0.9527303576469421,
+      "logits/rejected": -0.83274906873703,
+      "logps/chosen": -4.756344795227051,
+      "logps/rejected": -5.247215270996094,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.756344795227051,
+      "rewards/margins": 0.49087056517601013,
+      "rewards/rejected": -5.247215270996094,
+      "sft_loss": 4.509295463562012,
+      "step": 2845
+    },
+    {
+      "epoch": 1.5253386853988962,
+      "grad_norm": 0.5501878255143199,
+      "learning_rate": 5.721662854558084e-07,
+      "logits/chosen": -0.9534880518913269,
+      "logits/rejected": -0.9018028378486633,
+      "logps/chosen": -4.588342666625977,
+      "logps/rejected": -5.2513628005981445,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -4.588342666625977,
+      "rewards/margins": 0.6630192995071411,
+      "rewards/rejected": -5.2513628005981445,
+      "sft_loss": 4.240243911743164,
+      "step": 2850
+    },
+    {
+      "epoch": 1.5280147181802977,
+      "grad_norm": 0.4812915607269805,
+      "learning_rate": 5.706248424007545e-07,
+      "logits/chosen": -0.9855610728263855,
+      "logits/rejected": -0.7704046964645386,
+      "logps/chosen": -4.3409504890441895,
+      "logps/rejected": -4.86018180847168,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.3409504890441895,
+      "rewards/margins": 0.5192316174507141,
+      "rewards/rejected": -4.86018180847168,
+      "sft_loss": 4.120865821838379,
+      "step": 2855
+    },
+    {
+      "epoch": 1.5306907509616994,
+      "grad_norm": 0.45233228772754347,
+      "learning_rate": 5.690827141431699e-07,
+      "logits/chosen": -1.017059326171875,
+      "logits/rejected": -0.7848398089408875,
+      "logps/chosen": -4.593303680419922,
+      "logps/rejected": -5.0092973709106445,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.593303680419922,
+      "rewards/margins": 0.4159931242465973,
+      "rewards/rejected": -5.0092973709106445,
+      "sft_loss": 4.261613368988037,
+      "step": 2860
+    },
+    {
+      "epoch": 1.5333667837431009,
+      "grad_norm": 0.6506887003976763,
+      "learning_rate": 5.675399156447897e-07,
+      "logits/chosen": -1.0267689228057861,
+      "logits/rejected": -0.8769267797470093,
+      "logps/chosen": -4.629855155944824,
+      "logps/rejected": -5.098114967346191,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.629855155944824,
+      "rewards/margins": 0.4682607054710388,
+      "rewards/rejected": -5.098114967346191,
+      "sft_loss": 4.358528137207031,
+      "step": 2865
+    },
+    {
+      "epoch": 1.5360428165245024,
+      "grad_norm": 0.4798389011577337,
+      "learning_rate": 5.659964618738515e-07,
+      "logits/chosen": -0.9780286550521851,
+      "logits/rejected": -0.8610752820968628,
+      "logps/chosen": -4.556265830993652,
+      "logps/rejected": -5.136031150817871,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.556265830993652,
+      "rewards/margins": 0.5797653794288635,
+      "rewards/rejected": -5.136031150817871,
+      "sft_loss": 4.165777206420898,
+      "step": 2870
+    },
+    {
+      "epoch": 1.538718849305904,
+      "grad_norm": 0.5422032747817679,
+      "learning_rate": 5.644523678049509e-07,
+      "logits/chosen": -0.9687239527702332,
+      "logits/rejected": -0.8813830614089966,
+      "logps/chosen": -4.5033183097839355,
+      "logps/rejected": -4.969407558441162,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.5033183097839355,
+      "rewards/margins": 0.4660890996456146,
+      "rewards/rejected": -4.969407558441162,
+      "sft_loss": 4.0725483894348145,
+      "step": 2875
+    },
+    {
+      "epoch": 1.5413948820873056,
+      "grad_norm": 0.5901973530137451,
+      "learning_rate": 5.629076484188952e-07,
+      "logits/chosen": -0.7941701412200928,
+      "logits/rejected": -0.7139784693717957,
+      "logps/chosen": -4.662365913391113,
+      "logps/rejected": -5.221032619476318,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.662365913391113,
+      "rewards/margins": 0.5586673617362976,
+      "rewards/rejected": -5.221032619476318,
+      "sft_loss": 4.298089027404785,
+      "step": 2880
+    },
+    {
+      "epoch": 1.544070914868707,
+      "grad_norm": 0.364967962468647,
+      "learning_rate": 5.613623187025587e-07,
+      "logits/chosen": -0.8740784525871277,
+      "logits/rejected": -0.7707508206367493,
+      "logps/chosen": -4.610237121582031,
+      "logps/rejected": -5.228066444396973,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.610237121582031,
+      "rewards/margins": 0.617829442024231,
+      "rewards/rejected": -5.228066444396973,
+      "sft_loss": 4.25256872177124,
+      "step": 2885
+    },
+    {
+      "epoch": 1.5467469476501088,
+      "grad_norm": 0.479594255068986,
+      "learning_rate": 5.598163936487369e-07,
+      "logits/chosen": -0.8517980575561523,
+      "logits/rejected": -0.6656395196914673,
+      "logps/chosen": -4.480307102203369,
+      "logps/rejected": -5.1197099685668945,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.480307102203369,
+      "rewards/margins": 0.6394029259681702,
+      "rewards/rejected": -5.1197099685668945,
+      "sft_loss": 4.178659439086914,
+      "step": 2890
+    },
+    {
+      "epoch": 1.5494229804315103,
+      "grad_norm": 0.38971936523868217,
+      "learning_rate": 5.582698882560017e-07,
+      "logits/chosen": -0.8685995936393738,
+      "logits/rejected": -0.692309558391571,
+      "logps/chosen": -4.6656389236450195,
+      "logps/rejected": -5.286332130432129,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.6656389236450195,
+      "rewards/margins": 0.6206930875778198,
+      "rewards/rejected": -5.286332130432129,
+      "sft_loss": 4.3696160316467285,
+      "step": 2895
+    },
+    {
+      "epoch": 1.5520990132129118,
+      "grad_norm": 0.44187326201186605,
+      "learning_rate": 5.567228175285549e-07,
+      "logits/chosen": -0.706171452999115,
+      "logits/rejected": -0.6334139108657837,
+      "logps/chosen": -4.3591508865356445,
+      "logps/rejected": -4.947355270385742,
+      "loss": 0.0502,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.3591508865356445,
+      "rewards/margins": 0.5882046222686768,
+      "rewards/rejected": -4.947355270385742,
+      "sft_loss": 3.9303412437438965,
+      "step": 2900
+    },
+    {
+      "epoch": 1.5547750459943135,
+      "grad_norm": 0.525767187888611,
+      "learning_rate": 5.551751964760838e-07,
+      "logits/chosen": -0.6887973546981812,
+      "logits/rejected": -0.7219077348709106,
+      "logps/chosen": -4.609784126281738,
+      "logps/rejected": -5.058163642883301,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.609784126281738,
+      "rewards/margins": 0.4483796954154968,
+      "rewards/rejected": -5.058163642883301,
+      "sft_loss": 4.3274431228637695,
+      "step": 2905
+    },
+    {
+      "epoch": 1.557451078775715,
+      "grad_norm": 0.354163445464275,
+      "learning_rate": 5.536270401136145e-07,
+      "logits/chosen": -0.8383262753486633,
+      "logits/rejected": -0.7376397252082825,
+      "logps/chosen": -4.493884086608887,
+      "logps/rejected": -5.036890029907227,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.493884086608887,
+      "rewards/margins": 0.5430058836936951,
+      "rewards/rejected": -5.036890029907227,
+      "sft_loss": 4.180073261260986,
+      "step": 2910
+    },
+    {
+      "epoch": 1.5601271115571165,
+      "grad_norm": 0.4674214316597324,
+      "learning_rate": 5.520783634613667e-07,
+      "logits/chosen": -0.7461773753166199,
+      "logits/rejected": -0.522574245929718,
+      "logps/chosen": -4.679784774780273,
+      "logps/rejected": -5.226053714752197,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.679784774780273,
+      "rewards/margins": 0.5462688207626343,
+      "rewards/rejected": -5.226053714752197,
+      "sft_loss": 4.342376708984375,
+      "step": 2915
+    },
+    {
+      "epoch": 1.5628031443385182,
+      "grad_norm": 1.2543196580616705,
+      "learning_rate": 5.505291815446082e-07,
+      "logits/chosen": -0.749670147895813,
+      "logits/rejected": -0.6325221657752991,
+      "logps/chosen": -4.405003547668457,
+      "logps/rejected": -5.0354204177856445,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.405003547668457,
+      "rewards/margins": 0.6304169297218323,
+      "rewards/rejected": -5.0354204177856445,
+      "sft_loss": 4.188388347625732,
+      "step": 2920
+    },
+    {
+      "epoch": 1.5654791771199197,
+      "grad_norm": 0.5222585233235869,
+      "learning_rate": 5.489795093935089e-07,
+      "logits/chosen": -0.7664873600006104,
+      "logits/rejected": -0.679275631904602,
+      "logps/chosen": -4.5715131759643555,
+      "logps/rejected": -5.1358561515808105,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.5715131759643555,
+      "rewards/margins": 0.5643435716629028,
+      "rewards/rejected": -5.1358561515808105,
+      "sft_loss": 4.250007629394531,
+      "step": 2925
+    },
+    {
+      "epoch": 1.5681552099013212,
+      "grad_norm": 0.3737733749118266,
+      "learning_rate": 5.474293620429946e-07,
+      "logits/chosen": -0.9465273022651672,
+      "logits/rejected": -0.7799872159957886,
+      "logps/chosen": -4.570036888122559,
+      "logps/rejected": -5.4389424324035645,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.570036888122559,
+      "rewards/margins": 0.8689058423042297,
+      "rewards/rejected": -5.4389424324035645,
+      "sft_loss": 4.252045631408691,
+      "step": 2930
+    },
+    {
+      "epoch": 1.570831242682723,
+      "grad_norm": 0.5110524346117336,
+      "learning_rate": 5.458787545326018e-07,
+      "logits/chosen": -0.8712457418441772,
+      "logits/rejected": -0.7498850226402283,
+      "logps/chosen": -4.555843830108643,
+      "logps/rejected": -5.041597366333008,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.555843830108643,
+      "rewards/margins": 0.4857536256313324,
+      "rewards/rejected": -5.041597366333008,
+      "sft_loss": 4.287815570831299,
+      "step": 2935
+    },
+    {
+      "epoch": 1.5735072754641244,
+      "grad_norm": 0.4193933937806095,
+      "learning_rate": 5.443277019063311e-07,
+      "logits/chosen": -0.8290327787399292,
+      "logits/rejected": -0.6746954917907715,
+      "logps/chosen": -4.528292655944824,
+      "logps/rejected": -5.201540470123291,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.528292655944824,
+      "rewards/margins": 0.6732484698295593,
+      "rewards/rejected": -5.201540470123291,
+      "sft_loss": 4.318000793457031,
+      "step": 2940
+    },
+    {
+      "epoch": 1.5761833082455259,
+      "grad_norm": 0.7984256655785995,
+      "learning_rate": 5.427762192125023e-07,
+      "logits/chosen": -0.7302804589271545,
+      "logits/rejected": -0.6393738985061646,
+      "logps/chosen": -4.297086238861084,
+      "logps/rejected": -4.874281406402588,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.297086238861084,
+      "rewards/margins": 0.5771942734718323,
+      "rewards/rejected": -4.874281406402588,
+      "sft_loss": 4.018523693084717,
+      "step": 2945
+    },
+    {
+      "epoch": 1.5788593410269276,
+      "grad_norm": 0.5139984575136903,
+      "learning_rate": 5.41224321503607e-07,
+      "logits/chosen": -0.8039859533309937,
+      "logits/rejected": -0.5436501502990723,
+      "logps/chosen": -4.477331638336182,
+      "logps/rejected": -5.2113447189331055,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.477331638336182,
+      "rewards/margins": 0.7340143322944641,
+      "rewards/rejected": -5.2113447189331055,
+      "sft_loss": 4.270988464355469,
+      "step": 2950
+    },
+    {
+      "epoch": 1.5815353738083293,
+      "grad_norm": 0.5318063905528159,
+      "learning_rate": 5.396720238361637e-07,
+      "logits/chosen": -0.7131645679473877,
+      "logits/rejected": -0.5724089741706848,
+      "logps/chosen": -4.619770526885986,
+      "logps/rejected": -5.129917621612549,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.619770526885986,
+      "rewards/margins": 0.510147213935852,
+      "rewards/rejected": -5.129917621612549,
+      "sft_loss": 4.325153350830078,
+      "step": 2955
+    },
+    {
+      "epoch": 1.5842114065897306,
+      "grad_norm": 0.6270796231497009,
+      "learning_rate": 5.381193412705711e-07,
+      "logits/chosen": -0.8254559636116028,
+      "logits/rejected": -0.6606593728065491,
+      "logps/chosen": -4.438187122344971,
+      "logps/rejected": -5.023268222808838,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.438187122344971,
+      "rewards/margins": 0.5850812196731567,
+      "rewards/rejected": -5.023268222808838,
+      "sft_loss": 4.149109840393066,
+      "step": 2960
+    },
+    {
+      "epoch": 1.5868874393711323,
+      "grad_norm": 0.3628875605274503,
+      "learning_rate": 5.365662888709622e-07,
+      "logits/chosen": -0.8125503659248352,
+      "logits/rejected": -0.650180459022522,
+      "logps/chosen": -4.428151607513428,
+      "logps/rejected": -5.005454063415527,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.428151607513428,
+      "rewards/margins": 0.5773029923439026,
+      "rewards/rejected": -5.005454063415527,
+      "sft_loss": 4.177712917327881,
+      "step": 2965
+    },
+    {
+      "epoch": 1.589563472152534,
+      "grad_norm": 0.4408978324808102,
+      "learning_rate": 5.350128817050585e-07,
+      "logits/chosen": -0.8711091876029968,
+      "logits/rejected": -0.6683140993118286,
+      "logps/chosen": -4.5739264488220215,
+      "logps/rejected": -5.2639336585998535,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.5739264488220215,
+      "rewards/margins": 0.690007209777832,
+      "rewards/rejected": -5.2639336585998535,
+      "sft_loss": 4.209588050842285,
+      "step": 2970
+    },
+    {
+      "epoch": 1.5922395049339353,
+      "grad_norm": 0.634029397334541,
+      "learning_rate": 5.334591348440229e-07,
+      "logits/chosen": -0.8157389760017395,
+      "logits/rejected": -0.6465741395950317,
+      "logps/chosen": -4.551390171051025,
+      "logps/rejected": -5.187037944793701,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.551390171051025,
+      "rewards/margins": 0.6356481313705444,
+      "rewards/rejected": -5.187037944793701,
+      "sft_loss": 4.29290771484375,
+      "step": 2975
+    },
+    {
+      "epoch": 1.594915537715337,
+      "grad_norm": 0.4523710769588691,
+      "learning_rate": 5.319050633623141e-07,
+      "logits/chosen": -0.8819277882575989,
+      "logits/rejected": -0.7007160782814026,
+      "logps/chosen": -4.540030002593994,
+      "logps/rejected": -5.0742316246032715,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.540030002593994,
+      "rewards/margins": 0.5342013239860535,
+      "rewards/rejected": -5.0742316246032715,
+      "sft_loss": 4.217968463897705,
+      "step": 2980
+    },
+    {
+      "epoch": 1.5975915704967387,
+      "grad_norm": 0.5121523451628749,
+      "learning_rate": 5.303506823375409e-07,
+      "logits/chosen": -0.8410658836364746,
+      "logits/rejected": -0.6168532371520996,
+      "logps/chosen": -4.4125237464904785,
+      "logps/rejected": -5.1476545333862305,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.4125237464904785,
+      "rewards/margins": 0.7351310849189758,
+      "rewards/rejected": -5.1476545333862305,
+      "sft_loss": 4.170558929443359,
+      "step": 2985
+    },
+    {
+      "epoch": 1.60026760327814,
+      "grad_norm": 0.4242550067163679,
+      "learning_rate": 5.287960068503143e-07,
+      "logits/chosen": -0.8414437174797058,
+      "logits/rejected": -0.6301363110542297,
+      "logps/chosen": -4.453303337097168,
+      "logps/rejected": -5.125790596008301,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.453303337097168,
+      "rewards/margins": 0.6724871397018433,
+      "rewards/rejected": -5.125790596008301,
+      "sft_loss": 4.234655857086182,
+      "step": 2990
+    },
+    {
+      "epoch": 1.6029436360595417,
+      "grad_norm": 0.6150856743479923,
+      "learning_rate": 5.272410519841032e-07,
+      "logits/chosen": -0.7048937678337097,
+      "logits/rejected": -0.6164297461509705,
+      "logps/chosen": -4.490935325622559,
+      "logps/rejected": -5.310682773590088,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.490935325622559,
+      "rewards/margins": 0.819747805595398,
+      "rewards/rejected": -5.310682773590088,
+      "sft_loss": 4.104347229003906,
+      "step": 2995
+    },
+    {
+      "epoch": 1.6056196688409434,
+      "grad_norm": 0.44023608513343926,
+      "learning_rate": 5.256858328250861e-07,
+      "logits/chosen": -0.8334380388259888,
+      "logits/rejected": -0.6159783601760864,
+      "logps/chosen": -4.609116554260254,
+      "logps/rejected": -5.1678619384765625,
+      "loss": 0.052,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.609116554260254,
+      "rewards/margins": 0.5587445497512817,
+      "rewards/rejected": -5.1678619384765625,
+      "sft_loss": 4.265917778015137,
+      "step": 3000
+    },
+    {
+      "epoch": 1.608295701622345,
+      "grad_norm": 0.6277862194810916,
+      "learning_rate": 5.241303644620063e-07,
+      "logits/chosen": -0.9017225503921509,
+      "logits/rejected": -0.6855077147483826,
+      "logps/chosen": -4.580887794494629,
+      "logps/rejected": -5.039603233337402,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.580887794494629,
+      "rewards/margins": 0.4587152898311615,
+      "rewards/rejected": -5.039603233337402,
+      "sft_loss": 4.258471488952637,
+      "step": 3005
+    },
+    {
+      "epoch": 1.6109717344037464,
+      "grad_norm": 0.3973293347863802,
+      "learning_rate": 5.225746619860248e-07,
+      "logits/chosen": -0.9179320335388184,
+      "logits/rejected": -0.7894536852836609,
+      "logps/chosen": -4.528168678283691,
+      "logps/rejected": -5.234195709228516,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.528168678283691,
+      "rewards/margins": 0.7060272097587585,
+      "rewards/rejected": -5.234195709228516,
+      "sft_loss": 4.231138229370117,
+      "step": 3010
+    },
+    {
+      "epoch": 1.6136477671851481,
+      "grad_norm": 0.4052136199043096,
+      "learning_rate": 5.210187404905735e-07,
+      "logits/chosen": -0.7160056829452515,
+      "logits/rejected": -0.6443689465522766,
+      "logps/chosen": -4.8330841064453125,
+      "logps/rejected": -5.205080986022949,
+      "loss": 0.0539,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.8330841064453125,
+      "rewards/margins": 0.3719966411590576,
+      "rewards/rejected": -5.205080986022949,
+      "sft_loss": 4.528097152709961,
+      "step": 3015
+    },
+    {
+      "epoch": 1.6163237999665496,
+      "grad_norm": 0.327549476063512,
+      "learning_rate": 5.194626150712098e-07,
+      "logits/chosen": -0.9027112722396851,
+      "logits/rejected": -0.7617613077163696,
+      "logps/chosen": -4.600415229797363,
+      "logps/rejected": -5.1396894454956055,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.600415229797363,
+      "rewards/margins": 0.539274275302887,
+      "rewards/rejected": -5.1396894454956055,
+      "sft_loss": 4.4282941818237305,
+      "step": 3020
+    },
+    {
+      "epoch": 1.6189998327479511,
+      "grad_norm": 0.3866774364589384,
+      "learning_rate": 5.179063008254695e-07,
+      "logits/chosen": -0.8924050331115723,
+      "logits/rejected": -0.692057728767395,
+      "logps/chosen": -4.390013694763184,
+      "logps/rejected": -4.872861385345459,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.390013694763184,
+      "rewards/margins": 0.48284751176834106,
+      "rewards/rejected": -4.872861385345459,
+      "sft_loss": 4.109368324279785,
+      "step": 3025
+    },
+    {
+      "epoch": 1.6216758655293528,
+      "grad_norm": 0.5603972389461411,
+      "learning_rate": 5.163498128527199e-07,
+      "logits/chosen": -0.8043405413627625,
+      "logits/rejected": -0.6585147976875305,
+      "logps/chosen": -4.4106950759887695,
+      "logps/rejected": -4.972418308258057,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.4106950759887695,
+      "rewards/margins": 0.5617232322692871,
+      "rewards/rejected": -4.972418308258057,
+      "sft_loss": 4.162978172302246,
+      "step": 3030
+    },
+    {
+      "epoch": 1.6243518983107543,
+      "grad_norm": 0.5226850952217248,
+      "learning_rate": 5.147931662540144e-07,
+      "logits/chosen": -0.7547642588615417,
+      "logits/rejected": -0.6366636753082275,
+      "logps/chosen": -4.469838619232178,
+      "logps/rejected": -4.992844581604004,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.469838619232178,
+      "rewards/margins": 0.523006021976471,
+      "rewards/rejected": -4.992844581604004,
+      "sft_loss": 4.131374359130859,
+      "step": 3035
+    },
+    {
+      "epoch": 1.6270279310921558,
+      "grad_norm": 0.5621502773295101,
+      "learning_rate": 5.132363761319449e-07,
+      "logits/chosen": -0.8233652114868164,
+      "logits/rejected": -0.7582941651344299,
+      "logps/chosen": -4.578380584716797,
+      "logps/rejected": -5.224091053009033,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.578380584716797,
+      "rewards/margins": 0.6457099914550781,
+      "rewards/rejected": -5.224091053009033,
+      "sft_loss": 4.231387615203857,
+      "step": 3040
+    },
+    {
+      "epoch": 1.6297039638735575,
+      "grad_norm": 0.5384394600398351,
+      "learning_rate": 5.116794575904962e-07,
+      "logits/chosen": -0.6828616261482239,
+      "logits/rejected": -0.6190515756607056,
+      "logps/chosen": -4.537975788116455,
+      "logps/rejected": -5.16754150390625,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.537975788116455,
+      "rewards/margins": 0.6295658946037292,
+      "rewards/rejected": -5.16754150390625,
+      "sft_loss": 4.14799690246582,
+      "step": 3045
+    },
+    {
+      "epoch": 1.632379996654959,
+      "grad_norm": 0.3570274089495265,
+      "learning_rate": 5.101224257348987e-07,
+      "logits/chosen": -0.7118021845817566,
+      "logits/rejected": -0.6111310124397278,
+      "logps/chosen": -4.623868942260742,
+      "logps/rejected": -5.227258682250977,
+      "loss": 0.051,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.623868942260742,
+      "rewards/margins": 0.6033896207809448,
+      "rewards/rejected": -5.227258682250977,
+      "sft_loss": 4.14676570892334,
+      "step": 3050
+    },
+    {
+      "epoch": 1.6350560294363605,
+      "grad_norm": 0.41812252336808753,
+      "learning_rate": 5.085652956714823e-07,
+      "logits/chosen": -0.8265964388847351,
+      "logits/rejected": -0.6396560072898865,
+      "logps/chosen": -4.667733669281006,
+      "logps/rejected": -5.240296840667725,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.667733669281006,
+      "rewards/margins": 0.572563111782074,
+      "rewards/rejected": -5.240296840667725,
+      "sft_loss": 4.416812419891357,
+      "step": 3055
+    },
+    {
+      "epoch": 1.6377320622177622,
+      "grad_norm": 0.4141573304369751,
+      "learning_rate": 5.070080825075298e-07,
+      "logits/chosen": -0.6839703321456909,
+      "logits/rejected": -0.4868551194667816,
+      "logps/chosen": -4.377912998199463,
+      "logps/rejected": -5.105005741119385,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.377912998199463,
+      "rewards/margins": 0.7270928025245667,
+      "rewards/rejected": -5.105005741119385,
+      "sft_loss": 4.107635974884033,
+      "step": 3060
+    },
+    {
+      "epoch": 1.6404080949991637,
+      "grad_norm": 0.5889041881610368,
+      "learning_rate": 5.0545080135113e-07,
+      "logits/chosen": -0.6704410910606384,
+      "logits/rejected": -0.5773349404335022,
+      "logps/chosen": -4.510235786437988,
+      "logps/rejected": -5.127143383026123,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.510235786437988,
+      "rewards/margins": 0.6169074177742004,
+      "rewards/rejected": -5.127143383026123,
+      "sft_loss": 4.2573747634887695,
+      "step": 3065
+    },
+    {
+      "epoch": 1.6430841277805652,
+      "grad_norm": 0.5491081889665882,
+      "learning_rate": 5.038934673110316e-07,
+      "logits/chosen": -0.7946319580078125,
+      "logits/rejected": -0.6988022327423096,
+      "logps/chosen": -4.589323043823242,
+      "logps/rejected": -5.201617240905762,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.589323043823242,
+      "rewards/margins": 0.6122941374778748,
+      "rewards/rejected": -5.201617240905762,
+      "sft_loss": 4.306293487548828,
+      "step": 3070
+    },
+    {
+      "epoch": 1.645760160561967,
+      "grad_norm": 0.40370148653990523,
+      "learning_rate": 5.023360954964963e-07,
+      "logits/chosen": -0.8991419076919556,
+      "logits/rejected": -0.8406192660331726,
+      "logps/chosen": -4.753389358520508,
+      "logps/rejected": -5.2947564125061035,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.753389358520508,
+      "rewards/margins": 0.5413663983345032,
+      "rewards/rejected": -5.2947564125061035,
+      "sft_loss": 4.370591640472412,
+      "step": 3075
+    },
+    {
+      "epoch": 1.6484361933433684,
+      "grad_norm": 0.4667987266950202,
+      "learning_rate": 5.007787010171524e-07,
+      "logits/chosen": -0.926565945148468,
+      "logits/rejected": -0.7107774615287781,
+      "logps/chosen": -4.401252746582031,
+      "logps/rejected": -5.139235019683838,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.401252746582031,
+      "rewards/margins": 0.7379823923110962,
+      "rewards/rejected": -5.139235019683838,
+      "sft_loss": 4.14643669128418,
+      "step": 3080
+    },
+    {
+      "epoch": 1.65111222612477,
+      "grad_norm": 0.28977187648312863,
+      "learning_rate": 4.992212989828477e-07,
+      "logits/chosen": -0.7511423826217651,
+      "logits/rejected": -0.7302947044372559,
+      "logps/chosen": -4.628520965576172,
+      "logps/rejected": -5.08123779296875,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.628520965576172,
+      "rewards/margins": 0.45271721482276917,
+      "rewards/rejected": -5.08123779296875,
+      "sft_loss": 4.332284927368164,
+      "step": 3085
+    },
+    {
+      "epoch": 1.6537882589061716,
+      "grad_norm": 0.5128071746578113,
+      "learning_rate": 4.976639045035036e-07,
+      "logits/chosen": -0.7045882940292358,
+      "logits/rejected": -0.651595413684845,
+      "logps/chosen": -4.456996440887451,
+      "logps/rejected": -4.9166107177734375,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.456996440887451,
+      "rewards/margins": 0.45961475372314453,
+      "rewards/rejected": -4.9166107177734375,
+      "sft_loss": 4.227812767028809,
+      "step": 3090
+    },
+    {
+      "epoch": 1.6564642916875731,
+      "grad_norm": 0.7439117424034836,
+      "learning_rate": 4.961065326889683e-07,
+      "logits/chosen": -0.7401078939437866,
+      "logits/rejected": -0.5558581948280334,
+      "logps/chosen": -4.522768974304199,
+      "logps/rejected": -5.026648044586182,
+      "loss": 0.0539,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.522768974304199,
+      "rewards/margins": 0.5038790702819824,
+      "rewards/rejected": -5.026648044586182,
+      "sft_loss": 4.227447032928467,
+      "step": 3095
+    },
+    {
+      "epoch": 1.6591403244689746,
+      "grad_norm": 0.44974777016917117,
+      "learning_rate": 4.9454919864887e-07,
+      "logits/chosen": -0.8705316781997681,
+      "logits/rejected": -0.7544268369674683,
+      "logps/chosen": -4.580806732177734,
+      "logps/rejected": -5.142989158630371,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.580806732177734,
+      "rewards/margins": 0.5621822476387024,
+      "rewards/rejected": -5.142989158630371,
+      "sft_loss": 4.323142051696777,
+      "step": 3100
+    },
+    {
+      "epoch": 1.6618163572503764,
+      "grad_norm": 0.60351015808472,
+      "learning_rate": 4.929919174924701e-07,
+      "logits/chosen": -0.9598020315170288,
+      "logits/rejected": -0.7137739062309265,
+      "logps/chosen": -4.588383674621582,
+      "logps/rejected": -5.193715572357178,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.588383674621582,
+      "rewards/margins": 0.6053324937820435,
+      "rewards/rejected": -5.193715572357178,
+      "sft_loss": 4.364377498626709,
+      "step": 3105
+    },
+    {
+      "epoch": 1.6644923900317778,
+      "grad_norm": 0.39383948253171636,
+      "learning_rate": 4.914347043285177e-07,
+      "logits/chosen": -0.7553201913833618,
+      "logits/rejected": -0.5945597290992737,
+      "logps/chosen": -4.58227014541626,
+      "logps/rejected": -5.167197227478027,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.58227014541626,
+      "rewards/margins": 0.5849268436431885,
+      "rewards/rejected": -5.167197227478027,
+      "sft_loss": 4.150500297546387,
+      "step": 3110
+    },
+    {
+      "epoch": 1.6671684228131793,
+      "grad_norm": 0.5947338725544065,
+      "learning_rate": 4.898775742651013e-07,
+      "logits/chosen": -0.6695328950881958,
+      "logits/rejected": -0.5480534434318542,
+      "logps/chosen": -4.455540657043457,
+      "logps/rejected": -5.015857696533203,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.455540657043457,
+      "rewards/margins": 0.5603172183036804,
+      "rewards/rejected": -5.015857696533203,
+      "sft_loss": 4.052641868591309,
+      "step": 3115
+    },
+    {
+      "epoch": 1.669844455594581,
+      "grad_norm": 0.3739608220920118,
+      "learning_rate": 4.883205424095037e-07,
+      "logits/chosen": -0.8755930066108704,
+      "logits/rejected": -0.6813184022903442,
+      "logps/chosen": -4.374234199523926,
+      "logps/rejected": -5.120595455169678,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.374234199523926,
+      "rewards/margins": 0.7463610172271729,
+      "rewards/rejected": -5.120595455169678,
+      "sft_loss": 4.135296821594238,
+      "step": 3120
+    },
+    {
+      "epoch": 1.6725204883759828,
+      "grad_norm": 0.47999415721425,
+      "learning_rate": 4.86763623868055e-07,
+      "logits/chosen": -0.7871851921081543,
+      "logits/rejected": -0.6408876180648804,
+      "logps/chosen": -4.715963363647461,
+      "logps/rejected": -5.292166233062744,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.715963363647461,
+      "rewards/margins": 0.5762028098106384,
+      "rewards/rejected": -5.292166233062744,
+      "sft_loss": 4.366208076477051,
+      "step": 3125
+    },
+    {
+      "epoch": 1.675196521157384,
+      "grad_norm": 0.36852219394257385,
+      "learning_rate": 4.852068337459856e-07,
+      "logits/chosen": -0.7195180654525757,
+      "logits/rejected": -0.532809853553772,
+      "logps/chosen": -4.462833404541016,
+      "logps/rejected": -5.156439304351807,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.462833404541016,
+      "rewards/margins": 0.6936055421829224,
+      "rewards/rejected": -5.156439304351807,
+      "sft_loss": 4.119335174560547,
+      "step": 3130
+    },
+    {
+      "epoch": 1.6778725539387858,
+      "grad_norm": 0.40507193582200285,
+      "learning_rate": 4.8365018714728e-07,
+      "logits/chosen": -0.7378911375999451,
+      "logits/rejected": -0.6666657328605652,
+      "logps/chosen": -4.564920425415039,
+      "logps/rejected": -5.0844197273254395,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.564920425415039,
+      "rewards/margins": 0.5194991827011108,
+      "rewards/rejected": -5.0844197273254395,
+      "sft_loss": 4.275947093963623,
+      "step": 3135
+    },
+    {
+      "epoch": 1.6805485867201875,
+      "grad_norm": 0.31724766919051245,
+      "learning_rate": 4.820936991745304e-07,
+      "logits/chosen": -1.0948458909988403,
+      "logits/rejected": -0.9697257876396179,
+      "logps/chosen": -4.6035237312316895,
+      "logps/rejected": -5.034295082092285,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.6035237312316895,
+      "rewards/margins": 0.43077144026756287,
+      "rewards/rejected": -5.034295082092285,
+      "sft_loss": 4.244277000427246,
+      "step": 3140
+    },
+    {
+      "epoch": 1.6832246195015887,
+      "grad_norm": 0.5310039889396487,
+      "learning_rate": 4.8053738492879e-07,
+      "logits/chosen": -0.8384534120559692,
+      "logits/rejected": -0.7171124815940857,
+      "logps/chosen": -4.526313781738281,
+      "logps/rejected": -5.261399269104004,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.526313781738281,
+      "rewards/margins": 0.7350856065750122,
+      "rewards/rejected": -5.261399269104004,
+      "sft_loss": 4.204371452331543,
+      "step": 3145
+    },
+    {
+      "epoch": 1.6859006522829905,
+      "grad_norm": 0.5335564765964186,
+      "learning_rate": 4.789812595094265e-07,
+      "logits/chosen": -0.9797199368476868,
+      "logits/rejected": -0.8858574032783508,
+      "logps/chosen": -4.503859519958496,
+      "logps/rejected": -5.087957382202148,
+      "loss": 0.052,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.503859519958496,
+      "rewards/margins": 0.5840980410575867,
+      "rewards/rejected": -5.087957382202148,
+      "sft_loss": 4.208001613616943,
+      "step": 3150
+    },
+    {
+      "epoch": 1.6885766850643922,
+      "grad_norm": 0.42218902416314297,
+      "learning_rate": 4.774253380139752e-07,
+      "logits/chosen": -0.9380515813827515,
+      "logits/rejected": -0.8422372937202454,
+      "logps/chosen": -4.567145347595215,
+      "logps/rejected": -5.0723161697387695,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.567145347595215,
+      "rewards/margins": 0.5051710605621338,
+      "rewards/rejected": -5.0723161697387695,
+      "sft_loss": 4.258059501647949,
+      "step": 3155
+    },
+    {
+      "epoch": 1.6912527178457935,
+      "grad_norm": 0.48659534318798275,
+      "learning_rate": 4.758696355379936e-07,
+      "logits/chosen": -0.6386554837226868,
+      "logits/rejected": -0.7299365401268005,
+      "logps/chosen": -4.623234272003174,
+      "logps/rejected": -5.053004264831543,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.623234272003174,
+      "rewards/margins": 0.429770290851593,
+      "rewards/rejected": -5.053004264831543,
+      "sft_loss": 4.323592662811279,
+      "step": 3160
+    },
+    {
+      "epoch": 1.6939287506271952,
+      "grad_norm": 0.4845276460062366,
+      "learning_rate": 4.743141671749138e-07,
+      "logits/chosen": -0.9454347491264343,
+      "logits/rejected": -0.794151782989502,
+      "logps/chosen": -4.372522354125977,
+      "logps/rejected": -4.8364152908325195,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.372522354125977,
+      "rewards/margins": 0.46389341354370117,
+      "rewards/rejected": -4.8364152908325195,
+      "sft_loss": 4.175771236419678,
+      "step": 3165
+    },
+    {
+      "epoch": 1.6966047834085969,
+      "grad_norm": 0.3380040956576745,
+      "learning_rate": 4.727589480158968e-07,
+      "logits/chosen": -0.8804110288619995,
+      "logits/rejected": -0.7924290299415588,
+      "logps/chosen": -4.5793280601501465,
+      "logps/rejected": -5.261588096618652,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.5793280601501465,
+      "rewards/margins": 0.6822600960731506,
+      "rewards/rejected": -5.261588096618652,
+      "sft_loss": 4.343472003936768,
+      "step": 3170
+    },
+    {
+      "epoch": 1.6992808161899984,
+      "grad_norm": 0.4022845360049597,
+      "learning_rate": 4.712039931496855e-07,
+      "logits/chosen": -0.8952493667602539,
+      "logits/rejected": -0.7143739461898804,
+      "logps/chosen": -4.63895845413208,
+      "logps/rejected": -5.060461521148682,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.63895845413208,
+      "rewards/margins": 0.42150363326072693,
+      "rewards/rejected": -5.060461521148682,
+      "sft_loss": 4.36241340637207,
+      "step": 3175
+    },
+    {
+      "epoch": 1.7019568489713999,
+      "grad_norm": 0.35743884470081916,
+      "learning_rate": 4.6964931766245905e-07,
+      "logits/chosen": -0.7012723684310913,
+      "logits/rejected": -0.6638824343681335,
+      "logps/chosen": -4.4255876541137695,
+      "logps/rejected": -4.9864630699157715,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.4255876541137695,
+      "rewards/margins": 0.5608752369880676,
+      "rewards/rejected": -4.9864630699157715,
+      "sft_loss": 4.170031547546387,
+      "step": 3180
+    },
+    {
+      "epoch": 1.7046328817528016,
+      "grad_norm": 0.388598888460869,
+      "learning_rate": 4.6809493663768575e-07,
+      "logits/chosen": -0.7917588353157043,
+      "logits/rejected": -0.7726391553878784,
+      "logps/chosen": -4.632717132568359,
+      "logps/rejected": -4.928259372711182,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -4.632717132568359,
+      "rewards/margins": 0.29554182291030884,
+      "rewards/rejected": -4.928259372711182,
+      "sft_loss": 4.243005275726318,
+      "step": 3185
+    },
+    {
+      "epoch": 1.707308914534203,
+      "grad_norm": 0.43334284868642725,
+      "learning_rate": 4.6654086515597716e-07,
+      "logits/chosen": -0.9887200593948364,
+      "logits/rejected": -0.7736762762069702,
+      "logps/chosen": -4.42282772064209,
+      "logps/rejected": -5.094423294067383,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.42282772064209,
+      "rewards/margins": 0.6715952754020691,
+      "rewards/rejected": -5.094423294067383,
+      "sft_loss": 4.217164516448975,
+      "step": 3190
+    },
+    {
+      "epoch": 1.7099849473156046,
+      "grad_norm": 0.551121726681534,
+      "learning_rate": 4.6498711829494154e-07,
+      "logits/chosen": -0.9799394607543945,
+      "logits/rejected": -0.8590563535690308,
+      "logps/chosen": -4.606871604919434,
+      "logps/rejected": -5.045818328857422,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.606871604919434,
+      "rewards/margins": 0.43894606828689575,
+      "rewards/rejected": -5.045818328857422,
+      "sft_loss": 4.364339351654053,
+      "step": 3195
+    },
+    {
+      "epoch": 1.7126609800970063,
+      "grad_norm": 0.36960789827653123,
+      "learning_rate": 4.6343371112903777e-07,
+      "logits/chosen": -0.8044673204421997,
+      "logits/rejected": -0.6007072329521179,
+      "logps/chosen": -4.615143299102783,
+      "logps/rejected": -5.214327335357666,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.615143299102783,
+      "rewards/margins": 0.5991836786270142,
+      "rewards/rejected": -5.214327335357666,
+      "sft_loss": 4.31845760345459,
+      "step": 3200
+    },
+    {
+      "epoch": 1.7126609800970063,
+      "eval_logits/chosen": -0.29473957419395447,
+      "eval_logits/rejected": -0.2032015025615692,
+      "eval_logps/chosen": -4.496006011962891,
+      "eval_logps/rejected": -5.107307434082031,
+      "eval_loss": 0.050797030329704285,
+      "eval_rewards/accuracies": 0.6691394448280334,
+      "eval_rewards/chosen": -4.496006011962891,
+      "eval_rewards/margins": 0.6113012433052063,
+      "eval_rewards/rejected": -5.107307434082031,
+      "eval_runtime": 43.3125,
+      "eval_samples_per_second": 31.053,
+      "eval_sft_loss": 4.119299411773682,
+      "eval_steps_per_second": 7.781,
+      "step": 3200
+    },
+    {
+      "epoch": 1.7153370128784078,
+      "grad_norm": 0.44427092334138696,
+      "learning_rate": 4.618806587294291e-07,
+      "logits/chosen": -0.87445068359375,
+      "logits/rejected": -0.7708224058151245,
+      "logps/chosen": -4.504208564758301,
+      "logps/rejected": -5.129702568054199,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.504208564758301,
+      "rewards/margins": 0.6254942417144775,
+      "rewards/rejected": -5.129702568054199,
+      "sft_loss": 4.297804832458496,
+      "step": 3205
+    },
+    {
+      "epoch": 1.7180130456598093,
+      "grad_norm": 0.3724267286377576,
+      "learning_rate": 4.603279761638365e-07,
+      "logits/chosen": -0.8704094886779785,
+      "logits/rejected": -0.723287045955658,
+      "logps/chosen": -4.470217227935791,
+      "logps/rejected": -5.0563645362854,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.470217227935791,
+      "rewards/margins": 0.5861474871635437,
+      "rewards/rejected": -5.0563645362854,
+      "sft_loss": 4.2170891761779785,
+      "step": 3210
+    },
+    {
+      "epoch": 1.720689078441211,
+      "grad_norm": 0.41128865897944417,
+      "learning_rate": 4.5877567849639315e-07,
+      "logits/chosen": -0.8630671501159668,
+      "logits/rejected": -0.7631199955940247,
+      "logps/chosen": -4.550658226013184,
+      "logps/rejected": -5.094977378845215,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.550658226013184,
+      "rewards/margins": 0.5443195700645447,
+      "rewards/rejected": -5.094977378845215,
+      "sft_loss": 4.317914009094238,
+      "step": 3215
+    },
+    {
+      "epoch": 1.7233651112226125,
+      "grad_norm": 0.6568465025902127,
+      "learning_rate": 4.572237807874979e-07,
+      "logits/chosen": -1.0142710208892822,
+      "logits/rejected": -0.6812065243721008,
+      "logps/chosen": -4.546741008758545,
+      "logps/rejected": -5.181439399719238,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.546741008758545,
+      "rewards/margins": 0.6346983909606934,
+      "rewards/rejected": -5.181439399719238,
+      "sft_loss": 4.28690242767334,
+      "step": 3220
+    },
+    {
+      "epoch": 1.726041144004014,
+      "grad_norm": 0.43508632949046916,
+      "learning_rate": 4.5567229809366895e-07,
+      "logits/chosen": -0.8844995498657227,
+      "logits/rejected": -0.7484878301620483,
+      "logps/chosen": -4.440615653991699,
+      "logps/rejected": -4.99686336517334,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.440615653991699,
+      "rewards/margins": 0.5562475919723511,
+      "rewards/rejected": -4.99686336517334,
+      "sft_loss": 4.153700351715088,
+      "step": 3225
+    },
+    {
+      "epoch": 1.7287171767854157,
+      "grad_norm": 0.4122279782736568,
+      "learning_rate": 4.541212454673984e-07,
+      "logits/chosen": -0.9238206148147583,
+      "logits/rejected": -0.7181123495101929,
+      "logps/chosen": -4.5585713386535645,
+      "logps/rejected": -5.414360523223877,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.5585713386535645,
+      "rewards/margins": 0.8557893633842468,
+      "rewards/rejected": -5.414360523223877,
+      "sft_loss": 4.298130989074707,
+      "step": 3230
+    },
+    {
+      "epoch": 1.7313932095668172,
+      "grad_norm": 0.4026073166205477,
+      "learning_rate": 4.525706379570055e-07,
+      "logits/chosen": -0.9051357507705688,
+      "logits/rejected": -0.832781195640564,
+      "logps/chosen": -4.346682548522949,
+      "logps/rejected": -4.964540481567383,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.346682548522949,
+      "rewards/margins": 0.6178587675094604,
+      "rewards/rejected": -4.964540481567383,
+      "sft_loss": 4.1737565994262695,
+      "step": 3235
+    },
+    {
+      "epoch": 1.7340692423482187,
+      "grad_norm": 0.47424442428266617,
+      "learning_rate": 4.510204906064911e-07,
+      "logits/chosen": -0.8570457696914673,
+      "logits/rejected": -0.7040773630142212,
+      "logps/chosen": -4.4648237228393555,
+      "logps/rejected": -5.1131768226623535,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.4648237228393555,
+      "rewards/margins": 0.6483533978462219,
+      "rewards/rejected": -5.1131768226623535,
+      "sft_loss": 4.171679496765137,
+      "step": 3240
+    },
+    {
+      "epoch": 1.7367452751296204,
+      "grad_norm": 1.0429902624418217,
+      "learning_rate": 4.4947081845539177e-07,
+      "logits/chosen": -0.9616962671279907,
+      "logits/rejected": -0.8357839584350586,
+      "logps/chosen": -4.420161247253418,
+      "logps/rejected": -5.0600433349609375,
+      "loss": 0.052,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.420161247253418,
+      "rewards/margins": 0.6398815512657166,
+      "rewards/rejected": -5.0600433349609375,
+      "sft_loss": 4.109945297241211,
+      "step": 3245
+    },
+    {
+      "epoch": 1.739421307911022,
+      "grad_norm": 0.40018549193747066,
+      "learning_rate": 4.479216365386333e-07,
+      "logits/chosen": -0.772409200668335,
+      "logits/rejected": -0.5954066514968872,
+      "logps/chosen": -4.221619129180908,
+      "logps/rejected": -5.036656856536865,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.221619129180908,
+      "rewards/margins": 0.8150378465652466,
+      "rewards/rejected": -5.036656856536865,
+      "sft_loss": 3.9400768280029297,
+      "step": 3250
+    },
+    {
+      "epoch": 1.7420973406924234,
+      "grad_norm": 0.37909844451042285,
+      "learning_rate": 4.4637295988638555e-07,
+      "logits/chosen": -0.8761689066886902,
+      "logits/rejected": -0.8001937866210938,
+      "logps/chosen": -4.484442710876465,
+      "logps/rejected": -5.098120212554932,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.484442710876465,
+      "rewards/margins": 0.6136777400970459,
+      "rewards/rejected": -5.098120212554932,
+      "sft_loss": 4.218336582183838,
+      "step": 3255
+    },
+    {
+      "epoch": 1.744773373473825,
+      "grad_norm": 0.6816501770865704,
+      "learning_rate": 4.4482480352391623e-07,
+      "logits/chosen": -0.8988161087036133,
+      "logits/rejected": -0.7821734547615051,
+      "logps/chosen": -4.642391204833984,
+      "logps/rejected": -5.23095703125,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.642391204833984,
+      "rewards/margins": 0.5885659456253052,
+      "rewards/rejected": -5.23095703125,
+      "sft_loss": 4.404733657836914,
+      "step": 3260
+    },
+    {
+      "epoch": 1.7474494062552266,
+      "grad_norm": 0.5455705596230104,
+      "learning_rate": 4.4327718247144507e-07,
+      "logits/chosen": -0.7707226872444153,
+      "logits/rejected": -0.6102782487869263,
+      "logps/chosen": -4.860251426696777,
+      "logps/rejected": -5.384507656097412,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.860251426696777,
+      "rewards/margins": 0.5242565870285034,
+      "rewards/rejected": -5.384507656097412,
+      "sft_loss": 4.573927879333496,
+      "step": 3265
+    },
+    {
+      "epoch": 1.750125439036628,
+      "grad_norm": 0.5162517294345276,
+      "learning_rate": 4.417301117439984e-07,
+      "logits/chosen": -0.9343627095222473,
+      "logits/rejected": -0.7939955592155457,
+      "logps/chosen": -4.480029106140137,
+      "logps/rejected": -4.994220733642578,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.480029106140137,
+      "rewards/margins": 0.5141913294792175,
+      "rewards/rejected": -4.994220733642578,
+      "sft_loss": 4.212777137756348,
+      "step": 3270
+    },
+    {
+      "epoch": 1.7528014718180298,
+      "grad_norm": 0.4073109145029346,
+      "learning_rate": 4.401836063512631e-07,
+      "logits/chosen": -1.001089334487915,
+      "logits/rejected": -0.6555830240249634,
+      "logps/chosen": -4.451521873474121,
+      "logps/rejected": -5.125973224639893,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.451521873474121,
+      "rewards/margins": 0.6744511723518372,
+      "rewards/rejected": -5.125973224639893,
+      "sft_loss": 4.141744136810303,
+      "step": 3275
+    },
+    {
+      "epoch": 1.7554775045994313,
+      "grad_norm": 0.5772050447307934,
+      "learning_rate": 4.386376812974413e-07,
+      "logits/chosen": -0.9038535952568054,
+      "logits/rejected": -0.8890512585639954,
+      "logps/chosen": -4.574075698852539,
+      "logps/rejected": -5.134883880615234,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.574075698852539,
+      "rewards/margins": 0.5608078837394714,
+      "rewards/rejected": -5.134883880615234,
+      "sft_loss": 4.309540748596191,
+      "step": 3280
+    },
+    {
+      "epoch": 1.7581535373808328,
+      "grad_norm": 0.40686850263594554,
+      "learning_rate": 4.370923515811048e-07,
+      "logits/chosen": -0.9050912857055664,
+      "logits/rejected": -0.6649254560470581,
+      "logps/chosen": -4.3815155029296875,
+      "logps/rejected": -5.013126373291016,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.3815155029296875,
+      "rewards/margins": 0.6316103935241699,
+      "rewards/rejected": -5.013126373291016,
+      "sft_loss": 4.075606346130371,
+      "step": 3285
+    },
+    {
+      "epoch": 1.7608295701622345,
+      "grad_norm": 0.39012420613123405,
+      "learning_rate": 4.35547632195049e-07,
+      "logits/chosen": -0.8044376373291016,
+      "logits/rejected": -0.7201283574104309,
+      "logps/chosen": -4.696098327636719,
+      "logps/rejected": -5.314190864562988,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.696098327636719,
+      "rewards/margins": 0.6180926561355591,
+      "rewards/rejected": -5.314190864562988,
+      "sft_loss": 4.33831787109375,
+      "step": 3290
+    },
+    {
+      "epoch": 1.763505602943636,
+      "grad_norm": 0.7351191098993615,
+      "learning_rate": 4.340035381261484e-07,
+      "logits/chosen": -0.7771255373954773,
+      "logits/rejected": -0.7277175784111023,
+      "logps/chosen": -4.453921318054199,
+      "logps/rejected": -5.145596504211426,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.453921318054199,
+      "rewards/margins": 0.6916751861572266,
+      "rewards/rejected": -5.145596504211426,
+      "sft_loss": 4.129612922668457,
+      "step": 3295
+    },
+    {
+      "epoch": 1.7661816357250375,
+      "grad_norm": 0.35475089495116524,
+      "learning_rate": 4.324600843552104e-07,
+      "logits/chosen": -0.9506170153617859,
+      "logits/rejected": -0.8384958505630493,
+      "logps/chosen": -4.479653358459473,
+      "logps/rejected": -5.169857978820801,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.479653358459473,
+      "rewards/margins": 0.6902052760124207,
+      "rewards/rejected": -5.169857978820801,
+      "sft_loss": 4.17246150970459,
+      "step": 3300
+    },
+    {
+      "epoch": 1.7688576685064392,
+      "grad_norm": 0.3659908245134801,
+      "learning_rate": 4.309172858568302e-07,
+      "logits/chosen": -0.937427818775177,
+      "logits/rejected": -0.7483657598495483,
+      "logps/chosen": -4.533991813659668,
+      "logps/rejected": -5.1667327880859375,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.533991813659668,
+      "rewards/margins": 0.6327404975891113,
+      "rewards/rejected": -5.1667327880859375,
+      "sft_loss": 4.246024131774902,
+      "step": 3305
+    },
+    {
+      "epoch": 1.771533701287841,
+      "grad_norm": 0.6137087486247289,
+      "learning_rate": 4.293751575992455e-07,
+      "logits/chosen": -0.7637144327163696,
+      "logits/rejected": -0.7552968859672546,
+      "logps/chosen": -4.631504535675049,
+      "logps/rejected": -5.132235527038574,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.631504535675049,
+      "rewards/margins": 0.5007305145263672,
+      "rewards/rejected": -5.132235527038574,
+      "sft_loss": 4.390534400939941,
+      "step": 3310
+    },
+    {
+      "epoch": 1.7742097340692422,
+      "grad_norm": 0.4352540545382265,
+      "learning_rate": 4.278337145441916e-07,
+      "logits/chosen": -0.886570155620575,
+      "logits/rejected": -0.7136012315750122,
+      "logps/chosen": -4.416377067565918,
+      "logps/rejected": -5.063299179077148,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.416377067565918,
+      "rewards/margins": 0.6469219326972961,
+      "rewards/rejected": -5.063299179077148,
+      "sft_loss": 4.168293476104736,
+      "step": 3315
+    },
+    {
+      "epoch": 1.776885766850644,
+      "grad_norm": 0.3831451426766769,
+      "learning_rate": 4.262929716467556e-07,
+      "logits/chosen": -0.8285180926322937,
+      "logits/rejected": -0.5632731318473816,
+      "logps/chosen": -4.392007827758789,
+      "logps/rejected": -5.139756679534912,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.392007827758789,
+      "rewards/margins": 0.7477489709854126,
+      "rewards/rejected": -5.139756679534912,
+      "sft_loss": 4.160672187805176,
+      "step": 3320
+    },
+    {
+      "epoch": 1.7795617996320456,
+      "grad_norm": 0.39530178751687783,
+      "learning_rate": 4.247529438552321e-07,
+      "logits/chosen": -0.9192399978637695,
+      "logits/rejected": -0.7035531997680664,
+      "logps/chosen": -4.617929458618164,
+      "logps/rejected": -5.078191757202148,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.617929458618164,
+      "rewards/margins": 0.4602627754211426,
+      "rewards/rejected": -5.078191757202148,
+      "sft_loss": 4.380537986755371,
+      "step": 3325
+    },
+    {
+      "epoch": 1.782237832413447,
+      "grad_norm": 0.44253110919207433,
+      "learning_rate": 4.232136461109773e-07,
+      "logits/chosen": -0.8389490842819214,
+      "logits/rejected": -0.7441287040710449,
+      "logps/chosen": -4.518656253814697,
+      "logps/rejected": -5.131009101867676,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.518656253814697,
+      "rewards/margins": 0.6123533248901367,
+      "rewards/rejected": -5.131009101867676,
+      "sft_loss": 4.230744361877441,
+      "step": 3330
+    },
+    {
+      "epoch": 1.7849138651948486,
+      "grad_norm": 0.4149420144966067,
+      "learning_rate": 4.216750933482646e-07,
+      "logits/chosen": -0.8691641092300415,
+      "logits/rejected": -0.6906719207763672,
+      "logps/chosen": -4.620790958404541,
+      "logps/rejected": -5.059487342834473,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.620790958404541,
+      "rewards/margins": 0.4386964440345764,
+      "rewards/rejected": -5.059487342834473,
+      "sft_loss": 4.235035419464111,
+      "step": 3335
+    },
+    {
+      "epoch": 1.7875898979762503,
+      "grad_norm": 0.5139969806101036,
+      "learning_rate": 4.2013730049413986e-07,
+      "logits/chosen": -0.8421649932861328,
+      "logits/rejected": -0.6403535604476929,
+      "logps/chosen": -4.331358432769775,
+      "logps/rejected": -5.270960807800293,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -4.331358432769775,
+      "rewards/margins": 0.9396018981933594,
+      "rewards/rejected": -5.270960807800293,
+      "sft_loss": 4.097912788391113,
+      "step": 3340
+    },
+    {
+      "epoch": 1.7902659307576518,
+      "grad_norm": 0.30430095720378014,
+      "learning_rate": 4.1860028246827594e-07,
+      "logits/chosen": -0.8284981846809387,
+      "logits/rejected": -0.5819277167320251,
+      "logps/chosen": -4.4769392013549805,
+      "logps/rejected": -5.007475852966309,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.4769392013549805,
+      "rewards/margins": 0.5305370092391968,
+      "rewards/rejected": -5.007475852966309,
+      "sft_loss": 4.225216388702393,
+      "step": 3345
+    },
+    {
+      "epoch": 1.7929419635390533,
+      "grad_norm": 0.4921011154236414,
+      "learning_rate": 4.170640541828285e-07,
+      "logits/chosen": -0.9329277873039246,
+      "logits/rejected": -0.7737506031990051,
+      "logps/chosen": -4.421766757965088,
+      "logps/rejected": -5.002078056335449,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.421766757965088,
+      "rewards/margins": 0.5803118348121643,
+      "rewards/rejected": -5.002078056335449,
+      "sft_loss": 4.226648807525635,
+      "step": 3350
+    },
+    {
+      "epoch": 1.795617996320455,
+      "grad_norm": 0.43199463592891063,
+      "learning_rate": 4.1552863054229116e-07,
+      "logits/chosen": -0.7047543525695801,
+      "logits/rejected": -0.6609756350517273,
+      "logps/chosen": -4.531655788421631,
+      "logps/rejected": -5.140510559082031,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.531655788421631,
+      "rewards/margins": 0.6088550686836243,
+      "rewards/rejected": -5.140510559082031,
+      "sft_loss": 4.227361679077148,
+      "step": 3355
+    },
+    {
+      "epoch": 1.7982940291018565,
+      "grad_norm": 0.5426284178434481,
+      "learning_rate": 4.139940264433508e-07,
+      "logits/chosen": -0.7945530414581299,
+      "logits/rejected": -0.5765786170959473,
+      "logps/chosen": -4.41995906829834,
+      "logps/rejected": -5.090672016143799,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.41995906829834,
+      "rewards/margins": 0.6707130074501038,
+      "rewards/rejected": -5.090672016143799,
+      "sft_loss": 4.104195594787598,
+      "step": 3360
+    },
+    {
+      "epoch": 1.800970061883258,
+      "grad_norm": 0.3120787417194607,
+      "learning_rate": 4.1246025677474303e-07,
+      "logits/chosen": -0.8796011805534363,
+      "logits/rejected": -0.6404326558113098,
+      "logps/chosen": -4.570914268493652,
+      "logps/rejected": -5.127249240875244,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.570914268493652,
+      "rewards/margins": 0.5563352108001709,
+      "rewards/rejected": -5.127249240875244,
+      "sft_loss": 4.335757255554199,
+      "step": 3365
+    },
+    {
+      "epoch": 1.8036460946646597,
+      "grad_norm": 0.3718432668133724,
+      "learning_rate": 4.10927336417108e-07,
+      "logits/chosen": -0.8479071855545044,
+      "logits/rejected": -0.6178969740867615,
+      "logps/chosen": -4.612782955169678,
+      "logps/rejected": -5.014036655426025,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.612782955169678,
+      "rewards/margins": 0.40125328302383423,
+      "rewards/rejected": -5.014036655426025,
+      "sft_loss": 4.301538944244385,
+      "step": 3370
+    },
+    {
+      "epoch": 1.8063221274460612,
+      "grad_norm": 0.6878982267884615,
+      "learning_rate": 4.093952802428457e-07,
+      "logits/chosen": -0.6994894742965698,
+      "logits/rejected": -0.6803777813911438,
+      "logps/chosen": -4.756828308105469,
+      "logps/rejected": -5.290118217468262,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.756828308105469,
+      "rewards/margins": 0.5332905054092407,
+      "rewards/rejected": -5.290118217468262,
+      "sft_loss": 4.457432746887207,
+      "step": 3375
+    },
+    {
+      "epoch": 1.8089981602274627,
+      "grad_norm": 0.589099141329664,
+      "learning_rate": 4.0786410311597184e-07,
+      "logits/chosen": -0.9029039144515991,
+      "logits/rejected": -0.6868539452552795,
+      "logps/chosen": -4.576333522796631,
+      "logps/rejected": -5.159743309020996,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.576333522796631,
+      "rewards/margins": 0.5834100842475891,
+      "rewards/rejected": -5.159743309020996,
+      "sft_loss": 4.252318382263184,
+      "step": 3380
+    },
+    {
+      "epoch": 1.8116741930088645,
+      "grad_norm": 0.3999674824957761,
+      "learning_rate": 4.063338198919737e-07,
+      "logits/chosen": -0.8113029599189758,
+      "logits/rejected": -0.846416175365448,
+      "logps/chosen": -4.430078506469727,
+      "logps/rejected": -4.966437339782715,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.430078506469727,
+      "rewards/margins": 0.5363594889640808,
+      "rewards/rejected": -4.966437339782715,
+      "sft_loss": 4.174395561218262,
+      "step": 3385
+    },
+    {
+      "epoch": 1.814350225790266,
+      "grad_norm": 0.38273738849063044,
+      "learning_rate": 4.0480444541766575e-07,
+      "logits/chosen": -0.8961130380630493,
+      "logits/rejected": -0.7447247505187988,
+      "logps/chosen": -4.732843399047852,
+      "logps/rejected": -5.325850486755371,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.732843399047852,
+      "rewards/margins": 0.5930072665214539,
+      "rewards/rejected": -5.325850486755371,
+      "sft_loss": 4.414688587188721,
+      "step": 3390
+    },
+    {
+      "epoch": 1.8170262585716674,
+      "grad_norm": 0.5147586922678519,
+      "learning_rate": 4.0327599453104606e-07,
+      "logits/chosen": -0.9704347848892212,
+      "logits/rejected": -0.780985951423645,
+      "logps/chosen": -4.527245998382568,
+      "logps/rejected": -5.088501930236816,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.527245998382568,
+      "rewards/margins": 0.5612561106681824,
+      "rewards/rejected": -5.088501930236816,
+      "sft_loss": 4.192348480224609,
+      "step": 3395
+    },
+    {
+      "epoch": 1.8197022913530692,
+      "grad_norm": 0.5005840786352599,
+      "learning_rate": 4.017484820611514e-07,
+      "logits/chosen": -0.7550173997879028,
+      "logits/rejected": -0.6032567620277405,
+      "logps/chosen": -4.417336463928223,
+      "logps/rejected": -5.080150127410889,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.417336463928223,
+      "rewards/margins": 0.6628138422966003,
+      "rewards/rejected": -5.080150127410889,
+      "sft_loss": 4.079850673675537,
+      "step": 3400
+    },
+    {
+      "epoch": 1.8223783241344707,
+      "grad_norm": 0.8229748724775394,
+      "learning_rate": 4.002219228279148e-07,
+      "logits/chosen": -0.8199543952941895,
+      "logits/rejected": -0.6911274194717407,
+      "logps/chosen": -4.341639518737793,
+      "logps/rejected": -4.903729438781738,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.341639518737793,
+      "rewards/margins": 0.5620898008346558,
+      "rewards/rejected": -4.903729438781738,
+      "sft_loss": 4.065591812133789,
+      "step": 3405
+    },
+    {
+      "epoch": 1.8250543569158721,
+      "grad_norm": 0.4876509117776437,
+      "learning_rate": 3.9869633164202045e-07,
+      "logits/chosen": -0.8162205815315247,
+      "logits/rejected": -0.52099609375,
+      "logps/chosen": -4.461337089538574,
+      "logps/rejected": -5.199841022491455,
+      "loss": 0.051,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.461337089538574,
+      "rewards/margins": 0.7385039925575256,
+      "rewards/rejected": -5.199841022491455,
+      "sft_loss": 4.167636394500732,
+      "step": 3410
+    },
+    {
+      "epoch": 1.8277303896972739,
+      "grad_norm": 0.7065591387395672,
+      "learning_rate": 3.9717172330476077e-07,
+      "logits/chosen": -0.9044772982597351,
+      "logits/rejected": -0.7785995006561279,
+      "logps/chosen": -4.586330413818359,
+      "logps/rejected": -5.2429423332214355,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.586330413818359,
+      "rewards/margins": 0.6566125154495239,
+      "rewards/rejected": -5.2429423332214355,
+      "sft_loss": 4.347871780395508,
+      "step": 3415
+    },
+    {
+      "epoch": 1.8304064224786754,
+      "grad_norm": 0.5075784931382683,
+      "learning_rate": 3.956481126078927e-07,
+      "logits/chosen": -0.6835981607437134,
+      "logits/rejected": -0.5895021557807922,
+      "logps/chosen": -4.792338848114014,
+      "logps/rejected": -5.461629867553711,
+      "loss": 0.0543,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.792338848114014,
+      "rewards/margins": 0.6692904829978943,
+      "rewards/rejected": -5.461629867553711,
+      "sft_loss": 4.541243553161621,
+      "step": 3420
+    },
+    {
+      "epoch": 1.8330824552600768,
+      "grad_norm": 0.537368618423302,
+      "learning_rate": 3.941255143334937e-07,
+      "logits/chosen": -0.7706578969955444,
+      "logits/rejected": -0.7533372640609741,
+      "logps/chosen": -4.472137451171875,
+      "logps/rejected": -5.051485538482666,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.472137451171875,
+      "rewards/margins": 0.5793476700782776,
+      "rewards/rejected": -5.051485538482666,
+      "sft_loss": 4.173643589019775,
+      "step": 3425
+    },
+    {
+      "epoch": 1.8357584880414786,
+      "grad_norm": 0.35854577541397925,
+      "learning_rate": 3.9260394325381895e-07,
+      "logits/chosen": -0.7394391894340515,
+      "logits/rejected": -0.6220877766609192,
+      "logps/chosen": -4.258572578430176,
+      "logps/rejected": -5.175926685333252,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.258572578430176,
+      "rewards/margins": 0.9173545837402344,
+      "rewards/rejected": -5.175926685333252,
+      "sft_loss": 3.9583396911621094,
+      "step": 3430
+    },
+    {
+      "epoch": 1.83843452082288,
+      "grad_norm": 0.41343245811909185,
+      "learning_rate": 3.9108341413115784e-07,
+      "logits/chosen": -0.7238286733627319,
+      "logits/rejected": -0.6091488003730774,
+      "logps/chosen": -4.33715295791626,
+      "logps/rejected": -4.925466060638428,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.33715295791626,
+      "rewards/margins": 0.5883134603500366,
+      "rewards/rejected": -4.925466060638428,
+      "sft_loss": 3.97629976272583,
+      "step": 3435
+    },
+    {
+      "epoch": 1.8411105536042816,
+      "grad_norm": 0.41918254444927916,
+      "learning_rate": 3.895639417176905e-07,
+      "logits/chosen": -0.8348200917243958,
+      "logits/rejected": -0.707625687122345,
+      "logps/chosen": -4.645773887634277,
+      "logps/rejected": -5.314620018005371,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.645773887634277,
+      "rewards/margins": 0.668846607208252,
+      "rewards/rejected": -5.314620018005371,
+      "sft_loss": 4.392010688781738,
+      "step": 3440
+    },
+    {
+      "epoch": 1.8437865863856833,
+      "grad_norm": 0.42246506471392314,
+      "learning_rate": 3.8804554075534497e-07,
+      "logits/chosen": -0.8349407315254211,
+      "logits/rejected": -0.5855430960655212,
+      "logps/chosen": -4.490998268127441,
+      "logps/rejected": -5.2729082107543945,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.490998268127441,
+      "rewards/margins": 0.7819093465805054,
+      "rewards/rejected": -5.2729082107543945,
+      "sft_loss": 4.270726203918457,
+      "step": 3445
+    },
+    {
+      "epoch": 1.8464626191670848,
+      "grad_norm": 0.6390590052604445,
+      "learning_rate": 3.8652822597565403e-07,
+      "logits/chosen": -0.8645042181015015,
+      "logits/rejected": -0.6598786115646362,
+      "logps/chosen": -4.601441383361816,
+      "logps/rejected": -5.257026672363281,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.601441383361816,
+      "rewards/margins": 0.6555854678153992,
+      "rewards/rejected": -5.257026672363281,
+      "sft_loss": 4.356138229370117,
+      "step": 3450
+    },
+    {
+      "epoch": 1.8491386519484863,
+      "grad_norm": 0.39388686843240567,
+      "learning_rate": 3.850120120996123e-07,
+      "logits/chosen": -0.8234783411026001,
+      "logits/rejected": -0.5680242776870728,
+      "logps/chosen": -4.540238857269287,
+      "logps/rejected": -5.173902988433838,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.540238857269287,
+      "rewards/margins": 0.633664071559906,
+      "rewards/rejected": -5.173902988433838,
+      "sft_loss": 4.335484504699707,
+      "step": 3455
+    },
+    {
+      "epoch": 1.851814684729888,
+      "grad_norm": 0.4499215399446686,
+      "learning_rate": 3.8349691383753356e-07,
+      "logits/chosen": -0.6719542145729065,
+      "logits/rejected": -0.5450443029403687,
+      "logps/chosen": -4.431046962738037,
+      "logps/rejected": -5.082084655761719,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.431046962738037,
+      "rewards/margins": 0.6510375142097473,
+      "rewards/rejected": -5.082084655761719,
+      "sft_loss": 4.082747459411621,
+      "step": 3460
+    },
+    {
+      "epoch": 1.8544907175112895,
+      "grad_norm": 0.3778904426429339,
+      "learning_rate": 3.819829458889078e-07,
+      "logits/chosen": -0.7996759414672852,
+      "logits/rejected": -0.7100083231925964,
+      "logps/chosen": -4.312413215637207,
+      "logps/rejected": -4.77877950668335,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.312413215637207,
+      "rewards/margins": 0.4663669466972351,
+      "rewards/rejected": -4.77877950668335,
+      "sft_loss": 4.042895317077637,
+      "step": 3465
+    },
+    {
+      "epoch": 1.857166750292691,
+      "grad_norm": 0.5604460188611351,
+      "learning_rate": 3.804701229422585e-07,
+      "logits/chosen": -0.8722193837165833,
+      "logits/rejected": -0.8069941401481628,
+      "logps/chosen": -4.671416282653809,
+      "logps/rejected": -5.111981391906738,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.671416282653809,
+      "rewards/margins": 0.4405645728111267,
+      "rewards/rejected": -5.111981391906738,
+      "sft_loss": 4.324137210845947,
+      "step": 3470
+    },
+    {
+      "epoch": 1.8598427830740927,
+      "grad_norm": 0.46389515283591515,
+      "learning_rate": 3.789584596750007e-07,
+      "logits/chosen": -0.8924866914749146,
+      "logits/rejected": -0.86207515001297,
+      "logps/chosen": -4.578545570373535,
+      "logps/rejected": -5.2069993019104,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.578545570373535,
+      "rewards/margins": 0.62845379114151,
+      "rewards/rejected": -5.2069993019104,
+      "sft_loss": 4.352547645568848,
+      "step": 3475
+    },
+    {
+      "epoch": 1.8625188158554944,
+      "grad_norm": 0.3101780013628597,
+      "learning_rate": 3.77447970753298e-07,
+      "logits/chosen": -0.8054901957511902,
+      "logits/rejected": -0.8184728622436523,
+      "logps/chosen": -4.611974716186523,
+      "logps/rejected": -5.077882766723633,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.611974716186523,
+      "rewards/margins": 0.4659079909324646,
+      "rewards/rejected": -5.077882766723633,
+      "sft_loss": 4.400869369506836,
+      "step": 3480
+    },
+    {
+      "epoch": 1.8651948486368957,
+      "grad_norm": 0.4254690093635414,
+      "learning_rate": 3.7593867083192057e-07,
+      "logits/chosen": -0.8784816861152649,
+      "logits/rejected": -0.7168563604354858,
+      "logps/chosen": -4.520941257476807,
+      "logps/rejected": -5.078371524810791,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.520941257476807,
+      "rewards/margins": 0.5574301481246948,
+      "rewards/rejected": -5.078371524810791,
+      "sft_loss": 4.297972679138184,
+      "step": 3485
+    },
+    {
+      "epoch": 1.8678708814182974,
+      "grad_norm": 0.48612755028753435,
+      "learning_rate": 3.7443057455410276e-07,
+      "logits/chosen": -0.7678418159484863,
+      "logits/rejected": -0.6967147588729858,
+      "logps/chosen": -4.190707206726074,
+      "logps/rejected": -5.020942211151123,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.190707206726074,
+      "rewards/margins": 0.8302351832389832,
+      "rewards/rejected": -5.020942211151123,
+      "sft_loss": 4.027964115142822,
+      "step": 3490
+    },
+    {
+      "epoch": 1.870546914199699,
+      "grad_norm": 0.357070124593642,
+      "learning_rate": 3.7292369655140145e-07,
+      "logits/chosen": -1.0313518047332764,
+      "logits/rejected": -0.8303739428520203,
+      "logps/chosen": -4.52823543548584,
+      "logps/rejected": -5.011662006378174,
+      "loss": 0.054,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.52823543548584,
+      "rewards/margins": 0.4834265112876892,
+      "rewards/rejected": -5.011662006378174,
+      "sft_loss": 4.35888147354126,
+      "step": 3495
+    },
+    {
+      "epoch": 1.8732229469811004,
+      "grad_norm": 0.35556182344734116,
+      "learning_rate": 3.714180514435534e-07,
+      "logits/chosen": -0.8014265298843384,
+      "logits/rejected": -0.6077925562858582,
+      "logps/chosen": -4.6149725914001465,
+      "logps/rejected": -5.177509307861328,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.6149725914001465,
+      "rewards/margins": 0.562536895275116,
+      "rewards/rejected": -5.177509307861328,
+      "sft_loss": 4.300775527954102,
+      "step": 3500
+    },
+    {
+      "epoch": 1.875898979762502,
+      "grad_norm": 0.3562554445522406,
+      "learning_rate": 3.6991365383833426e-07,
+      "logits/chosen": -0.9311081171035767,
+      "logits/rejected": -0.7546035647392273,
+      "logps/chosen": -4.536148548126221,
+      "logps/rejected": -5.134714126586914,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.536148548126221,
+      "rewards/margins": 0.5985656976699829,
+      "rewards/rejected": -5.134714126586914,
+      "sft_loss": 4.202921390533447,
+      "step": 3505
+    },
+    {
+      "epoch": 1.8785750125439038,
+      "grad_norm": 0.4070594382170017,
+      "learning_rate": 3.684105183314162e-07,
+      "logits/chosen": -0.9334009289741516,
+      "logits/rejected": -0.8775346875190735,
+      "logps/chosen": -4.4893903732299805,
+      "logps/rejected": -5.005202293395996,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.4893903732299805,
+      "rewards/margins": 0.5158116221427917,
+      "rewards/rejected": -5.005202293395996,
+      "sft_loss": 4.235560894012451,
+      "step": 3510
+    },
+    {
+      "epoch": 1.881251045325305,
+      "grad_norm": 0.42486172086349194,
+      "learning_rate": 3.669086595062263e-07,
+      "logits/chosen": -0.8683802485466003,
+      "logits/rejected": -0.6446768641471863,
+      "logps/chosen": -4.424868583679199,
+      "logps/rejected": -5.10919713973999,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.424868583679199,
+      "rewards/margins": 0.6843288540840149,
+      "rewards/rejected": -5.10919713973999,
+      "sft_loss": 4.203177452087402,
+      "step": 3515
+    },
+    {
+      "epoch": 1.8839270781067068,
+      "grad_norm": 0.3631473467602663,
+      "learning_rate": 3.654080919338056e-07,
+      "logits/chosen": -0.9535540342330933,
+      "logits/rejected": -0.7772972583770752,
+      "logps/chosen": -4.488328456878662,
+      "logps/rejected": -5.187651634216309,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.488328456878662,
+      "rewards/margins": 0.6993231773376465,
+      "rewards/rejected": -5.187651634216309,
+      "sft_loss": 4.2527360916137695,
+      "step": 3520
+    },
+    {
+      "epoch": 1.8866031108881085,
+      "grad_norm": 1.108956434138034,
+      "learning_rate": 3.639088301726673e-07,
+      "logits/chosen": -0.8254637718200684,
+      "logits/rejected": -0.6029265522956848,
+      "logps/chosen": -4.488307476043701,
+      "logps/rejected": -5.122195720672607,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.488307476043701,
+      "rewards/margins": 0.633887529373169,
+      "rewards/rejected": -5.122195720672607,
+      "sft_loss": 4.214524745941162,
+      "step": 3525
+    },
+    {
+      "epoch": 1.88927914366951,
+      "grad_norm": 0.297083487299301,
+      "learning_rate": 3.624108887686556e-07,
+      "logits/chosen": -0.8513669967651367,
+      "logits/rejected": -0.8039869070053101,
+      "logps/chosen": -4.6177592277526855,
+      "logps/rejected": -5.003042697906494,
+      "loss": 0.053,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.6177592277526855,
+      "rewards/margins": 0.38528352975845337,
+      "rewards/rejected": -5.003042697906494,
+      "sft_loss": 4.321106910705566,
+      "step": 3530
+    },
+    {
+      "epoch": 1.8919551764509115,
+      "grad_norm": 0.3894391188758439,
+      "learning_rate": 3.6091428225480433e-07,
+      "logits/chosen": -1.000799536705017,
+      "logits/rejected": -0.8797224164009094,
+      "logps/chosen": -4.493613243103027,
+      "logps/rejected": -5.126000881195068,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.493613243103027,
+      "rewards/margins": 0.632387638092041,
+      "rewards/rejected": -5.126000881195068,
+      "sft_loss": 4.193148612976074,
+      "step": 3535
+    },
+    {
+      "epoch": 1.8946312092323132,
+      "grad_norm": 0.38449769948061835,
+      "learning_rate": 3.5941902515119674e-07,
+      "logits/chosen": -1.034938097000122,
+      "logits/rejected": -0.7592099905014038,
+      "logps/chosen": -4.570082664489746,
+      "logps/rejected": -5.047337532043457,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.570082664489746,
+      "rewards/margins": 0.47725504636764526,
+      "rewards/rejected": -5.047337532043457,
+      "sft_loss": 4.295851707458496,
+      "step": 3540
+    },
+    {
+      "epoch": 1.8973072420137147,
+      "grad_norm": 0.5234235514253388,
+      "learning_rate": 3.5792513196482373e-07,
+      "logits/chosen": -1.1635518074035645,
+      "logits/rejected": -0.8318096399307251,
+      "logps/chosen": -4.4717559814453125,
+      "logps/rejected": -5.075552463531494,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.4717559814453125,
+      "rewards/margins": 0.6037967205047607,
+      "rewards/rejected": -5.075552463531494,
+      "sft_loss": 4.137124538421631,
+      "step": 3545
+    },
+    {
+      "epoch": 1.8999832747951162,
+      "grad_norm": 0.33314820472642775,
+      "learning_rate": 3.5643261718944346e-07,
+      "logits/chosen": -0.8819448351860046,
+      "logits/rejected": -0.7907723188400269,
+      "logps/chosen": -4.5302557945251465,
+      "logps/rejected": -4.951289176940918,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -4.5302557945251465,
+      "rewards/margins": 0.4210330843925476,
+      "rewards/rejected": -4.951289176940918,
+      "sft_loss": 4.2429728507995605,
+      "step": 3550
+    },
+    {
+      "epoch": 1.902659307576518,
+      "grad_norm": 0.5263923862202771,
+      "learning_rate": 3.5494149530544087e-07,
+      "logits/chosen": -1.0538041591644287,
+      "logits/rejected": -0.9239507913589478,
+      "logps/chosen": -4.604650497436523,
+      "logps/rejected": -5.217805862426758,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.604650497436523,
+      "rewards/margins": 0.6131556034088135,
+      "rewards/rejected": -5.217805862426758,
+      "sft_loss": 4.297959327697754,
+      "step": 3555
+    },
+    {
+      "epoch": 1.9053353403579194,
+      "grad_norm": 0.5020939907325735,
+      "learning_rate": 3.534517807796871e-07,
+      "logits/chosen": -0.9725322723388672,
+      "logits/rejected": -0.8672218322753906,
+      "logps/chosen": -4.604799747467041,
+      "logps/rejected": -5.175978183746338,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.604799747467041,
+      "rewards/margins": 0.571178674697876,
+      "rewards/rejected": -5.175978183746338,
+      "sft_loss": 4.33532190322876,
+      "step": 3560
+    },
+    {
+      "epoch": 1.908011373139321,
+      "grad_norm": 0.43958453397135727,
+      "learning_rate": 3.519634880653988e-07,
+      "logits/chosen": -0.841769814491272,
+      "logits/rejected": -0.7862639427185059,
+      "logps/chosen": -4.502523422241211,
+      "logps/rejected": -5.1410603523254395,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.502523422241211,
+      "rewards/margins": 0.6385367512702942,
+      "rewards/rejected": -5.1410603523254395,
+      "sft_loss": 4.152388572692871,
+      "step": 3565
+    },
+    {
+      "epoch": 1.9106874059207226,
+      "grad_norm": 0.4603995201061191,
+      "learning_rate": 3.504766316019987e-07,
+      "logits/chosen": -0.8893829584121704,
+      "logits/rejected": -0.6943017840385437,
+      "logps/chosen": -4.210341453552246,
+      "logps/rejected": -4.964727401733398,
+      "loss": 0.0497,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.210341453552246,
+      "rewards/margins": 0.7543860673904419,
+      "rewards/rejected": -4.964727401733398,
+      "sft_loss": 3.8542957305908203,
+      "step": 3570
+    },
+    {
+      "epoch": 1.913363438702124,
+      "grad_norm": 0.4166426736441857,
+      "learning_rate": 3.489912258149745e-07,
+      "logits/chosen": -0.7692733407020569,
+      "logits/rejected": -0.6670472025871277,
+      "logps/chosen": -4.290536403656006,
+      "logps/rejected": -5.004633903503418,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.290536403656006,
+      "rewards/margins": 0.7140974998474121,
+      "rewards/rejected": -5.004633903503418,
+      "sft_loss": 3.9590721130371094,
+      "step": 3575
+    },
+    {
+      "epoch": 1.9160394714835256,
+      "grad_norm": 0.4202601946525645,
+      "learning_rate": 3.475072851157397e-07,
+      "logits/chosen": -0.8163237571716309,
+      "logits/rejected": -0.7968477010726929,
+      "logps/chosen": -4.64694881439209,
+      "logps/rejected": -5.253317356109619,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.64694881439209,
+      "rewards/margins": 0.6063681840896606,
+      "rewards/rejected": -5.253317356109619,
+      "sft_loss": 4.27789306640625,
+      "step": 3580
+    },
+    {
+      "epoch": 1.9187155042649273,
+      "grad_norm": 0.48439923094521753,
+      "learning_rate": 3.460248239014936e-07,
+      "logits/chosen": -0.7363717555999756,
+      "logits/rejected": -0.7516336441040039,
+      "logps/chosen": -4.712521553039551,
+      "logps/rejected": -5.380669593811035,
+      "loss": 0.052,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.712521553039551,
+      "rewards/margins": 0.6681480407714844,
+      "rewards/rejected": -5.380669593811035,
+      "sft_loss": 4.458998203277588,
+      "step": 3585
+    },
+    {
+      "epoch": 1.9213915370463288,
+      "grad_norm": 1.2607721506545888,
+      "learning_rate": 3.4454385655508134e-07,
+      "logits/chosen": -0.7563374042510986,
+      "logits/rejected": -0.7634093165397644,
+      "logps/chosen": -4.5644426345825195,
+      "logps/rejected": -5.0168280601501465,
+      "loss": 0.0544,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -4.5644426345825195,
+      "rewards/margins": 0.45238548517227173,
+      "rewards/rejected": -5.0168280601501465,
+      "sft_loss": 4.332982540130615,
+      "step": 3590
+    },
+    {
+      "epoch": 1.9240675698277303,
+      "grad_norm": 0.5141778910520323,
+      "learning_rate": 3.4306439744485447e-07,
+      "logits/chosen": -0.9014706611633301,
+      "logits/rejected": -0.6987265348434448,
+      "logps/chosen": -4.463529586791992,
+      "logps/rejected": -5.1967034339904785,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.463529586791992,
+      "rewards/margins": 0.7331734895706177,
+      "rewards/rejected": -5.1967034339904785,
+      "sft_loss": 4.167792320251465,
+      "step": 3595
+    },
+    {
+      "epoch": 1.926743602609132,
+      "grad_norm": 0.7170772833367152,
+      "learning_rate": 3.415864609245322e-07,
+      "logits/chosen": -0.8435789942741394,
+      "logits/rejected": -0.6109930276870728,
+      "logps/chosen": -4.614388465881348,
+      "logps/rejected": -5.201871871948242,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -4.614388465881348,
+      "rewards/margins": 0.5874830484390259,
+      "rewards/rejected": -5.201871871948242,
+      "sft_loss": 4.346115589141846,
+      "step": 3600
+    },
+    {
+      "epoch": 1.926743602609132,
+      "eval_logits/chosen": -0.32916495203971863,
+      "eval_logits/rejected": -0.2475946843624115,
+      "eval_logps/chosen": -4.419311046600342,
+      "eval_logps/rejected": -5.063793659210205,
+      "eval_loss": 0.050544675439596176,
+      "eval_rewards/accuracies": 0.6847180724143982,
+      "eval_rewards/chosen": -4.419311046600342,
+      "eval_rewards/margins": 0.6444829702377319,
+      "eval_rewards/rejected": -5.063793659210205,
+      "eval_runtime": 43.3773,
+      "eval_samples_per_second": 31.007,
+      "eval_sft_loss": 4.043381214141846,
+      "eval_steps_per_second": 7.769,
+      "step": 3600
+    },
+    {
+      "epoch": 1.9294196353905335,
+      "grad_norm": 0.4483136535653781,
+      "learning_rate": 3.401100613330605e-07,
+      "logits/chosen": -0.9276592135429382,
+      "logits/rejected": -0.9318366050720215,
+      "logps/chosen": -4.496493339538574,
+      "logps/rejected": -4.951340675354004,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.496493339538574,
+      "rewards/margins": 0.45484742522239685,
+      "rewards/rejected": -4.951340675354004,
+      "sft_loss": 4.23056173324585,
+      "step": 3605
+    },
+    {
+      "epoch": 1.932095668171935,
+      "grad_norm": 0.31593869243204614,
+      "learning_rate": 3.3863521299447514e-07,
+      "logits/chosen": -0.9971421360969543,
+      "logits/rejected": -0.8217166662216187,
+      "logps/chosen": -4.537590980529785,
+      "logps/rejected": -5.150124549865723,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.537590980529785,
+      "rewards/margins": 0.6125333905220032,
+      "rewards/rejected": -5.150124549865723,
+      "sft_loss": 4.283602714538574,
+      "step": 3610
+    },
+    {
+      "epoch": 1.9347717009533367,
+      "grad_norm": 0.40521049552058575,
+      "learning_rate": 3.371619302177609e-07,
+      "logits/chosen": -0.8417550921440125,
+      "logits/rejected": -0.700288712978363,
+      "logps/chosen": -4.353780269622803,
+      "logps/rejected": -4.941786766052246,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.353780269622803,
+      "rewards/margins": 0.5880061388015747,
+      "rewards/rejected": -4.941786766052246,
+      "sft_loss": 4.163086891174316,
+      "step": 3615
+    },
+    {
+      "epoch": 1.9374477337347382,
+      "grad_norm": 0.5389906830610456,
+      "learning_rate": 3.3569022729671393e-07,
+      "logits/chosen": -0.9811625480651855,
+      "logits/rejected": -0.8910226821899414,
+      "logps/chosen": -4.600131511688232,
+      "logps/rejected": -5.100063800811768,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.600131511688232,
+      "rewards/margins": 0.4999319016933441,
+      "rewards/rejected": -5.100063800811768,
+      "sft_loss": 4.426603317260742,
+      "step": 3620
+    },
+    {
+      "epoch": 1.9401237665161397,
+      "grad_norm": 0.4394386395821723,
+      "learning_rate": 3.342201185098024e-07,
+      "logits/chosen": -0.8498247265815735,
+      "logits/rejected": -0.9369813203811646,
+      "logps/chosen": -4.571401119232178,
+      "logps/rejected": -4.972775936126709,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.571401119232178,
+      "rewards/margins": 0.4013746380805969,
+      "rewards/rejected": -4.972775936126709,
+      "sft_loss": 4.325285911560059,
+      "step": 3625
+    },
+    {
+      "epoch": 1.9427997992975414,
+      "grad_norm": 0.38906540416917657,
+      "learning_rate": 3.3275161812002807e-07,
+      "logits/chosen": -0.9747118949890137,
+      "logits/rejected": -0.9587947726249695,
+      "logps/chosen": -4.523557662963867,
+      "logps/rejected": -5.116204261779785,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.523557662963867,
+      "rewards/margins": 0.5926466584205627,
+      "rewards/rejected": -5.116204261779785,
+      "sft_loss": 4.328993797302246,
+      "step": 3630
+    },
+    {
+      "epoch": 1.945475832078943,
+      "grad_norm": 0.37734258771151635,
+      "learning_rate": 3.312847403747883e-07,
+      "logits/chosen": -1.0632799863815308,
+      "logits/rejected": -0.9446859359741211,
+      "logps/chosen": -4.447041988372803,
+      "logps/rejected": -5.011303424835205,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.447041988372803,
+      "rewards/margins": 0.5642611384391785,
+      "rewards/rejected": -5.011303424835205,
+      "sft_loss": 4.248290061950684,
+      "step": 3635
+    },
+    {
+      "epoch": 1.9481518648603444,
+      "grad_norm": 0.3448461543727165,
+      "learning_rate": 3.2981949950573733e-07,
+      "logits/chosen": -0.950855553150177,
+      "logits/rejected": -0.9059017300605774,
+      "logps/chosen": -4.567697048187256,
+      "logps/rejected": -4.874037742614746,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.567697048187256,
+      "rewards/margins": 0.3063400685787201,
+      "rewards/rejected": -4.874037742614746,
+      "sft_loss": 4.307459831237793,
+      "step": 3640
+    },
+    {
+      "epoch": 1.9508278976417461,
+      "grad_norm": 0.3138553731803001,
+      "learning_rate": 3.283559097286486e-07,
+      "logits/chosen": -0.9531084895133972,
+      "logits/rejected": -0.8209444284439087,
+      "logps/chosen": -4.5451555252075195,
+      "logps/rejected": -5.012529373168945,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.5451555252075195,
+      "rewards/margins": 0.4673749506473541,
+      "rewards/rejected": -5.012529373168945,
+      "sft_loss": 4.314141273498535,
+      "step": 3645
+    },
+    {
+      "epoch": 1.9535039304231478,
+      "grad_norm": 0.4786368381669681,
+      "learning_rate": 3.268939852432765e-07,
+      "logits/chosen": -1.0215755701065063,
+      "logits/rejected": -0.9069620370864868,
+      "logps/chosen": -4.601208686828613,
+      "logps/rejected": -4.959532737731934,
+      "loss": 0.054,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.601208686828613,
+      "rewards/margins": 0.3583240211009979,
+      "rewards/rejected": -4.959532737731934,
+      "sft_loss": 4.377420425415039,
+      "step": 3650
+    },
+    {
+      "epoch": 1.9561799632045491,
+      "grad_norm": 0.560950653630944,
+      "learning_rate": 3.254337402332187e-07,
+      "logits/chosen": -0.8691121935844421,
+      "logits/rejected": -0.7701493501663208,
+      "logps/chosen": -4.460612773895264,
+      "logps/rejected": -5.0053582191467285,
+      "loss": 0.052,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.460612773895264,
+      "rewards/margins": 0.5447447299957275,
+      "rewards/rejected": -5.0053582191467285,
+      "sft_loss": 4.138766288757324,
+      "step": 3655
+    },
+    {
+      "epoch": 1.9588559959859508,
+      "grad_norm": 0.7040297008633517,
+      "learning_rate": 3.239751888657788e-07,
+      "logits/chosen": -0.9016326665878296,
+      "logits/rejected": -0.7556005120277405,
+      "logps/chosen": -4.512946605682373,
+      "logps/rejected": -5.122220993041992,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.512946605682373,
+      "rewards/margins": 0.6092746257781982,
+      "rewards/rejected": -5.122220993041992,
+      "sft_loss": 4.221314907073975,
+      "step": 3660
+    },
+    {
+      "epoch": 1.9615320287673526,
+      "grad_norm": 0.4938417120856117,
+      "learning_rate": 3.2251834529182856e-07,
+      "logits/chosen": -0.9205803871154785,
+      "logits/rejected": -0.8506709337234497,
+      "logps/chosen": -4.682583332061768,
+      "logps/rejected": -5.245231628417969,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.682583332061768,
+      "rewards/margins": 0.5626480579376221,
+      "rewards/rejected": -5.245231628417969,
+      "sft_loss": 4.3715009689331055,
+      "step": 3665
+    },
+    {
+      "epoch": 1.9642080615487538,
+      "grad_norm": 0.6781558880822852,
+      "learning_rate": 3.2106322364567075e-07,
+      "logits/chosen": -0.9177119135856628,
+      "logits/rejected": -0.7638968825340271,
+      "logps/chosen": -4.520151138305664,
+      "logps/rejected": -5.164031028747559,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.520151138305664,
+      "rewards/margins": 0.6438802480697632,
+      "rewards/rejected": -5.164031028747559,
+      "sft_loss": 4.398355484008789,
+      "step": 3670
+    },
+    {
+      "epoch": 1.9668840943301555,
+      "grad_norm": 0.43076111918754245,
+      "learning_rate": 3.1960983804490183e-07,
+      "logits/chosen": -0.8420068621635437,
+      "logits/rejected": -0.6891841888427734,
+      "logps/chosen": -4.482913017272949,
+      "logps/rejected": -5.048392295837402,
+      "loss": 0.053,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.482913017272949,
+      "rewards/margins": 0.5654793977737427,
+      "rewards/rejected": -5.048392295837402,
+      "sft_loss": 4.185931205749512,
+      "step": 3675
+    },
+    {
+      "epoch": 1.9695601271115573,
+      "grad_norm": 0.47109133227895944,
+      "learning_rate": 3.1815820259027537e-07,
+      "logits/chosen": -0.8213092088699341,
+      "logits/rejected": -0.7251837849617004,
+      "logps/chosen": -4.373992919921875,
+      "logps/rejected": -4.963822841644287,
+      "loss": 0.051,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.373992919921875,
+      "rewards/margins": 0.5898297429084778,
+      "rewards/rejected": -4.963822841644287,
+      "sft_loss": 4.038293838500977,
+      "step": 3680
+    },
+    {
+      "epoch": 1.9722361598929585,
+      "grad_norm": 0.4587038973394695,
+      "learning_rate": 3.16708331365565e-07,
+      "logits/chosen": -0.8804284334182739,
+      "logits/rejected": -0.7934475541114807,
+      "logps/chosen": -4.585146903991699,
+      "logps/rejected": -5.182665824890137,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.585146903991699,
+      "rewards/margins": 0.5975189805030823,
+      "rewards/rejected": -5.182665824890137,
+      "sft_loss": 4.350895881652832,
+      "step": 3685
+    },
+    {
+      "epoch": 1.9749121926743602,
+      "grad_norm": 0.4353653760418129,
+      "learning_rate": 3.152602384374275e-07,
+      "logits/chosen": -0.8628435134887695,
+      "logits/rejected": -0.6308306455612183,
+      "logps/chosen": -4.509026527404785,
+      "logps/rejected": -5.229661464691162,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.509026527404785,
+      "rewards/margins": 0.720634937286377,
+      "rewards/rejected": -5.229661464691162,
+      "sft_loss": 4.181239128112793,
+      "step": 3690
+    },
+    {
+      "epoch": 1.977588225455762,
+      "grad_norm": 0.4775253211912713,
+      "learning_rate": 3.1381393785526697e-07,
+      "logits/chosen": -0.8128175735473633,
+      "logits/rejected": -0.7428448796272278,
+      "logps/chosen": -4.55011510848999,
+      "logps/rejected": -5.149205207824707,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.55011510848999,
+      "rewards/margins": 0.5990898013114929,
+      "rewards/rejected": -5.149205207824707,
+      "sft_loss": 4.240631103515625,
+      "step": 3695
+    },
+    {
+      "epoch": 1.9802642582371635,
+      "grad_norm": 0.4083359155614466,
+      "learning_rate": 3.123694436510979e-07,
+      "logits/chosen": -0.7938219308853149,
+      "logits/rejected": -0.6490747928619385,
+      "logps/chosen": -4.4928998947143555,
+      "logps/rejected": -5.018836498260498,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.4928998947143555,
+      "rewards/margins": 0.5259365439414978,
+      "rewards/rejected": -5.018836498260498,
+      "sft_loss": 4.249953269958496,
+      "step": 3700
+    },
+    {
+      "epoch": 1.982940291018565,
+      "grad_norm": 0.4099432277759763,
+      "learning_rate": 3.1092676983940946e-07,
+      "logits/chosen": -0.8014167547225952,
+      "logits/rejected": -0.7310387492179871,
+      "logps/chosen": -4.47890567779541,
+      "logps/rejected": -5.038219928741455,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.47890567779541,
+      "rewards/margins": 0.5593137145042419,
+      "rewards/rejected": -5.038219928741455,
+      "sft_loss": 4.150893211364746,
+      "step": 3705
+    },
+    {
+      "epoch": 1.9856163237999667,
+      "grad_norm": 0.3350153632175618,
+      "learning_rate": 3.094859304170293e-07,
+      "logits/chosen": -0.6266270875930786,
+      "logits/rejected": -0.6359224915504456,
+      "logps/chosen": -4.500874042510986,
+      "logps/rejected": -5.046133995056152,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.500874042510986,
+      "rewards/margins": 0.5452605485916138,
+      "rewards/rejected": -5.046133995056152,
+      "sft_loss": 4.219162940979004,
+      "step": 3710
+    },
+    {
+      "epoch": 1.9882923565813682,
+      "grad_norm": 0.3536268621775762,
+      "learning_rate": 3.0804693936298795e-07,
+      "logits/chosen": -0.8250945210456848,
+      "logits/rejected": -0.7901517152786255,
+      "logps/chosen": -4.515018463134766,
+      "logps/rejected": -5.176741600036621,
+      "loss": 0.052,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.515018463134766,
+      "rewards/margins": 0.6617237329483032,
+      "rewards/rejected": -5.176741600036621,
+      "sft_loss": 4.3591413497924805,
+      "step": 3715
+    },
+    {
+      "epoch": 1.9909683893627697,
+      "grad_norm": 0.40375392263119575,
+      "learning_rate": 3.066098106383826e-07,
+      "logits/chosen": -0.8308378458023071,
+      "logits/rejected": -0.7185466289520264,
+      "logps/chosen": -4.537173271179199,
+      "logps/rejected": -5.094812870025635,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.537173271179199,
+      "rewards/margins": 0.5576392412185669,
+      "rewards/rejected": -5.094812870025635,
+      "sft_loss": 4.137363433837891,
+      "step": 3720
+    },
+    {
+      "epoch": 1.9936444221441714,
+      "grad_norm": 0.48157595238238843,
+      "learning_rate": 3.0517455818624263e-07,
+      "logits/chosen": -0.8573415875434875,
+      "logits/rejected": -0.7897814512252808,
+      "logps/chosen": -4.485922813415527,
+      "logps/rejected": -5.143821716308594,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.485922813415527,
+      "rewards/margins": 0.6578995585441589,
+      "rewards/rejected": -5.143821716308594,
+      "sft_loss": 4.253503799438477,
+      "step": 3725
+    },
+    {
+      "epoch": 1.9963204549255729,
+      "grad_norm": 0.4845596241734785,
+      "learning_rate": 3.037411959313936e-07,
+      "logits/chosen": -0.7231793403625488,
+      "logits/rejected": -0.5665433406829834,
+      "logps/chosen": -4.50700044631958,
+      "logps/rejected": -5.043286323547363,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.50700044631958,
+      "rewards/margins": 0.5362854599952698,
+      "rewards/rejected": -5.043286323547363,
+      "sft_loss": 4.2585368156433105,
+      "step": 3730
+    },
+    {
+      "epoch": 1.9989964877069744,
+      "grad_norm": 1.072754132396118,
+      "learning_rate": 3.023097377803224e-07,
+      "logits/chosen": -0.7113522887229919,
+      "logits/rejected": -0.6499849557876587,
+      "logps/chosen": -4.468755722045898,
+      "logps/rejected": -5.006053447723389,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.468755722045898,
+      "rewards/margins": 0.5372985005378723,
+      "rewards/rejected": -5.006053447723389,
+      "sft_loss": 4.215538024902344,
+      "step": 3735
+    },
+    {
+      "epoch": 2.001672520488376,
+      "grad_norm": 0.4718352567577755,
+      "learning_rate": 3.008801976210423e-07,
+      "logits/chosen": -0.7235435843467712,
+      "logits/rejected": -0.6755806803703308,
+      "logps/chosen": -4.443969249725342,
+      "logps/rejected": -4.865131855010986,
+      "loss": 0.053,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.443969249725342,
+      "rewards/margins": 0.42116230726242065,
+      "rewards/rejected": -4.865131855010986,
+      "sft_loss": 4.133395195007324,
+      "step": 3740
+    },
+    {
+      "epoch": 2.0043485532697773,
+      "grad_norm": 0.45285574666734346,
+      "learning_rate": 2.994525893229581e-07,
+      "logits/chosen": -0.7920453548431396,
+      "logits/rejected": -0.7268325090408325,
+      "logps/chosen": -4.540145397186279,
+      "logps/rejected": -5.200549125671387,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.540145397186279,
+      "rewards/margins": 0.6604036688804626,
+      "rewards/rejected": -5.200549125671387,
+      "sft_loss": 4.260335445404053,
+      "step": 3745
+    },
+    {
+      "epoch": 2.007024586051179,
+      "grad_norm": 0.4211428890658203,
+      "learning_rate": 2.98026926736732e-07,
+      "logits/chosen": -0.8909608721733093,
+      "logits/rejected": -0.7725866436958313,
+      "logps/chosen": -4.55959415435791,
+      "logps/rejected": -5.068253040313721,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.55959415435791,
+      "rewards/margins": 0.5086590647697449,
+      "rewards/rejected": -5.068253040313721,
+      "sft_loss": 4.258984088897705,
+      "step": 3750
+    },
+    {
+      "epoch": 2.0097006188325808,
+      "grad_norm": 0.4308395644884328,
+      "learning_rate": 2.9660322369414846e-07,
+      "logits/chosen": -0.8293676376342773,
+      "logits/rejected": -0.6661375761032104,
+      "logps/chosen": -4.575963973999023,
+      "logps/rejected": -5.214879989624023,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.575963973999023,
+      "rewards/margins": 0.638916015625,
+      "rewards/rejected": -5.214879989624023,
+      "sft_loss": 4.343644618988037,
+      "step": 3755
+    },
+    {
+      "epoch": 2.0123766516139825,
+      "grad_norm": 0.2876446754182886,
+      "learning_rate": 2.9518149400798063e-07,
+      "logits/chosen": -0.850312352180481,
+      "logits/rejected": -0.8274194002151489,
+      "logps/chosen": -4.500282287597656,
+      "logps/rejected": -5.1650567054748535,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.500282287597656,
+      "rewards/margins": 0.6647745370864868,
+      "rewards/rejected": -5.1650567054748535,
+      "sft_loss": 4.275763511657715,
+      "step": 3760
+    },
+    {
+      "epoch": 2.0150526843953838,
+      "grad_norm": 0.39282700418404237,
+      "learning_rate": 2.9376175147185633e-07,
+      "logits/chosen": -0.844325065612793,
+      "logits/rejected": -0.5920781493186951,
+      "logps/chosen": -4.502379894256592,
+      "logps/rejected": -5.151594638824463,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.502379894256592,
+      "rewards/margins": 0.649213969707489,
+      "rewards/rejected": -5.151594638824463,
+      "sft_loss": 4.227784633636475,
+      "step": 3765
+    },
+    {
+      "epoch": 2.0177287171767855,
+      "grad_norm": 0.42074838124805847,
+      "learning_rate": 2.9234400986012376e-07,
+      "logits/chosen": -0.8818314671516418,
+      "logits/rejected": -0.6762841939926147,
+      "logps/chosen": -4.428236961364746,
+      "logps/rejected": -5.1282124519348145,
+      "loss": 0.051,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.428236961364746,
+      "rewards/margins": 0.699974775314331,
+      "rewards/rejected": -5.1282124519348145,
+      "sft_loss": 4.132842063903809,
+      "step": 3770
+    },
+    {
+      "epoch": 2.020404749958187,
+      "grad_norm": 0.4956780345069873,
+      "learning_rate": 2.9092828292771817e-07,
+      "logits/chosen": -0.7929924130439758,
+      "logits/rejected": -0.7674404382705688,
+      "logps/chosen": -4.421003818511963,
+      "logps/rejected": -5.078767776489258,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.421003818511963,
+      "rewards/margins": 0.6577636003494263,
+      "rewards/rejected": -5.078767776489258,
+      "sft_loss": 4.150858402252197,
+      "step": 3775
+    },
+    {
+      "epoch": 2.0230807827395885,
+      "grad_norm": 0.3934700234275401,
+      "learning_rate": 2.8951458441002875e-07,
+      "logits/chosen": -0.698670506477356,
+      "logits/rejected": -0.7248165011405945,
+      "logps/chosen": -4.2970147132873535,
+      "logps/rejected": -4.952702522277832,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.2970147132873535,
+      "rewards/margins": 0.655687689781189,
+      "rewards/rejected": -4.952702522277832,
+      "sft_loss": 4.0070271492004395,
+      "step": 3780
+    },
+    {
+      "epoch": 2.02575681552099,
+      "grad_norm": 0.514669682771898,
+      "learning_rate": 2.881029280227643e-07,
+      "logits/chosen": -0.8328143358230591,
+      "logits/rejected": -0.6541591882705688,
+      "logps/chosen": -4.35631799697876,
+      "logps/rejected": -5.068148612976074,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.35631799697876,
+      "rewards/margins": 0.7118302583694458,
+      "rewards/rejected": -5.068148612976074,
+      "sft_loss": 4.113364219665527,
+      "step": 3785
+    },
+    {
+      "epoch": 2.028432848302392,
+      "grad_norm": 0.5041909432877334,
+      "learning_rate": 2.8669332746182177e-07,
+      "logits/chosen": -0.9019726514816284,
+      "logits/rejected": -0.7289915084838867,
+      "logps/chosen": -4.497717380523682,
+      "logps/rejected": -5.219419002532959,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.497717380523682,
+      "rewards/margins": 0.7217018604278564,
+      "rewards/rejected": -5.219419002532959,
+      "sft_loss": 4.304531097412109,
+      "step": 3790
+    },
+    {
+      "epoch": 2.031108881083793,
+      "grad_norm": 0.39504962692275486,
+      "learning_rate": 2.8528579640315156e-07,
+      "logits/chosen": -0.7584611177444458,
+      "logits/rejected": -0.7723220586776733,
+      "logps/chosen": -4.636214733123779,
+      "logps/rejected": -5.066348075866699,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.636214733123779,
+      "rewards/margins": 0.43013325333595276,
+      "rewards/rejected": -5.066348075866699,
+      "sft_loss": 4.299112796783447,
+      "step": 3795
+    },
+    {
+      "epoch": 2.033784913865195,
+      "grad_norm": 0.670166400454059,
+      "learning_rate": 2.8388034850262646e-07,
+      "logits/chosen": -0.7787135243415833,
+      "logits/rejected": -0.6269806623458862,
+      "logps/chosen": -4.497548580169678,
+      "logps/rejected": -5.134943962097168,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.497548580169678,
+      "rewards/margins": 0.637395441532135,
+      "rewards/rejected": -5.134943962097168,
+      "sft_loss": 4.2226152420043945,
+      "step": 3800
+    },
+    {
+      "epoch": 2.0364609466465966,
+      "grad_norm": 0.41224442261042604,
+      "learning_rate": 2.824769973959079e-07,
+      "logits/chosen": -0.770029604434967,
+      "logits/rejected": -0.6335395574569702,
+      "logps/chosen": -4.504012584686279,
+      "logps/rejected": -5.04500150680542,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.504012584686279,
+      "rewards/margins": 0.5409889221191406,
+      "rewards/rejected": -5.04500150680542,
+      "sft_loss": 4.187629222869873,
+      "step": 3805
+    },
+    {
+      "epoch": 2.039136979427998,
+      "grad_norm": 0.44522007655542895,
+      "learning_rate": 2.81075756698315e-07,
+      "logits/chosen": -0.615141749382019,
+      "logits/rejected": -0.5446062088012695,
+      "logps/chosen": -4.580349922180176,
+      "logps/rejected": -5.290804862976074,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.580349922180176,
+      "rewards/margins": 0.7104545831680298,
+      "rewards/rejected": -5.290804862976074,
+      "sft_loss": 4.226175785064697,
+      "step": 3810
+    },
+    {
+      "epoch": 2.0418130122093996,
+      "grad_norm": 0.36982501701742904,
+      "learning_rate": 2.7967664000469035e-07,
+      "logits/chosen": -0.8423372507095337,
+      "logits/rejected": -0.7574399709701538,
+      "logps/chosen": -4.433310031890869,
+      "logps/rejected": -5.036944389343262,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.433310031890869,
+      "rewards/margins": 0.6036348342895508,
+      "rewards/rejected": -5.036944389343262,
+      "sft_loss": 4.126814842224121,
+      "step": 3815
+    },
+    {
+      "epoch": 2.0444890449908013,
+      "grad_norm": 0.4483686878186856,
+      "learning_rate": 2.7827966088927095e-07,
+      "logits/chosen": -0.8709455728530884,
+      "logits/rejected": -0.6395670175552368,
+      "logps/chosen": -4.461734294891357,
+      "logps/rejected": -5.337707042694092,
+      "loss": 0.051,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.461734294891357,
+      "rewards/margins": 0.8759723901748657,
+      "rewards/rejected": -5.337707042694092,
+      "sft_loss": 4.206548690795898,
+      "step": 3820
+    },
+    {
+      "epoch": 2.0471650777722026,
+      "grad_norm": 0.5525955846494557,
+      "learning_rate": 2.768848329055538e-07,
+      "logits/chosen": -0.7681508660316467,
+      "logits/rejected": -0.725921630859375,
+      "logps/chosen": -4.424405574798584,
+      "logps/rejected": -5.064883232116699,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.424405574798584,
+      "rewards/margins": 0.6404775977134705,
+      "rewards/rejected": -5.064883232116699,
+      "sft_loss": 4.222466945648193,
+      "step": 3825
+    },
+    {
+      "epoch": 2.0498411105536043,
+      "grad_norm": 0.6304674537764908,
+      "learning_rate": 2.7549216958616657e-07,
+      "logits/chosen": -0.8746339678764343,
+      "logits/rejected": -0.734113872051239,
+      "logps/chosen": -4.404292106628418,
+      "logps/rejected": -5.094416618347168,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.404292106628418,
+      "rewards/margins": 0.6901249885559082,
+      "rewards/rejected": -5.094416618347168,
+      "sft_loss": 4.037627220153809,
+      "step": 3830
+    },
+    {
+      "epoch": 2.052517143335006,
+      "grad_norm": 0.5009731987661138,
+      "learning_rate": 2.741016844427344e-07,
+      "logits/chosen": -0.7817445993423462,
+      "logits/rejected": -0.601385235786438,
+      "logps/chosen": -4.428588390350342,
+      "logps/rejected": -5.240962028503418,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.428588390350342,
+      "rewards/margins": 0.8123737573623657,
+      "rewards/rejected": -5.240962028503418,
+      "sft_loss": 4.188836097717285,
+      "step": 3835
+    },
+    {
+      "epoch": 2.0551931761164073,
+      "grad_norm": 0.6299463597143712,
+      "learning_rate": 2.7271339096575073e-07,
+      "logits/chosen": -0.7218152284622192,
+      "logits/rejected": -0.577179491519928,
+      "logps/chosen": -4.575904846191406,
+      "logps/rejected": -5.296813011169434,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.575904846191406,
+      "rewards/margins": 0.7209089994430542,
+      "rewards/rejected": -5.296813011169434,
+      "sft_loss": 4.2866058349609375,
+      "step": 3840
+    },
+    {
+      "epoch": 2.057869208897809,
+      "grad_norm": 0.4385588051996929,
+      "learning_rate": 2.713273026244446e-07,
+      "logits/chosen": -0.8641475439071655,
+      "logits/rejected": -0.6361261606216431,
+      "logps/chosen": -4.40885066986084,
+      "logps/rejected": -5.324732303619385,
+      "loss": 0.0502,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -4.40885066986084,
+      "rewards/margins": 0.9158821105957031,
+      "rewards/rejected": -5.324732303619385,
+      "sft_loss": 4.178675651550293,
+      "step": 3845
+    },
+    {
+      "epoch": 2.0605452416792107,
+      "grad_norm": 0.4168058120641021,
+      "learning_rate": 2.6994343286665156e-07,
+      "logits/chosen": -0.8316490054130554,
+      "logits/rejected": -0.619785726070404,
+      "logps/chosen": -4.314603328704834,
+      "logps/rejected": -5.067779064178467,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.314603328704834,
+      "rewards/margins": 0.7531753778457642,
+      "rewards/rejected": -5.067779064178467,
+      "sft_loss": 4.156607627868652,
+      "step": 3850
+    },
+    {
+      "epoch": 2.063221274460612,
+      "grad_norm": 0.5075649961997,
+      "learning_rate": 2.6856179511868156e-07,
+      "logits/chosen": -0.7296597957611084,
+      "logits/rejected": -0.5145146250724792,
+      "logps/chosen": -4.354414463043213,
+      "logps/rejected": -5.239541530609131,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.354414463043213,
+      "rewards/margins": 0.8851274251937866,
+      "rewards/rejected": -5.239541530609131,
+      "sft_loss": 4.124449253082275,
+      "step": 3855
+    },
+    {
+      "epoch": 2.0658973072420137,
+      "grad_norm": 1.1238836661784852,
+      "learning_rate": 2.6718240278519056e-07,
+      "logits/chosen": -0.6999574899673462,
+      "logits/rejected": -0.5747981071472168,
+      "logps/chosen": -4.261233329772949,
+      "logps/rejected": -5.282289981842041,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.261233329772949,
+      "rewards/margins": 1.021056056022644,
+      "rewards/rejected": -5.282289981842041,
+      "sft_loss": 4.051290035247803,
+      "step": 3860
+    },
+    {
+      "epoch": 2.0685733400234154,
+      "grad_norm": 0.4627375151319719,
+      "learning_rate": 2.6580526924904866e-07,
+      "logits/chosen": -0.8693877458572388,
+      "logits/rejected": -0.7199397087097168,
+      "logps/chosen": -4.682175636291504,
+      "logps/rejected": -5.245555400848389,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.682175636291504,
+      "rewards/margins": 0.5633805990219116,
+      "rewards/rejected": -5.245555400848389,
+      "sft_loss": 4.376290321350098,
+      "step": 3865
+    },
+    {
+      "epoch": 2.0712493728048167,
+      "grad_norm": 0.5160298725641026,
+      "learning_rate": 2.6443040787121186e-07,
+      "logits/chosen": -0.9046756029129028,
+      "logits/rejected": -0.8257226943969727,
+      "logps/chosen": -4.422331809997559,
+      "logps/rejected": -5.142786979675293,
+      "loss": 0.051,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.422331809997559,
+      "rewards/margins": 0.720454752445221,
+      "rewards/rejected": -5.142786979675293,
+      "sft_loss": 4.169841289520264,
+      "step": 3870
+    },
+    {
+      "epoch": 2.0739254055862184,
+      "grad_norm": 0.5638296933110125,
+      "learning_rate": 2.6305783199059084e-07,
+      "logits/chosen": -0.7855437397956848,
+      "logits/rejected": -0.7157570123672485,
+      "logps/chosen": -4.402723789215088,
+      "logps/rejected": -4.955962181091309,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.402723789215088,
+      "rewards/margins": 0.5532382726669312,
+      "rewards/rejected": -4.955962181091309,
+      "sft_loss": 4.0784478187561035,
+      "step": 3875
+    },
+    {
+      "epoch": 2.07660143836762,
+      "grad_norm": 0.38165204325516433,
+      "learning_rate": 2.6168755492392324e-07,
+      "logits/chosen": -0.7893201112747192,
+      "logits/rejected": -0.637191653251648,
+      "logps/chosen": -4.291540145874023,
+      "logps/rejected": -5.171633720397949,
+      "loss": 0.0489,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.291540145874023,
+      "rewards/margins": 0.8800934553146362,
+      "rewards/rejected": -5.171633720397949,
+      "sft_loss": 3.9720988273620605,
+      "step": 3880
+    },
+    {
+      "epoch": 2.0792774711490214,
+      "grad_norm": 0.6583163664132554,
+      "learning_rate": 2.6031958996564274e-07,
+      "logits/chosen": -0.8037883043289185,
+      "logits/rejected": -0.7117749452590942,
+      "logps/chosen": -4.445651054382324,
+      "logps/rejected": -5.30559778213501,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.445651054382324,
+      "rewards/margins": 0.859946608543396,
+      "rewards/rejected": -5.30559778213501,
+      "sft_loss": 4.167903900146484,
+      "step": 3885
+    },
+    {
+      "epoch": 2.081953503930423,
+      "grad_norm": 0.4715461336878274,
+      "learning_rate": 2.589539503877518e-07,
+      "logits/chosen": -0.7256428599357605,
+      "logits/rejected": -0.6271122097969055,
+      "logps/chosen": -4.560754776000977,
+      "logps/rejected": -5.249648094177246,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.560754776000977,
+      "rewards/margins": 0.6888927817344666,
+      "rewards/rejected": -5.249648094177246,
+      "sft_loss": 4.257047176361084,
+      "step": 3890
+    },
+    {
+      "epoch": 2.084629536711825,
+      "grad_norm": 0.5647077173254447,
+      "learning_rate": 2.5759064943969125e-07,
+      "logits/chosen": -0.8433957099914551,
+      "logits/rejected": -0.6119260787963867,
+      "logps/chosen": -4.4959588050842285,
+      "logps/rejected": -5.253961086273193,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.4959588050842285,
+      "rewards/margins": 0.7580021023750305,
+      "rewards/rejected": -5.253961086273193,
+      "sft_loss": 4.209029197692871,
+      "step": 3895
+    },
+    {
+      "epoch": 2.087305569493226,
+      "grad_norm": 0.3465141084373431,
+      "learning_rate": 2.562297003482131e-07,
+      "logits/chosen": -0.660112202167511,
+      "logits/rejected": -0.6809910535812378,
+      "logps/chosen": -4.294495582580566,
+      "logps/rejected": -5.017926216125488,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.294495582580566,
+      "rewards/margins": 0.7234315872192383,
+      "rewards/rejected": -5.017926216125488,
+      "sft_loss": 4.066096782684326,
+      "step": 3900
+    },
+    {
+      "epoch": 2.089981602274628,
+      "grad_norm": 0.3979943305801828,
+      "learning_rate": 2.548711163172512e-07,
+      "logits/chosen": -0.700593888759613,
+      "logits/rejected": -0.64739990234375,
+      "logps/chosen": -4.466081142425537,
+      "logps/rejected": -4.965723514556885,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.466081142425537,
+      "rewards/margins": 0.49964267015457153,
+      "rewards/rejected": -4.965723514556885,
+      "sft_loss": 4.193602561950684,
+      "step": 3905
+    },
+    {
+      "epoch": 2.0926576350560295,
+      "grad_norm": 0.42928400117151966,
+      "learning_rate": 2.53514910527794e-07,
+      "logits/chosen": -0.727473795413971,
+      "logits/rejected": -0.5996404886245728,
+      "logps/chosen": -4.518040180206299,
+      "logps/rejected": -5.068653583526611,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.518040180206299,
+      "rewards/margins": 0.5506137609481812,
+      "rewards/rejected": -5.068653583526611,
+      "sft_loss": 4.262098789215088,
+      "step": 3910
+    },
+    {
+      "epoch": 2.095333667837431,
+      "grad_norm": 0.37355707424726675,
+      "learning_rate": 2.5216109613775573e-07,
+      "logits/chosen": -0.8052509427070618,
+      "logits/rejected": -0.6294256448745728,
+      "logps/chosen": -4.585465431213379,
+      "logps/rejected": -5.209853649139404,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.585465431213379,
+      "rewards/margins": 0.6243882179260254,
+      "rewards/rejected": -5.209853649139404,
+      "sft_loss": 4.374361991882324,
+      "step": 3915
+    },
+    {
+      "epoch": 2.0980097006188325,
+      "grad_norm": 0.48572956394977185,
+      "learning_rate": 2.5080968628184993e-07,
+      "logits/chosen": -0.7453235387802124,
+      "logits/rejected": -0.5784560441970825,
+      "logps/chosen": -4.471060752868652,
+      "logps/rejected": -5.356838226318359,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.471060752868652,
+      "rewards/margins": 0.8857781291007996,
+      "rewards/rejected": -5.356838226318359,
+      "sft_loss": 4.242326736450195,
+      "step": 3920
+    },
+    {
+      "epoch": 2.1006857334002342,
+      "grad_norm": 0.6413231575213167,
+      "learning_rate": 2.494606940714605e-07,
+      "logits/chosen": -0.694869875907898,
+      "logits/rejected": -0.641283392906189,
+      "logps/chosen": -4.33608865737915,
+      "logps/rejected": -5.122622489929199,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.33608865737915,
+      "rewards/margins": 0.7865341305732727,
+      "rewards/rejected": -5.122622489929199,
+      "sft_loss": 4.0861897468566895,
+      "step": 3925
+    },
+    {
+      "epoch": 2.103361766181636,
+      "grad_norm": 0.6794536127536217,
+      "learning_rate": 2.4811413259451625e-07,
+      "logits/chosen": -0.8177544474601746,
+      "logits/rejected": -0.6390523910522461,
+      "logps/chosen": -4.208698272705078,
+      "logps/rejected": -4.995844841003418,
+      "loss": 0.051,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.208698272705078,
+      "rewards/margins": 0.7871468663215637,
+      "rewards/rejected": -4.995844841003418,
+      "sft_loss": 4.0283613204956055,
+      "step": 3930
+    },
+    {
+      "epoch": 2.106037798963037,
+      "grad_norm": 0.635316600850136,
+      "learning_rate": 2.46770014915362e-07,
+      "logits/chosen": -0.7075114846229553,
+      "logits/rejected": -0.6430591344833374,
+      "logps/chosen": -4.4350762367248535,
+      "logps/rejected": -5.117311000823975,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.4350762367248535,
+      "rewards/margins": 0.6822348833084106,
+      "rewards/rejected": -5.117311000823975,
+      "sft_loss": 4.175159454345703,
+      "step": 3935
+    },
+    {
+      "epoch": 2.108713831744439,
+      "grad_norm": 0.5542245200634366,
+      "learning_rate": 2.45428354074634e-07,
+      "logits/chosen": -0.7162960171699524,
+      "logits/rejected": -0.6563288569450378,
+      "logps/chosen": -4.496886730194092,
+      "logps/rejected": -5.226065635681152,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.496886730194092,
+      "rewards/margins": 0.7291792631149292,
+      "rewards/rejected": -5.226065635681152,
+      "sft_loss": 4.221349716186523,
+      "step": 3940
+    },
+    {
+      "epoch": 2.1113898645258407,
+      "grad_norm": 0.6123482841719484,
+      "learning_rate": 2.4408916308913105e-07,
+      "logits/chosen": -0.7921692132949829,
+      "logits/rejected": -0.5897735357284546,
+      "logps/chosen": -4.634129524230957,
+      "logps/rejected": -5.069488525390625,
+      "loss": 0.0544,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -4.634129524230957,
+      "rewards/margins": 0.4353586733341217,
+      "rewards/rejected": -5.069488525390625,
+      "sft_loss": 4.371542930603027,
+      "step": 3945
+    },
+    {
+      "epoch": 2.114065897307242,
+      "grad_norm": 0.4166900257171104,
+      "learning_rate": 2.4275245495169025e-07,
+      "logits/chosen": -0.694656491279602,
+      "logits/rejected": -0.5563549399375916,
+      "logps/chosen": -4.529080867767334,
+      "logps/rejected": -5.242933750152588,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.529080867767334,
+      "rewards/margins": 0.7138529419898987,
+      "rewards/rejected": -5.242933750152588,
+      "sft_loss": 4.234822750091553,
+      "step": 3950
+    },
+    {
+      "epoch": 2.1167419300886436,
+      "grad_norm": 0.4856713048710508,
+      "learning_rate": 2.414182426310597e-07,
+      "logits/chosen": -0.7959322333335876,
+      "logits/rejected": -0.769672691822052,
+      "logps/chosen": -4.510166168212891,
+      "logps/rejected": -5.311366081237793,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.510166168212891,
+      "rewards/margins": 0.801199734210968,
+      "rewards/rejected": -5.311366081237793,
+      "sft_loss": 4.296000003814697,
+      "step": 3955
+    },
+    {
+      "epoch": 2.1194179628700454,
+      "grad_norm": 0.6243875236957093,
+      "learning_rate": 2.400865390717734e-07,
+      "logits/chosen": -0.7460509538650513,
+      "logits/rejected": -0.6690871119499207,
+      "logps/chosen": -4.389871120452881,
+      "logps/rejected": -5.315016746520996,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.389871120452881,
+      "rewards/margins": 0.9251459240913391,
+      "rewards/rejected": -5.315016746520996,
+      "sft_loss": 4.183564186096191,
+      "step": 3960
+    },
+    {
+      "epoch": 2.1220939956514466,
+      "grad_norm": 0.5658353195120891,
+      "learning_rate": 2.3875735719402475e-07,
+      "logits/chosen": -0.7674288749694824,
+      "logits/rejected": -0.634276807308197,
+      "logps/chosen": -4.3222336769104,
+      "logps/rejected": -5.103603839874268,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.3222336769104,
+      "rewards/margins": 0.7813698649406433,
+      "rewards/rejected": -5.103603839874268,
+      "sft_loss": 4.120206356048584,
+      "step": 3965
+    },
+    {
+      "epoch": 2.1247700284328483,
+      "grad_norm": 0.5640306652342747,
+      "learning_rate": 2.3743070989354258e-07,
+      "logits/chosen": -0.7015902996063232,
+      "logits/rejected": -0.6395735740661621,
+      "logps/chosen": -4.416680335998535,
+      "logps/rejected": -5.131161689758301,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.416680335998535,
+      "rewards/margins": 0.7144818305969238,
+      "rewards/rejected": -5.131161689758301,
+      "sft_loss": 4.159177780151367,
+      "step": 3970
+    },
+    {
+      "epoch": 2.12744606121425,
+      "grad_norm": 0.8818729398968036,
+      "learning_rate": 2.3610661004146454e-07,
+      "logits/chosen": -0.6776810884475708,
+      "logits/rejected": -0.5723994374275208,
+      "logps/chosen": -4.257053375244141,
+      "logps/rejected": -4.910529136657715,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.257053375244141,
+      "rewards/margins": 0.6534755229949951,
+      "rewards/rejected": -4.910529136657715,
+      "sft_loss": 3.978513240814209,
+      "step": 3975
+    },
+    {
+      "epoch": 2.1301220939956513,
+      "grad_norm": 0.42266618942146955,
+      "learning_rate": 2.3478507048421314e-07,
+      "logits/chosen": -0.7549012899398804,
+      "logits/rejected": -0.6445156335830688,
+      "logps/chosen": -4.521634578704834,
+      "logps/rejected": -5.223923683166504,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.521634578704834,
+      "rewards/margins": 0.7022888660430908,
+      "rewards/rejected": -5.223923683166504,
+      "sft_loss": 4.206809997558594,
+      "step": 3980
+    },
+    {
+      "epoch": 2.132798126777053,
+      "grad_norm": 0.46043866853182125,
+      "learning_rate": 2.334661040433713e-07,
+      "logits/chosen": -0.8515769243240356,
+      "logits/rejected": -0.7206335663795471,
+      "logps/chosen": -4.5367536544799805,
+      "logps/rejected": -5.210167407989502,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.5367536544799805,
+      "rewards/margins": 0.6734140515327454,
+      "rewards/rejected": -5.210167407989502,
+      "sft_loss": 4.247954368591309,
+      "step": 3985
+    },
+    {
+      "epoch": 2.1354741595584548,
+      "grad_norm": 0.5358993416695645,
+      "learning_rate": 2.321497235155568e-07,
+      "logits/chosen": -0.8475052118301392,
+      "logits/rejected": -0.688243567943573,
+      "logps/chosen": -4.262228965759277,
+      "logps/rejected": -5.043619632720947,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -4.262228965759277,
+      "rewards/margins": 0.7813904881477356,
+      "rewards/rejected": -5.043619632720947,
+      "sft_loss": 4.01896858215332,
+      "step": 3990
+    },
+    {
+      "epoch": 2.138150192339856,
+      "grad_norm": 0.4150651391308651,
+      "learning_rate": 2.3083594167229965e-07,
+      "logits/chosen": -0.8656677007675171,
+      "logits/rejected": -0.5582382678985596,
+      "logps/chosen": -4.472632884979248,
+      "logps/rejected": -5.255242347717285,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.472632884979248,
+      "rewards/margins": 0.7826098203659058,
+      "rewards/rejected": -5.255242347717285,
+      "sft_loss": 4.183423042297363,
+      "step": 3995
+    },
+    {
+      "epoch": 2.1408262251212578,
+      "grad_norm": 0.4563166112067603,
+      "learning_rate": 2.295247712599167e-07,
+      "logits/chosen": -0.7128655910491943,
+      "logits/rejected": -0.6650645136833191,
+      "logps/chosen": -4.469484806060791,
+      "logps/rejected": -5.218777179718018,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.469484806060791,
+      "rewards/margins": 0.7492923736572266,
+      "rewards/rejected": -5.218777179718018,
+      "sft_loss": 4.08309268951416,
+      "step": 4000
+    },
+    {
+      "epoch": 2.1408262251212578,
+      "eval_logits/chosen": -0.2918679714202881,
+      "eval_logits/rejected": -0.21033975481987,
+      "eval_logps/chosen": -4.464621543884277,
+      "eval_logps/rejected": -5.165764808654785,
+      "eval_loss": 0.050455424934625626,
+      "eval_rewards/accuracies": 0.6839762330055237,
+      "eval_rewards/chosen": -4.464621543884277,
+      "eval_rewards/margins": 0.7011440396308899,
+      "eval_rewards/rejected": -5.165764808654785,
+      "eval_runtime": 43.4608,
+      "eval_samples_per_second": 30.947,
+      "eval_sft_loss": 4.058549404144287,
+      "eval_steps_per_second": 7.754,
+      "step": 4000
+    },
+    {
+      "epoch": 2.1435022579026595,
+      "grad_norm": 0.6727805412244077,
+      "learning_rate": 2.2821622499938948e-07,
+      "logits/chosen": -0.7068689465522766,
+      "logits/rejected": -0.47361889481544495,
+      "logps/chosen": -4.4505085945129395,
+      "logps/rejected": -5.127585411071777,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.4505085945129395,
+      "rewards/margins": 0.6770769357681274,
+      "rewards/rejected": -5.127585411071777,
+      "sft_loss": 4.1432719230651855,
+      "step": 4005
+    },
+    {
+      "epoch": 2.1461782906840607,
+      "grad_norm": 0.32224127814306514,
+      "learning_rate": 2.269103155862391e-07,
+      "logits/chosen": -0.8249640464782715,
+      "logits/rejected": -0.7345031499862671,
+      "logps/chosen": -4.459612846374512,
+      "logps/rejected": -5.158649444580078,
+      "loss": 0.05,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.459612846374512,
+      "rewards/margins": 0.6990362405776978,
+      "rewards/rejected": -5.158649444580078,
+      "sft_loss": 4.074860095977783,
+      "step": 4010
+    },
+    {
+      "epoch": 2.1488543234654625,
+      "grad_norm": 0.5506638370672858,
+      "learning_rate": 2.2560705569040483e-07,
+      "logits/chosen": -0.7756333351135254,
+      "logits/rejected": -0.47715824842453003,
+      "logps/chosen": -4.469395637512207,
+      "logps/rejected": -5.134026527404785,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.469395637512207,
+      "rewards/margins": 0.6646314859390259,
+      "rewards/rejected": -5.134026527404785,
+      "sft_loss": 4.218961238861084,
+      "step": 4015
+    },
+    {
+      "epoch": 2.151530356246864,
+      "grad_norm": 0.3443569983012991,
+      "learning_rate": 2.2430645795611963e-07,
+      "logits/chosen": -0.8699603080749512,
+      "logits/rejected": -0.7662721276283264,
+      "logps/chosen": -4.326430320739746,
+      "logps/rejected": -5.181424140930176,
+      "loss": 0.0499,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.326430320739746,
+      "rewards/margins": 0.854993999004364,
+      "rewards/rejected": -5.181424140930176,
+      "sft_loss": 4.081225395202637,
+      "step": 4020
+    },
+    {
+      "epoch": 2.1542063890282654,
+      "grad_norm": 0.46911133108933106,
+      "learning_rate": 2.230085350017884e-07,
+      "logits/chosen": -0.7873638868331909,
+      "logits/rejected": -0.6871055364608765,
+      "logps/chosen": -4.547414302825928,
+      "logps/rejected": -5.233478546142578,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.547414302825928,
+      "rewards/margins": 0.6860648393630981,
+      "rewards/rejected": -5.233478546142578,
+      "sft_loss": 4.271318435668945,
+      "step": 4025
+    },
+    {
+      "epoch": 2.156882421809667,
+      "grad_norm": 0.3329126350332984,
+      "learning_rate": 2.2171329941986554e-07,
+      "logits/chosen": -0.805925726890564,
+      "logits/rejected": -0.7726612091064453,
+      "logps/chosen": -4.549063682556152,
+      "logps/rejected": -5.230555057525635,
+      "loss": 0.0496,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.549063682556152,
+      "rewards/margins": 0.6814913153648376,
+      "rewards/rejected": -5.230555057525635,
+      "sft_loss": 4.1083083152771,
+      "step": 4030
+    },
+    {
+      "epoch": 2.159558454591069,
+      "grad_norm": 0.35451698745934257,
+      "learning_rate": 2.2042076377673202e-07,
+      "logits/chosen": -0.7483960390090942,
+      "logits/rejected": -0.8004158139228821,
+      "logps/chosen": -4.400989055633545,
+      "logps/rejected": -4.926598072052002,
+      "loss": 0.052,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.400989055633545,
+      "rewards/margins": 0.5256087183952332,
+      "rewards/rejected": -4.926598072052002,
+      "sft_loss": 4.080153465270996,
+      "step": 4035
+    },
+    {
+      "epoch": 2.16223448737247,
+      "grad_norm": 0.3835866808895003,
+      "learning_rate": 2.1913094061257476e-07,
+      "logits/chosen": -0.7398586869239807,
+      "logits/rejected": -0.7862161993980408,
+      "logps/chosen": -4.51585054397583,
+      "logps/rejected": -5.192282676696777,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.51585054397583,
+      "rewards/margins": 0.6764318346977234,
+      "rewards/rejected": -5.192282676696777,
+      "sft_loss": 4.22283935546875,
+      "step": 4040
+    },
+    {
+      "epoch": 2.164910520153872,
+      "grad_norm": 0.5104943845035439,
+      "learning_rate": 2.178438424412633e-07,
+      "logits/chosen": -0.7213504314422607,
+      "logits/rejected": -0.576121985912323,
+      "logps/chosen": -4.430332183837891,
+      "logps/rejected": -5.1058149337768555,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.430332183837891,
+      "rewards/margins": 0.6754826307296753,
+      "rewards/rejected": -5.1058149337768555,
+      "sft_loss": 4.154946804046631,
+      "step": 4045
+    },
+    {
+      "epoch": 2.1675865529352736,
+      "grad_norm": 0.4674501996338367,
+      "learning_rate": 2.165594817502302e-07,
+      "logits/chosen": -0.8302208781242371,
+      "logits/rejected": -0.6873995065689087,
+      "logps/chosen": -4.677610397338867,
+      "logps/rejected": -5.197269916534424,
+      "loss": 0.0549,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.677610397338867,
+      "rewards/margins": 0.5196588635444641,
+      "rewards/rejected": -5.197269916534424,
+      "sft_loss": 4.4770989418029785,
+      "step": 4050
+    },
+    {
+      "epoch": 2.170262585716675,
+      "grad_norm": 0.7933969079589734,
+      "learning_rate": 2.1527787100034806e-07,
+      "logits/chosen": -0.646473228931427,
+      "logits/rejected": -0.5958948135375977,
+      "logps/chosen": -4.4888916015625,
+      "logps/rejected": -5.003674030303955,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.4888916015625,
+      "rewards/margins": 0.5147822499275208,
+      "rewards/rejected": -5.003674030303955,
+      "sft_loss": 4.261991024017334,
+      "step": 4055
+    },
+    {
+      "epoch": 2.1729386184980766,
+      "grad_norm": 0.4931963118357809,
+      "learning_rate": 2.1399902262581037e-07,
+      "logits/chosen": -0.7125464677810669,
+      "logits/rejected": -0.5157756209373474,
+      "logps/chosen": -4.63405704498291,
+      "logps/rejected": -5.100339412689209,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.63405704498291,
+      "rewards/margins": 0.4662818908691406,
+      "rewards/rejected": -5.100339412689209,
+      "sft_loss": 4.32558012008667,
+      "step": 4060
+    },
+    {
+      "epoch": 2.1756146512794783,
+      "grad_norm": 0.4731138425152067,
+      "learning_rate": 2.127229490340094e-07,
+      "logits/chosen": -0.8338751792907715,
+      "logits/rejected": -0.767744243144989,
+      "logps/chosen": -4.563037872314453,
+      "logps/rejected": -5.240099906921387,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.563037872314453,
+      "rewards/margins": 0.6770623922348022,
+      "rewards/rejected": -5.240099906921387,
+      "sft_loss": 4.253331661224365,
+      "step": 4065
+    },
+    {
+      "epoch": 2.1782906840608796,
+      "grad_norm": 0.6017968419842186,
+      "learning_rate": 2.1144966260541698e-07,
+      "logits/chosen": -0.7297824025154114,
+      "logits/rejected": -0.48667675256729126,
+      "logps/chosen": -4.502832412719727,
+      "logps/rejected": -5.1569108963012695,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.502832412719727,
+      "rewards/margins": 0.6540783047676086,
+      "rewards/rejected": -5.1569108963012695,
+      "sft_loss": 4.205613613128662,
+      "step": 4070
+    },
+    {
+      "epoch": 2.1809667168422813,
+      "grad_norm": 0.605056601130136,
+      "learning_rate": 2.1017917569346332e-07,
+      "logits/chosen": -0.7562541961669922,
+      "logits/rejected": -0.5662790536880493,
+      "logps/chosen": -4.355158805847168,
+      "logps/rejected": -5.044077396392822,
+      "loss": 0.0498,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.355158805847168,
+      "rewards/margins": 0.6889182329177856,
+      "rewards/rejected": -5.044077396392822,
+      "sft_loss": 4.024393081665039,
+      "step": 4075
+    },
+    {
+      "epoch": 2.183642749623683,
+      "grad_norm": 0.4321712990803465,
+      "learning_rate": 2.0891150062441837e-07,
+      "logits/chosen": -0.7883024215698242,
+      "logits/rejected": -0.6575873494148254,
+      "logps/chosen": -4.471930503845215,
+      "logps/rejected": -5.290343284606934,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.471930503845215,
+      "rewards/margins": 0.8184129595756531,
+      "rewards/rejected": -5.290343284606934,
+      "sft_loss": 4.168588161468506,
+      "step": 4080
+    },
+    {
+      "epoch": 2.1863187824050843,
+      "grad_norm": 0.5782685093734775,
+      "learning_rate": 2.0764664969727086e-07,
+      "logits/chosen": -0.6935213804244995,
+      "logits/rejected": -0.6597651243209839,
+      "logps/chosen": -4.432676792144775,
+      "logps/rejected": -5.141415596008301,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.432676792144775,
+      "rewards/margins": 0.708739161491394,
+      "rewards/rejected": -5.141415596008301,
+      "sft_loss": 4.131624221801758,
+      "step": 4085
+    },
+    {
+      "epoch": 2.188994815186486,
+      "grad_norm": 0.4630350649187951,
+      "learning_rate": 2.0638463518361033e-07,
+      "logits/chosen": -0.8443444967269897,
+      "logits/rejected": -0.6547593474388123,
+      "logps/chosen": -4.442532539367676,
+      "logps/rejected": -5.188077449798584,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.442532539367676,
+      "rewards/margins": 0.7455454468727112,
+      "rewards/rejected": -5.188077449798584,
+      "sft_loss": 4.115714073181152,
+      "step": 4090
+    },
+    {
+      "epoch": 2.1916708479678877,
+      "grad_norm": 0.38394800610862523,
+      "learning_rate": 2.0512546932750702e-07,
+      "logits/chosen": -0.8786581158638,
+      "logits/rejected": -0.8057855367660522,
+      "logps/chosen": -4.48517370223999,
+      "logps/rejected": -5.181807041168213,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.48517370223999,
+      "rewards/margins": 0.6966326236724854,
+      "rewards/rejected": -5.181807041168213,
+      "sft_loss": 4.173416614532471,
+      "step": 4095
+    },
+    {
+      "epoch": 2.194346880749289,
+      "grad_norm": 0.6202912991395683,
+      "learning_rate": 2.0386916434539343e-07,
+      "logits/chosen": -0.7296054363250732,
+      "logits/rejected": -0.5482798218727112,
+      "logps/chosen": -4.548017501831055,
+      "logps/rejected": -5.236302375793457,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.548017501831055,
+      "rewards/margins": 0.6882843971252441,
+      "rewards/rejected": -5.236302375793457,
+      "sft_loss": 4.268096446990967,
+      "step": 4100
+    },
+    {
+      "epoch": 2.1970229135306907,
+      "grad_norm": 0.3488816624853031,
+      "learning_rate": 2.0261573242594627e-07,
+      "logits/chosen": -0.795524001121521,
+      "logits/rejected": -0.5811234712600708,
+      "logps/chosen": -4.477652072906494,
+      "logps/rejected": -5.104580879211426,
+      "loss": 0.051,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.477652072906494,
+      "rewards/margins": 0.6269285678863525,
+      "rewards/rejected": -5.104580879211426,
+      "sft_loss": 4.149083137512207,
+      "step": 4105
+    },
+    {
+      "epoch": 2.1996989463120924,
+      "grad_norm": 0.6510997768380627,
+      "learning_rate": 2.0136518572996724e-07,
+      "logits/chosen": -0.7480728030204773,
+      "logits/rejected": -0.5544711351394653,
+      "logps/chosen": -4.376706600189209,
+      "logps/rejected": -5.170914173126221,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.376706600189209,
+      "rewards/margins": 0.7942078709602356,
+      "rewards/rejected": -5.170914173126221,
+      "sft_loss": 4.084542274475098,
+      "step": 4110
+    },
+    {
+      "epoch": 2.202374979093494,
+      "grad_norm": 0.7380670951006642,
+      "learning_rate": 2.0011753639026617e-07,
+      "logits/chosen": -0.722009003162384,
+      "logits/rejected": -0.6640048027038574,
+      "logps/chosen": -4.3249921798706055,
+      "logps/rejected": -5.086312294006348,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.3249921798706055,
+      "rewards/margins": 0.7613199949264526,
+      "rewards/rejected": -5.086312294006348,
+      "sft_loss": 4.073752403259277,
+      "step": 4115
+    },
+    {
+      "epoch": 2.2050510118748954,
+      "grad_norm": 0.6413941733357926,
+      "learning_rate": 1.988727965115421e-07,
+      "logits/chosen": -0.7592009902000427,
+      "logits/rejected": -0.6666856408119202,
+      "logps/chosen": -4.346851348876953,
+      "logps/rejected": -5.01780366897583,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.346851348876953,
+      "rewards/margins": 0.6709519028663635,
+      "rewards/rejected": -5.01780366897583,
+      "sft_loss": 4.029229640960693,
+      "step": 4120
+    },
+    {
+      "epoch": 2.207727044656297,
+      "grad_norm": 0.513048432496639,
+      "learning_rate": 1.9763097817026713e-07,
+      "logits/chosen": -0.8778330087661743,
+      "logits/rejected": -0.6732766032218933,
+      "logps/chosen": -4.389376640319824,
+      "logps/rejected": -5.263741493225098,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.389376640319824,
+      "rewards/margins": 0.8743650317192078,
+      "rewards/rejected": -5.263741493225098,
+      "sft_loss": 4.129411697387695,
+      "step": 4125
+    },
+    {
+      "epoch": 2.210403077437699,
+      "grad_norm": 0.6110412211607691,
+      "learning_rate": 1.9639209341456796e-07,
+      "logits/chosen": -0.713258683681488,
+      "logits/rejected": -0.632881760597229,
+      "logps/chosen": -4.497132778167725,
+      "logps/rejected": -5.232693195343018,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.497132778167725,
+      "rewards/margins": 0.7355600595474243,
+      "rewards/rejected": -5.232693195343018,
+      "sft_loss": 4.2595624923706055,
+      "step": 4130
+    },
+    {
+      "epoch": 2.2130791102191,
+      "grad_norm": 0.6504637140199618,
+      "learning_rate": 1.951561542641102e-07,
+      "logits/chosen": -0.6895782947540283,
+      "logits/rejected": -0.7371370792388916,
+      "logps/chosen": -4.693241119384766,
+      "logps/rejected": -5.284863471984863,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.693241119384766,
+      "rewards/margins": 0.5916224718093872,
+      "rewards/rejected": -5.284863471984863,
+      "sft_loss": 4.434107780456543,
+      "step": 4135
+    },
+    {
+      "epoch": 2.215755143000502,
+      "grad_norm": 0.4543352377658658,
+      "learning_rate": 1.939231727099806e-07,
+      "logits/chosen": -0.9554224014282227,
+      "logits/rejected": -0.9246322512626648,
+      "logps/chosen": -4.4521613121032715,
+      "logps/rejected": -5.085289001464844,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.4521613121032715,
+      "rewards/margins": 0.6331278085708618,
+      "rewards/rejected": -5.085289001464844,
+      "sft_loss": 4.227784633636475,
+      "step": 4140
+    },
+    {
+      "epoch": 2.2184311757819035,
+      "grad_norm": 0.4284689952652344,
+      "learning_rate": 1.926931607145719e-07,
+      "logits/chosen": -0.6809648871421814,
+      "logits/rejected": -0.5421003103256226,
+      "logps/chosen": -4.518945693969727,
+      "logps/rejected": -5.160582065582275,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.518945693969727,
+      "rewards/margins": 0.6416367292404175,
+      "rewards/rejected": -5.160582065582275,
+      "sft_loss": 4.263835906982422,
+      "step": 4145
+    },
+    {
+      "epoch": 2.221107208563305,
+      "grad_norm": 0.43424034162181147,
+      "learning_rate": 1.9146613021146564e-07,
+      "logits/chosen": -0.7884284257888794,
+      "logits/rejected": -0.680195152759552,
+      "logps/chosen": -4.412850379943848,
+      "logps/rejected": -5.129262924194336,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.412850379943848,
+      "rewards/margins": 0.7164131999015808,
+      "rewards/rejected": -5.129262924194336,
+      "sft_loss": 4.188337326049805,
+      "step": 4150
+    },
+    {
+      "epoch": 2.2237832413447065,
+      "grad_norm": 0.4962627859609203,
+      "learning_rate": 1.9024209310531736e-07,
+      "logits/chosen": -0.7695158123970032,
+      "logits/rejected": -0.7895294427871704,
+      "logps/chosen": -4.313672065734863,
+      "logps/rejected": -4.9209136962890625,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.313672065734863,
+      "rewards/margins": 0.6072418689727783,
+      "rewards/rejected": -4.9209136962890625,
+      "sft_loss": 3.9769134521484375,
+      "step": 4155
+    },
+    {
+      "epoch": 2.2264592741261082,
+      "grad_norm": 0.481643698172507,
+      "learning_rate": 1.890210612717401e-07,
+      "logits/chosen": -0.7807799577713013,
+      "logits/rejected": -0.6693819165229797,
+      "logps/chosen": -4.398434638977051,
+      "logps/rejected": -5.057561874389648,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.398434638977051,
+      "rewards/margins": 0.6591275334358215,
+      "rewards/rejected": -5.057561874389648,
+      "sft_loss": 4.108274936676025,
+      "step": 4160
+    },
+    {
+      "epoch": 2.2291353069075095,
+      "grad_norm": 0.7413910120393763,
+      "learning_rate": 1.8780304655719054e-07,
+      "logits/chosen": -0.8045433163642883,
+      "logits/rejected": -0.6661940813064575,
+      "logps/chosen": -4.460057258605957,
+      "logps/rejected": -5.185713291168213,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.460057258605957,
+      "rewards/margins": 0.725655734539032,
+      "rewards/rejected": -5.185713291168213,
+      "sft_loss": 4.1655497550964355,
+      "step": 4165
+    },
+    {
+      "epoch": 2.231811339688911,
+      "grad_norm": 0.4732959746390582,
+      "learning_rate": 1.865880607788523e-07,
+      "logits/chosen": -0.6616698503494263,
+      "logits/rejected": -0.5911990404129028,
+      "logps/chosen": -4.5712995529174805,
+      "logps/rejected": -5.2418389320373535,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.5712995529174805,
+      "rewards/margins": 0.6705387234687805,
+      "rewards/rejected": -5.2418389320373535,
+      "sft_loss": 4.364117622375488,
+      "step": 4170
+    },
+    {
+      "epoch": 2.234487372470313,
+      "grad_norm": 0.8094323886363942,
+      "learning_rate": 1.8537611572452316e-07,
+      "logits/chosen": -0.794582724571228,
+      "logits/rejected": -0.7341277599334717,
+      "logps/chosen": -4.37639856338501,
+      "logps/rejected": -5.012908935546875,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.37639856338501,
+      "rewards/margins": 0.6365109086036682,
+      "rewards/rejected": -5.012908935546875,
+      "sft_loss": 4.13184118270874,
+      "step": 4175
+    },
+    {
+      "epoch": 2.237163405251714,
+      "grad_norm": 0.4541511078659696,
+      "learning_rate": 1.84167223152499e-07,
+      "logits/chosen": -0.7890401482582092,
+      "logits/rejected": -0.5393490791320801,
+      "logps/chosen": -4.449801921844482,
+      "logps/rejected": -5.224527835845947,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.449801921844482,
+      "rewards/margins": 0.7747262120246887,
+      "rewards/rejected": -5.224527835845947,
+      "sft_loss": 4.180199146270752,
+      "step": 4180
+    },
+    {
+      "epoch": 2.239839438033116,
+      "grad_norm": 0.48971091863328464,
+      "learning_rate": 1.8296139479146112e-07,
+      "logits/chosen": -0.733080267906189,
+      "logits/rejected": -0.7528313398361206,
+      "logps/chosen": -4.379557132720947,
+      "logps/rejected": -4.981081008911133,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.379557132720947,
+      "rewards/margins": 0.6015235781669617,
+      "rewards/rejected": -4.981081008911133,
+      "sft_loss": 4.055811405181885,
+      "step": 4185
+    },
+    {
+      "epoch": 2.2425154708145176,
+      "grad_norm": 0.4429743125528724,
+      "learning_rate": 1.8175864234036132e-07,
+      "logits/chosen": -0.5849838852882385,
+      "logits/rejected": -0.5171209573745728,
+      "logps/chosen": -4.486813545227051,
+      "logps/rejected": -5.217012405395508,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.486813545227051,
+      "rewards/margins": 0.7301994562149048,
+      "rewards/rejected": -5.217012405395508,
+      "sft_loss": 4.215363502502441,
+      "step": 4190
+    },
+    {
+      "epoch": 2.245191503595919,
+      "grad_norm": 0.3529342606467267,
+      "learning_rate": 1.805589774683094e-07,
+      "logits/chosen": -0.8781973719596863,
+      "logits/rejected": -0.7594443559646606,
+      "logps/chosen": -4.630221366882324,
+      "logps/rejected": -5.19809627532959,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.630221366882324,
+      "rewards/margins": 0.5678743124008179,
+      "rewards/rejected": -5.19809627532959,
+      "sft_loss": 4.259702205657959,
+      "step": 4195
+    },
+    {
+      "epoch": 2.2478675363773206,
+      "grad_norm": 0.7539970445961938,
+      "learning_rate": 1.79362411814459e-07,
+      "logits/chosen": -0.5950525999069214,
+      "logits/rejected": -0.6381107568740845,
+      "logps/chosen": -4.614771842956543,
+      "logps/rejected": -5.1353583335876465,
+      "loss": 0.0539,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.614771842956543,
+      "rewards/margins": 0.5205863118171692,
+      "rewards/rejected": -5.1353583335876465,
+      "sft_loss": 4.3355183601379395,
+      "step": 4200
+    },
+    {
+      "epoch": 2.2505435691587223,
+      "grad_norm": 0.47071934411282107,
+      "learning_rate": 1.7816895698789552e-07,
+      "logits/chosen": -0.8427804112434387,
+      "logits/rejected": -0.7817627191543579,
+      "logps/chosen": -4.487260341644287,
+      "logps/rejected": -5.036301612854004,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.487260341644287,
+      "rewards/margins": 0.5490409135818481,
+      "rewards/rejected": -5.036301612854004,
+      "sft_loss": 4.125479698181152,
+      "step": 4205
+    },
+    {
+      "epoch": 2.2532196019401236,
+      "grad_norm": 0.5406217374875316,
+      "learning_rate": 1.7697862456752271e-07,
+      "logits/chosen": -0.8153530955314636,
+      "logits/rejected": -0.690669059753418,
+      "logps/chosen": -4.521656036376953,
+      "logps/rejected": -5.366150856018066,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.521656036376953,
+      "rewards/margins": 0.8444948196411133,
+      "rewards/rejected": -5.366150856018066,
+      "sft_loss": 4.276778221130371,
+      "step": 4210
+    },
+    {
+      "epoch": 2.2558956347215253,
+      "grad_norm": 0.5810986187677788,
+      "learning_rate": 1.7579142610195124e-07,
+      "logits/chosen": -0.7868109941482544,
+      "logits/rejected": -0.6204288601875305,
+      "logps/chosen": -4.511598587036133,
+      "logps/rejected": -5.312621116638184,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.511598587036133,
+      "rewards/margins": 0.8010231256484985,
+      "rewards/rejected": -5.312621116638184,
+      "sft_loss": 4.231732368469238,
+      "step": 4215
+    },
+    {
+      "epoch": 2.258571667502927,
+      "grad_norm": 0.3654874885769355,
+      "learning_rate": 1.7460737310938568e-07,
+      "logits/chosen": -0.8439321517944336,
+      "logits/rejected": -0.6458622217178345,
+      "logps/chosen": -4.324358940124512,
+      "logps/rejected": -5.175654411315918,
+      "loss": 0.05,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -4.324358940124512,
+      "rewards/margins": 0.8512958288192749,
+      "rewards/rejected": -5.175654411315918,
+      "sft_loss": 4.1335344314575195,
+      "step": 4220
+    },
+    {
+      "epoch": 2.2612477002843283,
+      "grad_norm": 0.3690452777813944,
+      "learning_rate": 1.734264770775133e-07,
+      "logits/chosen": -0.8422131538391113,
+      "logits/rejected": -0.5545368790626526,
+      "logps/chosen": -4.361656665802002,
+      "logps/rejected": -5.022474765777588,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.361656665802002,
+      "rewards/margins": 0.660818338394165,
+      "rewards/rejected": -5.022474765777588,
+      "sft_loss": 4.097499847412109,
+      "step": 4225
+    },
+    {
+      "epoch": 2.26392373306573,
+      "grad_norm": 0.40414939716552284,
+      "learning_rate": 1.7224874946339241e-07,
+      "logits/chosen": -0.799502968788147,
+      "logits/rejected": -0.7204964756965637,
+      "logps/chosen": -4.439566612243652,
+      "logps/rejected": -5.170541763305664,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.439566612243652,
+      "rewards/margins": 0.7309752106666565,
+      "rewards/rejected": -5.170541763305664,
+      "sft_loss": 4.066781044006348,
+      "step": 4230
+    },
+    {
+      "epoch": 2.2665997658471317,
+      "grad_norm": 0.5952940841735112,
+      "learning_rate": 1.7107420169334186e-07,
+      "logits/chosen": -0.7833027243614197,
+      "logits/rejected": -0.7465739846229553,
+      "logps/chosen": -4.582724094390869,
+      "logps/rejected": -5.141078472137451,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.582724094390869,
+      "rewards/margins": 0.5583544969558716,
+      "rewards/rejected": -5.141078472137451,
+      "sft_loss": 4.265813827514648,
+      "step": 4235
+    },
+    {
+      "epoch": 2.269275798628533,
+      "grad_norm": 0.4398041156053364,
+      "learning_rate": 1.6990284516282893e-07,
+      "logits/chosen": -0.7631226778030396,
+      "logits/rejected": -0.6925583481788635,
+      "logps/chosen": -4.404193878173828,
+      "logps/rejected": -5.143115520477295,
+      "loss": 0.05,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.404193878173828,
+      "rewards/margins": 0.7389219403266907,
+      "rewards/rejected": -5.143115520477295,
+      "sft_loss": 4.091065883636475,
+      "step": 4240
+    },
+    {
+      "epoch": 2.2719518314099347,
+      "grad_norm": 0.5284040359104598,
+      "learning_rate": 1.687346912363602e-07,
+      "logits/chosen": -0.8135977983474731,
+      "logits/rejected": -0.6727418303489685,
+      "logps/chosen": -4.361421585083008,
+      "logps/rejected": -5.143485069274902,
+      "loss": 0.0502,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.361421585083008,
+      "rewards/margins": 0.7820636630058289,
+      "rewards/rejected": -5.143485069274902,
+      "sft_loss": 4.039944648742676,
+      "step": 4245
+    },
+    {
+      "epoch": 2.2746278641913364,
+      "grad_norm": 0.427930060724418,
+      "learning_rate": 1.675697512473697e-07,
+      "logits/chosen": -0.7885524034500122,
+      "logits/rejected": -0.5956434011459351,
+      "logps/chosen": -4.521124362945557,
+      "logps/rejected": -5.362552642822266,
+      "loss": 0.0496,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -4.521124362945557,
+      "rewards/margins": 0.8414284586906433,
+      "rewards/rejected": -5.362552642822266,
+      "sft_loss": 4.1074538230896,
+      "step": 4250
+    },
+    {
+      "epoch": 2.2773038969727377,
+      "grad_norm": 0.4548490650516944,
+      "learning_rate": 1.6640803649811087e-07,
+      "logits/chosen": -0.806907057762146,
+      "logits/rejected": -0.5140247344970703,
+      "logps/chosen": -4.435779571533203,
+      "logps/rejected": -5.372751712799072,
+      "loss": 0.0502,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.435779571533203,
+      "rewards/margins": 0.9369718432426453,
+      "rewards/rejected": -5.372751712799072,
+      "sft_loss": 4.071789741516113,
+      "step": 4255
+    },
+    {
+      "epoch": 2.2799799297541394,
+      "grad_norm": 0.433676020311226,
+      "learning_rate": 1.6524955825954472e-07,
+      "logits/chosen": -0.7338653802871704,
+      "logits/rejected": -0.675475001335144,
+      "logps/chosen": -4.471889019012451,
+      "logps/rejected": -5.032140254974365,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.471889019012451,
+      "rewards/margins": 0.5602517127990723,
+      "rewards/rejected": -5.032140254974365,
+      "sft_loss": 4.22143030166626,
+      "step": 4260
+    },
+    {
+      "epoch": 2.282655962535541,
+      "grad_norm": 0.44709679685057646,
+      "learning_rate": 1.6409432777123277e-07,
+      "logits/chosen": -0.8582879304885864,
+      "logits/rejected": -0.7037560939788818,
+      "logps/chosen": -4.3130292892456055,
+      "logps/rejected": -5.295225143432617,
+      "loss": 0.049,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.3130292892456055,
+      "rewards/margins": 0.9821959733963013,
+      "rewards/rejected": -5.295225143432617,
+      "sft_loss": 4.0433030128479,
+      "step": 4265
+    },
+    {
+      "epoch": 2.285331995316943,
+      "grad_norm": 0.6454731179033166,
+      "learning_rate": 1.6294235624122577e-07,
+      "logits/chosen": -0.6963884234428406,
+      "logits/rejected": -0.4233129620552063,
+      "logps/chosen": -4.356287956237793,
+      "logps/rejected": -5.090372085571289,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.356287956237793,
+      "rewards/margins": 0.7340839505195618,
+      "rewards/rejected": -5.090372085571289,
+      "sft_loss": 4.082320213317871,
+      "step": 4270
+    },
+    {
+      "epoch": 2.288008028098344,
+      "grad_norm": 0.4092375456675233,
+      "learning_rate": 1.6179365484595697e-07,
+      "logits/chosen": -0.7554842233657837,
+      "logits/rejected": -0.6546878218650818,
+      "logps/chosen": -4.4962849617004395,
+      "logps/rejected": -5.201930999755859,
+      "loss": 0.052,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.4962849617004395,
+      "rewards/margins": 0.7056463360786438,
+      "rewards/rejected": -5.201930999755859,
+      "sft_loss": 4.25368595123291,
+      "step": 4275
+    },
+    {
+      "epoch": 2.290684060879746,
+      "grad_norm": 0.7796030085865294,
+      "learning_rate": 1.60648234730132e-07,
+      "logits/chosen": -0.8003331422805786,
+      "logits/rejected": -0.7303368449211121,
+      "logps/chosen": -4.419107913970947,
+      "logps/rejected": -5.249577522277832,
+      "loss": 0.05,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -4.419107913970947,
+      "rewards/margins": 0.8304697275161743,
+      "rewards/rejected": -5.249577522277832,
+      "sft_loss": 4.124005317687988,
+      "step": 4280
+    },
+    {
+      "epoch": 2.293360093661147,
+      "grad_norm": 0.5726370771240188,
+      "learning_rate": 1.595061070066222e-07,
+      "logits/chosen": -0.6955457925796509,
+      "logits/rejected": -0.7629978060722351,
+      "logps/chosen": -4.541815757751465,
+      "logps/rejected": -5.194106101989746,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.541815757751465,
+      "rewards/margins": 0.6522905826568604,
+      "rewards/rejected": -5.194106101989746,
+      "sft_loss": 4.267590522766113,
+      "step": 4285
+    },
+    {
+      "epoch": 2.296036126442549,
+      "grad_norm": 0.47755328394996693,
+      "learning_rate": 1.5836728275635542e-07,
+      "logits/chosen": -0.8143297433853149,
+      "logits/rejected": -0.6354633569717407,
+      "logps/chosen": -4.457322120666504,
+      "logps/rejected": -5.215367317199707,
+      "loss": 0.051,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.457322120666504,
+      "rewards/margins": 0.7580451965332031,
+      "rewards/rejected": -5.215367317199707,
+      "sft_loss": 4.198925971984863,
+      "step": 4290
+    },
+    {
+      "epoch": 2.2987121592239506,
+      "grad_norm": 0.3876912944758544,
+      "learning_rate": 1.5723177302820984e-07,
+      "logits/chosen": -0.8010802268981934,
+      "logits/rejected": -0.7300946712493896,
+      "logps/chosen": -4.525595664978027,
+      "logps/rejected": -5.123916149139404,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.525595664978027,
+      "rewards/margins": 0.5983200073242188,
+      "rewards/rejected": -5.123916149139404,
+      "sft_loss": 4.291536331176758,
+      "step": 4295
+    },
+    {
+      "epoch": 2.3013881920053523,
+      "grad_norm": 0.4036539451488429,
+      "learning_rate": 1.5609958883890544e-07,
+      "logits/chosen": -0.6942816376686096,
+      "logits/rejected": -0.5985075235366821,
+      "logps/chosen": -4.369370937347412,
+      "logps/rejected": -5.014688014984131,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.369370937347412,
+      "rewards/margins": 0.6453171968460083,
+      "rewards/rejected": -5.014688014984131,
+      "sft_loss": 4.008292198181152,
+      "step": 4300
+    },
+    {
+      "epoch": 2.3040642247867535,
+      "grad_norm": 0.5265430776252449,
+      "learning_rate": 1.5497074117289865e-07,
+      "logits/chosen": -0.753690242767334,
+      "logits/rejected": -0.634030282497406,
+      "logps/chosen": -4.488044738769531,
+      "logps/rejected": -5.264333724975586,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.488044738769531,
+      "rewards/margins": 0.7762894034385681,
+      "rewards/rejected": -5.264333724975586,
+      "sft_loss": 4.194975852966309,
+      "step": 4305
+    },
+    {
+      "epoch": 2.3067402575681553,
+      "grad_norm": 0.3790394544769755,
+      "learning_rate": 1.5384524098227402e-07,
+      "logits/chosen": -0.7402902841567993,
+      "logits/rejected": -0.5147100687026978,
+      "logps/chosen": -4.349900245666504,
+      "logps/rejected": -5.224468231201172,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.349900245666504,
+      "rewards/margins": 0.8745684623718262,
+      "rewards/rejected": -5.224468231201172,
+      "sft_loss": 4.091513156890869,
+      "step": 4310
+    },
+    {
+      "epoch": 2.3094162903495565,
+      "grad_norm": 0.5268560797622209,
+      "learning_rate": 1.5272309918663974e-07,
+      "logits/chosen": -0.7533577084541321,
+      "logits/rejected": -0.5765639543533325,
+      "logps/chosen": -4.552016735076904,
+      "logps/rejected": -5.05606746673584,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.552016735076904,
+      "rewards/margins": 0.5040509104728699,
+      "rewards/rejected": -5.05606746673584,
+      "sft_loss": 4.245696544647217,
+      "step": 4315
+    },
+    {
+      "epoch": 2.3120923231309582,
+      "grad_norm": 0.497137273147704,
+      "learning_rate": 1.516043266730201e-07,
+      "logits/chosen": -0.7740308046340942,
+      "logits/rejected": -0.6450439095497131,
+      "logps/chosen": -4.40801477432251,
+      "logps/rejected": -5.11243200302124,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.40801477432251,
+      "rewards/margins": 0.7044172286987305,
+      "rewards/rejected": -5.11243200302124,
+      "sft_loss": 4.104895114898682,
+      "step": 4320
+    },
+    {
+      "epoch": 2.31476835591236,
+      "grad_norm": 0.5593452729007138,
+      "learning_rate": 1.504889342957512e-07,
+      "logits/chosen": -0.7910572290420532,
+      "logits/rejected": -0.5866572260856628,
+      "logps/chosen": -4.533371448516846,
+      "logps/rejected": -5.306161403656006,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.533371448516846,
+      "rewards/margins": 0.7727904319763184,
+      "rewards/rejected": -5.306161403656006,
+      "sft_loss": 4.273785591125488,
+      "step": 4325
+    },
+    {
+      "epoch": 2.3174443886937617,
+      "grad_norm": 0.4308516372542784,
+      "learning_rate": 1.4937693287637453e-07,
+      "logits/chosen": -0.811202883720398,
+      "logits/rejected": -0.6923697590827942,
+      "logps/chosen": -4.52161169052124,
+      "logps/rejected": -5.191643238067627,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.52161169052124,
+      "rewards/margins": 0.670030951499939,
+      "rewards/rejected": -5.191643238067627,
+      "sft_loss": 4.221057415008545,
+      "step": 4330
+    },
+    {
+      "epoch": 2.320120421475163,
+      "grad_norm": 0.37138887048768443,
+      "learning_rate": 1.4826833320353305e-07,
+      "logits/chosen": -0.7932590246200562,
+      "logits/rejected": -0.7206074595451355,
+      "logps/chosen": -4.624764919281006,
+      "logps/rejected": -5.276534557342529,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.624764919281006,
+      "rewards/margins": 0.6517696976661682,
+      "rewards/rejected": -5.276534557342529,
+      "sft_loss": 4.188200950622559,
+      "step": 4335
+    },
+    {
+      "epoch": 2.3227964542565647,
+      "grad_norm": 0.4345851380582179,
+      "learning_rate": 1.4716314603286528e-07,
+      "logits/chosen": -0.8649484515190125,
+      "logits/rejected": -0.634868323802948,
+      "logps/chosen": -4.535511493682861,
+      "logps/rejected": -5.249735355377197,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.535511493682861,
+      "rewards/margins": 0.7142241597175598,
+      "rewards/rejected": -5.249735355377197,
+      "sft_loss": 4.229328632354736,
+      "step": 4340
+    },
+    {
+      "epoch": 2.3254724870379664,
+      "grad_norm": 0.5104610877999559,
+      "learning_rate": 1.4606138208690233e-07,
+      "logits/chosen": -0.7997044324874878,
+      "logits/rejected": -0.7385850548744202,
+      "logps/chosen": -4.5416460037231445,
+      "logps/rejected": -5.044976711273193,
+      "loss": 0.052,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.5416460037231445,
+      "rewards/margins": 0.5033308267593384,
+      "rewards/rejected": -5.044976711273193,
+      "sft_loss": 4.25277853012085,
+      "step": 4345
+    },
+    {
+      "epoch": 2.3281485198193677,
+      "grad_norm": 0.3642995428464966,
+      "learning_rate": 1.4496305205496251e-07,
+      "logits/chosen": -0.7962000966072083,
+      "logits/rejected": -0.749233067035675,
+      "logps/chosen": -4.49139404296875,
+      "logps/rejected": -5.204357624053955,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.49139404296875,
+      "rewards/margins": 0.7129632830619812,
+      "rewards/rejected": -5.204357624053955,
+      "sft_loss": 4.271798133850098,
+      "step": 4350
+    },
+    {
+      "epoch": 2.3308245526007694,
+      "grad_norm": 0.45592266136726567,
+      "learning_rate": 1.4386816659304895e-07,
+      "logits/chosen": -0.9152060747146606,
+      "logits/rejected": -0.7515543699264526,
+      "logps/chosen": -4.481812477111816,
+      "logps/rejected": -5.05482292175293,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.481812477111816,
+      "rewards/margins": 0.5730103254318237,
+      "rewards/rejected": -5.05482292175293,
+      "sft_loss": 4.24124002456665,
+      "step": 4355
+    },
+    {
+      "epoch": 2.333500585382171,
+      "grad_norm": 0.43698490211025665,
+      "learning_rate": 1.4277673632374492e-07,
+      "logits/chosen": -0.8096843957901001,
+      "logits/rejected": -0.5594863891601562,
+      "logps/chosen": -4.307278156280518,
+      "logps/rejected": -5.064208984375,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.307278156280518,
+      "rewards/margins": 0.7569302320480347,
+      "rewards/rejected": -5.064208984375,
+      "sft_loss": 4.024901866912842,
+      "step": 4360
+    },
+    {
+      "epoch": 2.3361766181635724,
+      "grad_norm": 0.40380334550713853,
+      "learning_rate": 1.416887718361119e-07,
+      "logits/chosen": -0.7265334725379944,
+      "logits/rejected": -0.7379759550094604,
+      "logps/chosen": -4.446936130523682,
+      "logps/rejected": -5.041685581207275,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.446936130523682,
+      "rewards/margins": 0.5947496294975281,
+      "rewards/rejected": -5.041685581207275,
+      "sft_loss": 4.248701095581055,
+      "step": 4365
+    },
+    {
+      "epoch": 2.338852650944974,
+      "grad_norm": 0.4955611851621508,
+      "learning_rate": 1.406042836855859e-07,
+      "logits/chosen": -0.74248206615448,
+      "logits/rejected": -0.6353031396865845,
+      "logps/chosen": -4.549657344818115,
+      "logps/rejected": -5.308493137359619,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.549657344818115,
+      "rewards/margins": 0.7588354349136353,
+      "rewards/rejected": -5.308493137359619,
+      "sft_loss": 4.329131603240967,
+      "step": 4370
+    },
+    {
+      "epoch": 2.341528683726376,
+      "grad_norm": 0.37714745130252286,
+      "learning_rate": 1.3952328239387595e-07,
+      "logits/chosen": -0.8745020627975464,
+      "logits/rejected": -0.6099725365638733,
+      "logps/chosen": -4.425809860229492,
+      "logps/rejected": -5.203674793243408,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.425809860229492,
+      "rewards/margins": 0.7778650522232056,
+      "rewards/rejected": -5.203674793243408,
+      "sft_loss": 4.267220973968506,
+      "step": 4375
+    },
+    {
+      "epoch": 2.344204716507777,
+      "grad_norm": 0.4350403366163347,
+      "learning_rate": 1.3844577844886109e-07,
+      "logits/chosen": -0.8680629730224609,
+      "logits/rejected": -0.635954737663269,
+      "logps/chosen": -4.190207004547119,
+      "logps/rejected": -5.01309871673584,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.190207004547119,
+      "rewards/margins": 0.8228910565376282,
+      "rewards/rejected": -5.01309871673584,
+      "sft_loss": 4.028267860412598,
+      "step": 4380
+    },
+    {
+      "epoch": 2.346880749289179,
+      "grad_norm": 0.5090011205479518,
+      "learning_rate": 1.3737178230448955e-07,
+      "logits/chosen": -0.8712724447250366,
+      "logits/rejected": -0.7374725341796875,
+      "logps/chosen": -4.453272819519043,
+      "logps/rejected": -5.0184431076049805,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.453272819519043,
+      "rewards/margins": 0.5651699900627136,
+      "rewards/rejected": -5.0184431076049805,
+      "sft_loss": 4.107565879821777,
+      "step": 4385
+    },
+    {
+      "epoch": 2.3495567820705805,
+      "grad_norm": 0.44470986998643264,
+      "learning_rate": 1.363013043806764e-07,
+      "logits/chosen": -0.8503999710083008,
+      "logits/rejected": -0.7448652386665344,
+      "logps/chosen": -4.280064105987549,
+      "logps/rejected": -4.982614994049072,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.280064105987549,
+      "rewards/margins": 0.7025504112243652,
+      "rewards/rejected": -4.982614994049072,
+      "sft_loss": 4.109427452087402,
+      "step": 4390
+    },
+    {
+      "epoch": 2.3522328148519818,
+      "grad_norm": 0.7032192811206371,
+      "learning_rate": 1.352343550632034e-07,
+      "logits/chosen": -0.8059431910514832,
+      "logits/rejected": -0.6396316885948181,
+      "logps/chosen": -4.340572357177734,
+      "logps/rejected": -5.229712009429932,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.340572357177734,
+      "rewards/margins": 0.8891401290893555,
+      "rewards/rejected": -5.229712009429932,
+      "sft_loss": 4.127902030944824,
+      "step": 4395
+    },
+    {
+      "epoch": 2.3549088476333835,
+      "grad_norm": 0.39015834229113866,
+      "learning_rate": 1.3417094470361722e-07,
+      "logits/chosen": -0.8471466302871704,
+      "logits/rejected": -0.6921052932739258,
+      "logps/chosen": -4.541820049285889,
+      "logps/rejected": -5.182088375091553,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.541820049285889,
+      "rewards/margins": 0.6402683258056641,
+      "rewards/rejected": -5.182088375091553,
+      "sft_loss": 4.28817081451416,
+      "step": 4400
+    },
+    {
+      "epoch": 2.3549088476333835,
+      "eval_logits/chosen": -0.3632008731365204,
+      "eval_logits/rejected": -0.285022109746933,
+      "eval_logps/chosen": -4.476656436920166,
+      "eval_logps/rejected": -5.172234535217285,
+      "eval_loss": 0.050458874553442,
+      "eval_rewards/accuracies": 0.6839762330055237,
+      "eval_rewards/chosen": -4.476656436920166,
+      "eval_rewards/margins": 0.6955785751342773,
+      "eval_rewards/rejected": -5.172234535217285,
+      "eval_runtime": 43.4365,
+      "eval_samples_per_second": 30.965,
+      "eval_sft_loss": 4.090471267700195,
+      "eval_steps_per_second": 7.758,
+      "step": 4400
+    },
+    {
+      "epoch": 2.357584880414785,
+      "grad_norm": 0.41993569063387165,
+      "learning_rate": 1.3311108361913015e-07,
+      "logits/chosen": -0.899549663066864,
+      "logits/rejected": -0.9365586042404175,
+      "logps/chosen": -4.511231422424316,
+      "logps/rejected": -5.088454246520996,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.511231422424316,
+      "rewards/margins": 0.5772226452827454,
+      "rewards/rejected": -5.088454246520996,
+      "sft_loss": 4.286912441253662,
+      "step": 4405
+    },
+    {
+      "epoch": 2.3602609131961865,
+      "grad_norm": 0.43817622591238337,
+      "learning_rate": 1.3205478209251874e-07,
+      "logits/chosen": -0.7712717056274414,
+      "logits/rejected": -0.7058557868003845,
+      "logps/chosen": -4.5705180168151855,
+      "logps/rejected": -5.281630039215088,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.5705180168151855,
+      "rewards/margins": 0.711111307144165,
+      "rewards/rejected": -5.281630039215088,
+      "sft_loss": 4.32778787612915,
+      "step": 4410
+    },
+    {
+      "epoch": 2.362936945977588,
+      "grad_norm": 0.4837368965922269,
+      "learning_rate": 1.310020503720254e-07,
+      "logits/chosen": -0.793159008026123,
+      "logits/rejected": -0.62529456615448,
+      "logps/chosen": -4.411933898925781,
+      "logps/rejected": -5.071666240692139,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.411933898925781,
+      "rewards/margins": 0.6597329378128052,
+      "rewards/rejected": -5.071666240692139,
+      "sft_loss": 4.156750679016113,
+      "step": 4415
+    },
+    {
+      "epoch": 2.36561297875899,
+      "grad_norm": 0.4008401185189836,
+      "learning_rate": 1.2995289867125752e-07,
+      "logits/chosen": -0.823562741279602,
+      "logits/rejected": -0.7247136831283569,
+      "logps/chosen": -4.479001045227051,
+      "logps/rejected": -5.045378684997559,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.479001045227051,
+      "rewards/margins": 0.5663775205612183,
+      "rewards/rejected": -5.045378684997559,
+      "sft_loss": 4.197256565093994,
+      "step": 4420
+    },
+    {
+      "epoch": 2.368289011540391,
+      "grad_norm": 0.3267884216984576,
+      "learning_rate": 1.2890733716908986e-07,
+      "logits/chosen": -0.7804628014564514,
+      "logits/rejected": -0.7346547842025757,
+      "logps/chosen": -4.284626483917236,
+      "logps/rejected": -4.872819423675537,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.284626483917236,
+      "rewards/margins": 0.5881929397583008,
+      "rewards/rejected": -4.872819423675537,
+      "sft_loss": 4.072215557098389,
+      "step": 4425
+    },
+    {
+      "epoch": 2.370965044321793,
+      "grad_norm": 0.4383847142525197,
+      "learning_rate": 1.2786537600956454e-07,
+      "logits/chosen": -0.8587814569473267,
+      "logits/rejected": -0.6906665563583374,
+      "logps/chosen": -4.313345432281494,
+      "logps/rejected": -5.195896148681641,
+      "loss": 0.0487,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.313345432281494,
+      "rewards/margins": 0.8825508952140808,
+      "rewards/rejected": -5.195896148681641,
+      "sft_loss": 3.9952876567840576,
+      "step": 4430
+    },
+    {
+      "epoch": 2.3736410771031946,
+      "grad_norm": 0.45896207390131155,
+      "learning_rate": 1.268270253017933e-07,
+      "logits/chosen": -0.854796290397644,
+      "logits/rejected": -0.6479583382606506,
+      "logps/chosen": -4.433968544006348,
+      "logps/rejected": -5.119320869445801,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.433968544006348,
+      "rewards/margins": 0.6853523850440979,
+      "rewards/rejected": -5.119320869445801,
+      "sft_loss": 4.195663928985596,
+      "step": 4435
+    },
+    {
+      "epoch": 2.376317109884596,
+      "grad_norm": 0.44999129974127655,
+      "learning_rate": 1.257922951198591e-07,
+      "logits/chosen": -0.9596928358078003,
+      "logits/rejected": -0.657541036605835,
+      "logps/chosen": -4.332813262939453,
+      "logps/rejected": -5.0844526290893555,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.332813262939453,
+      "rewards/margins": 0.7516393065452576,
+      "rewards/rejected": -5.0844526290893555,
+      "sft_loss": 4.116638660430908,
+      "step": 4440
+    },
+    {
+      "epoch": 2.3789931426659976,
+      "grad_norm": 0.5436196141333159,
+      "learning_rate": 1.24761195502719e-07,
+      "logits/chosen": -0.8771514892578125,
+      "logits/rejected": -0.6101125478744507,
+      "logps/chosen": -4.607757091522217,
+      "logps/rejected": -5.213659763336182,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.607757091522217,
+      "rewards/margins": 0.6059027910232544,
+      "rewards/rejected": -5.213659763336182,
+      "sft_loss": 4.251969337463379,
+      "step": 4445
+    },
+    {
+      "epoch": 2.3816691754473993,
+      "grad_norm": 0.7332787813082794,
+      "learning_rate": 1.2373373645410573e-07,
+      "logits/chosen": -0.762103259563446,
+      "logits/rejected": -0.624140739440918,
+      "logps/chosen": -4.644742012023926,
+      "logps/rejected": -5.275030136108398,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.644742012023926,
+      "rewards/margins": 0.6302889585494995,
+      "rewards/rejected": -5.275030136108398,
+      "sft_loss": 4.331053733825684,
+      "step": 4450
+    },
+    {
+      "epoch": 2.384345208228801,
+      "grad_norm": 0.4631029256293614,
+      "learning_rate": 1.2270992794243175e-07,
+      "logits/chosen": -0.8583146333694458,
+      "logits/rejected": -0.7607332468032837,
+      "logps/chosen": -4.444066524505615,
+      "logps/rejected": -5.150336265563965,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.444066524505615,
+      "rewards/margins": 0.7062696218490601,
+      "rewards/rejected": -5.150336265563965,
+      "sft_loss": 4.155452728271484,
+      "step": 4455
+    },
+    {
+      "epoch": 2.3870212410102023,
+      "grad_norm": 0.8308078175306337,
+      "learning_rate": 1.2168977990069147e-07,
+      "logits/chosen": -0.8691369891166687,
+      "logits/rejected": -0.6420689821243286,
+      "logps/chosen": -4.282433032989502,
+      "logps/rejected": -5.122197151184082,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.282433032989502,
+      "rewards/margins": 0.839765191078186,
+      "rewards/rejected": -5.122197151184082,
+      "sft_loss": 3.9892375469207764,
+      "step": 4460
+    },
+    {
+      "epoch": 2.389697273791604,
+      "grad_norm": 0.3763620124349001,
+      "learning_rate": 1.206733022263659e-07,
+      "logits/chosen": -0.8173881769180298,
+      "logits/rejected": -0.6439284086227417,
+      "logps/chosen": -4.300229549407959,
+      "logps/rejected": -5.05617094039917,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.300229549407959,
+      "rewards/margins": 0.7559418678283691,
+      "rewards/rejected": -5.05617094039917,
+      "sft_loss": 3.9857208728790283,
+      "step": 4465
+    },
+    {
+      "epoch": 2.3923733065730053,
+      "grad_norm": 0.5485172489909115,
+      "learning_rate": 1.1966050478132572e-07,
+      "logits/chosen": -0.7517123222351074,
+      "logits/rejected": -0.6935483813285828,
+      "logps/chosen": -4.504305362701416,
+      "logps/rejected": -5.118971347808838,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.504305362701416,
+      "rewards/margins": 0.6146660447120667,
+      "rewards/rejected": -5.118971347808838,
+      "sft_loss": 4.205090522766113,
+      "step": 4470
+    },
+    {
+      "epoch": 2.395049339354407,
+      "grad_norm": 0.42943747081626427,
+      "learning_rate": 1.1865139739173635e-07,
+      "logits/chosen": -0.8406542539596558,
+      "logits/rejected": -0.6299400329589844,
+      "logps/chosen": -4.632096290588379,
+      "logps/rejected": -5.228485584259033,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.632096290588379,
+      "rewards/margins": 0.5963901281356812,
+      "rewards/rejected": -5.228485584259033,
+      "sft_loss": 4.2738494873046875,
+      "step": 4475
+    },
+    {
+      "epoch": 2.3977253721358087,
+      "grad_norm": 0.4209416780986708,
+      "learning_rate": 1.1764598984796187e-07,
+      "logits/chosen": -0.9492759704589844,
+      "logits/rejected": -0.8266083598136902,
+      "logps/chosen": -4.4401984214782715,
+      "logps/rejected": -5.127415657043457,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.4401984214782715,
+      "rewards/margins": 0.6872166395187378,
+      "rewards/rejected": -5.127415657043457,
+      "sft_loss": 4.163792133331299,
+      "step": 4480
+    },
+    {
+      "epoch": 2.4004014049172104,
+      "grad_norm": 0.404322929379254,
+      "learning_rate": 1.1664429190447095e-07,
+      "logits/chosen": -0.7948734760284424,
+      "logits/rejected": -0.7301923036575317,
+      "logps/chosen": -4.572030544281006,
+      "logps/rejected": -5.346141338348389,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.572030544281006,
+      "rewards/margins": 0.7741105556488037,
+      "rewards/rejected": -5.346141338348389,
+      "sft_loss": 4.311251640319824,
+      "step": 4485
+    },
+    {
+      "epoch": 2.4030774376986117,
+      "grad_norm": 0.3949020633907638,
+      "learning_rate": 1.1564631327974122e-07,
+      "logits/chosen": -0.9128694534301758,
+      "logits/rejected": -0.6827823519706726,
+      "logps/chosen": -4.563012599945068,
+      "logps/rejected": -5.230254173278809,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.563012599945068,
+      "rewards/margins": 0.6672413349151611,
+      "rewards/rejected": -5.230254173278809,
+      "sft_loss": 4.242611408233643,
+      "step": 4490
+    },
+    {
+      "epoch": 2.4057534704800134,
+      "grad_norm": 0.553553569804726,
+      "learning_rate": 1.1465206365616587e-07,
+      "logits/chosen": -0.9420261383056641,
+      "logits/rejected": -0.7209168672561646,
+      "logps/chosen": -4.519663333892822,
+      "logps/rejected": -5.303266525268555,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.519663333892822,
+      "rewards/margins": 0.7836031913757324,
+      "rewards/rejected": -5.303266525268555,
+      "sft_loss": 4.19036340713501,
+      "step": 4495
+    },
+    {
+      "epoch": 2.408429503261415,
+      "grad_norm": 0.3632794095375088,
+      "learning_rate": 1.1366155267995887e-07,
+      "logits/chosen": -0.7734476327896118,
+      "logits/rejected": -0.7625142931938171,
+      "logps/chosen": -4.501950263977051,
+      "logps/rejected": -5.078164100646973,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.501950263977051,
+      "rewards/margins": 0.5762136578559875,
+      "rewards/rejected": -5.078164100646973,
+      "sft_loss": 4.153041839599609,
+      "step": 4500
+    },
+    {
+      "epoch": 2.4111055360428164,
+      "grad_norm": 0.45684321809705974,
+      "learning_rate": 1.1267478996106228e-07,
+      "logits/chosen": -0.8146098852157593,
+      "logits/rejected": -0.6006115078926086,
+      "logps/chosen": -4.422109127044678,
+      "logps/rejected": -5.165881156921387,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.422109127044678,
+      "rewards/margins": 0.7437718510627747,
+      "rewards/rejected": -5.165881156921387,
+      "sft_loss": 4.101393699645996,
+      "step": 4505
+    },
+    {
+      "epoch": 2.413781568824218,
+      "grad_norm": 0.4183010395909284,
+      "learning_rate": 1.116917850730521e-07,
+      "logits/chosen": -0.8815043568611145,
+      "logits/rejected": -0.7283543348312378,
+      "logps/chosen": -4.545543193817139,
+      "logps/rejected": -5.047765254974365,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.545543193817139,
+      "rewards/margins": 0.5022218823432922,
+      "rewards/rejected": -5.047765254974365,
+      "sft_loss": 4.220363616943359,
+      "step": 4510
+    },
+    {
+      "epoch": 2.41645760160562,
+      "grad_norm": 0.505742763058506,
+      "learning_rate": 1.1071254755304637e-07,
+      "logits/chosen": -0.8288267850875854,
+      "logits/rejected": -0.7838674187660217,
+      "logps/chosen": -4.297197341918945,
+      "logps/rejected": -5.042912483215332,
+      "loss": 0.0494,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.297197341918945,
+      "rewards/margins": 0.7457149028778076,
+      "rewards/rejected": -5.042912483215332,
+      "sft_loss": 3.972548007965088,
+      "step": 4515
+    },
+    {
+      "epoch": 2.419133634387021,
+      "grad_norm": 0.4752082402543453,
+      "learning_rate": 1.0973708690161143e-07,
+      "logits/chosen": -0.8543336987495422,
+      "logits/rejected": -0.74104243516922,
+      "logps/chosen": -4.514694690704346,
+      "logps/rejected": -5.1261162757873535,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.514694690704346,
+      "rewards/margins": 0.6114215850830078,
+      "rewards/rejected": -5.1261162757873535,
+      "sft_loss": 4.206650257110596,
+      "step": 4520
+    },
+    {
+      "epoch": 2.421809667168423,
+      "grad_norm": 0.47586666938227407,
+      "learning_rate": 1.0876541258267119e-07,
+      "logits/chosen": -0.8609308004379272,
+      "logits/rejected": -0.6737977266311646,
+      "logps/chosen": -4.337949275970459,
+      "logps/rejected": -5.158361911773682,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.337949275970459,
+      "rewards/margins": 0.8204119801521301,
+      "rewards/rejected": -5.158361911773682,
+      "sft_loss": 4.102989196777344,
+      "step": 4525
+    },
+    {
+      "epoch": 2.4244856999498245,
+      "grad_norm": 0.7712988555674866,
+      "learning_rate": 1.0779753402341379e-07,
+      "logits/chosen": -0.9207685589790344,
+      "logits/rejected": -0.84931480884552,
+      "logps/chosen": -4.338393688201904,
+      "logps/rejected": -5.032431602478027,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.338393688201904,
+      "rewards/margins": 0.6940376162528992,
+      "rewards/rejected": -5.032431602478027,
+      "sft_loss": 4.081472396850586,
+      "step": 4530
+    },
+    {
+      "epoch": 2.427161732731226,
+      "grad_norm": 0.5783647376843744,
+      "learning_rate": 1.0683346061420157e-07,
+      "logits/chosen": -0.7379239797592163,
+      "logits/rejected": -0.6506637334823608,
+      "logps/chosen": -4.515606880187988,
+      "logps/rejected": -5.266798973083496,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.515606880187988,
+      "rewards/margins": 0.7511924505233765,
+      "rewards/rejected": -5.266798973083496,
+      "sft_loss": 4.281412124633789,
+      "step": 4535
+    },
+    {
+      "epoch": 2.4298377655126275,
+      "grad_norm": 0.44600935802431835,
+      "learning_rate": 1.0587320170847874e-07,
+      "logits/chosen": -0.8168829083442688,
+      "logits/rejected": -0.7206194400787354,
+      "logps/chosen": -4.545700550079346,
+      "logps/rejected": -5.104142189025879,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.545700550079346,
+      "rewards/margins": 0.5584414601325989,
+      "rewards/rejected": -5.104142189025879,
+      "sft_loss": 4.292700290679932,
+      "step": 4540
+    },
+    {
+      "epoch": 2.4325137982940293,
+      "grad_norm": 0.36067335080276597,
+      "learning_rate": 1.0491676662268156e-07,
+      "logits/chosen": -0.7393335103988647,
+      "logits/rejected": -0.6619399785995483,
+      "logps/chosen": -4.518947601318359,
+      "logps/rejected": -5.257824897766113,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.518947601318359,
+      "rewards/margins": 0.7388771176338196,
+      "rewards/rejected": -5.257824897766113,
+      "sft_loss": 4.184815406799316,
+      "step": 4545
+    },
+    {
+      "epoch": 2.4351898310754305,
+      "grad_norm": 0.5409398505388431,
+      "learning_rate": 1.0396416463614732e-07,
+      "logits/chosen": -0.8931136131286621,
+      "logits/rejected": -0.8121291995048523,
+      "logps/chosen": -4.453886985778809,
+      "logps/rejected": -5.1217451095581055,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.453886985778809,
+      "rewards/margins": 0.667857825756073,
+      "rewards/rejected": -5.1217451095581055,
+      "sft_loss": 4.168035507202148,
+      "step": 4550
+    },
+    {
+      "epoch": 2.4378658638568322,
+      "grad_norm": 1.0388002306851687,
+      "learning_rate": 1.0301540499102479e-07,
+      "logits/chosen": -0.7921608686447144,
+      "logits/rejected": -0.7310789227485657,
+      "logps/chosen": -4.489283561706543,
+      "logps/rejected": -5.021276950836182,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.489283561706543,
+      "rewards/margins": 0.5319927334785461,
+      "rewards/rejected": -5.021276950836182,
+      "sft_loss": 4.189752101898193,
+      "step": 4555
+    },
+    {
+      "epoch": 2.440541896638234,
+      "grad_norm": 0.55059593910096,
+      "learning_rate": 1.0207049689218405e-07,
+      "logits/chosen": -0.9317811131477356,
+      "logits/rejected": -0.6625012755393982,
+      "logps/chosen": -4.478198051452637,
+      "logps/rejected": -5.189414978027344,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.478198051452637,
+      "rewards/margins": 0.7112170457839966,
+      "rewards/rejected": -5.189414978027344,
+      "sft_loss": 4.156213760375977,
+      "step": 4560
+    },
+    {
+      "epoch": 2.4432179294196352,
+      "grad_norm": 0.3662390057934976,
+      "learning_rate": 1.0112944950712782e-07,
+      "logits/chosen": -0.8565780520439148,
+      "logits/rejected": -0.7790525555610657,
+      "logps/chosen": -4.4673357009887695,
+      "logps/rejected": -5.257838249206543,
+      "loss": 0.0502,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.4673357009887695,
+      "rewards/margins": 0.7905027866363525,
+      "rewards/rejected": -5.257838249206543,
+      "sft_loss": 4.1379194259643555,
+      "step": 4565
+    },
+    {
+      "epoch": 2.445893962201037,
+      "grad_norm": 0.59163711890118,
+      "learning_rate": 1.0019227196590174e-07,
+      "logits/chosen": -0.8295857310295105,
+      "logits/rejected": -0.6748948097229004,
+      "logps/chosen": -4.575309753417969,
+      "logps/rejected": -5.304391384124756,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.575309753417969,
+      "rewards/margins": 0.7290816903114319,
+      "rewards/rejected": -5.304391384124756,
+      "sft_loss": 4.324819087982178,
+      "step": 4570
+    },
+    {
+      "epoch": 2.4485699949824387,
+      "grad_norm": 0.3863191151206876,
+      "learning_rate": 9.925897336100664e-08,
+      "logits/chosen": -0.7855613827705383,
+      "logits/rejected": -0.7414531111717224,
+      "logps/chosen": -4.514120101928711,
+      "logps/rejected": -5.226920127868652,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.514120101928711,
+      "rewards/margins": 0.7128003239631653,
+      "rewards/rejected": -5.226920127868652,
+      "sft_loss": 4.242648124694824,
+      "step": 4575
+    },
+    {
+      "epoch": 2.45124602776384,
+      "grad_norm": 0.5313135323825595,
+      "learning_rate": 9.832956274730946e-08,
+      "logits/chosen": -0.7910794019699097,
+      "logits/rejected": -0.7502156496047974,
+      "logps/chosen": -4.682957172393799,
+      "logps/rejected": -5.110443115234375,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.682957172393799,
+      "rewards/margins": 0.42748576402664185,
+      "rewards/rejected": -5.110443115234375,
+      "sft_loss": 4.3094916343688965,
+      "step": 4580
+    },
+    {
+      "epoch": 2.4539220605452416,
+      "grad_norm": 0.6164310535748478,
+      "learning_rate": 9.740404914195633e-08,
+      "logits/chosen": -0.8025332689285278,
+      "logits/rejected": -0.6623189449310303,
+      "logps/chosen": -4.397765159606934,
+      "logps/rejected": -5.139482021331787,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.397765159606934,
+      "rewards/margins": 0.7417163252830505,
+      "rewards/rejected": -5.139482021331787,
+      "sft_loss": 4.196730613708496,
+      "step": 4585
+    },
+    {
+      "epoch": 2.4565980933266434,
+      "grad_norm": 0.5277575602538304,
+      "learning_rate": 9.648244152428392e-08,
+      "logits/chosen": -0.8694614171981812,
+      "logits/rejected": -0.7217963933944702,
+      "logps/chosen": -4.478517055511475,
+      "logps/rejected": -4.989750862121582,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.478517055511475,
+      "rewards/margins": 0.5112346410751343,
+      "rewards/rejected": -4.989750862121582,
+      "sft_loss": 4.229212760925293,
+      "step": 4590
+    },
+    {
+      "epoch": 2.4592741261080446,
+      "grad_norm": 0.3465526549100429,
+      "learning_rate": 9.556474883573379e-08,
+      "logits/chosen": -0.8793339729309082,
+      "logits/rejected": -0.7501915693283081,
+      "logps/chosen": -4.311750411987305,
+      "logps/rejected": -5.096708297729492,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.311750411987305,
+      "rewards/margins": 0.7849579453468323,
+      "rewards/rejected": -5.096708297729492,
+      "sft_loss": 4.058889389038086,
+      "step": 4595
+    },
+    {
+      "epoch": 2.4619501588894463,
+      "grad_norm": 0.6362567491428798,
+      "learning_rate": 9.465097997976412e-08,
+      "logits/chosen": -0.8457925915718079,
+      "logits/rejected": -0.6267693042755127,
+      "logps/chosen": -4.335329532623291,
+      "logps/rejected": -5.132666110992432,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.335329532623291,
+      "rewards/margins": 0.7973363399505615,
+      "rewards/rejected": -5.132666110992432,
+      "sft_loss": 4.092240333557129,
+      "step": 4600
+    },
+    {
+      "epoch": 2.464626191670848,
+      "grad_norm": 0.5709648129829065,
+      "learning_rate": 9.374114382176457e-08,
+      "logits/chosen": -0.8717010617256165,
+      "logits/rejected": -0.6866291761398315,
+      "logps/chosen": -4.533945560455322,
+      "logps/rejected": -5.346493721008301,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.533945560455322,
+      "rewards/margins": 0.8125476837158203,
+      "rewards/rejected": -5.346493721008301,
+      "sft_loss": 4.200273036956787,
+      "step": 4605
+    },
+    {
+      "epoch": 2.46730222445225,
+      "grad_norm": 0.41256938888295447,
+      "learning_rate": 9.283524918896945e-08,
+      "logits/chosen": -0.9007472991943359,
+      "logits/rejected": -0.7356555461883545,
+      "logps/chosen": -4.5387139320373535,
+      "logps/rejected": -5.191751956939697,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.5387139320373535,
+      "rewards/margins": 0.6530376076698303,
+      "rewards/rejected": -5.191751956939697,
+      "sft_loss": 4.2472429275512695,
+      "step": 4610
+    },
+    {
+      "epoch": 2.469978257233651,
+      "grad_norm": 0.46700861734247096,
+      "learning_rate": 9.193330487037232e-08,
+      "logits/chosen": -0.8507378697395325,
+      "logits/rejected": -0.6919487714767456,
+      "logps/chosen": -4.463172912597656,
+      "logps/rejected": -5.266392230987549,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.463172912597656,
+      "rewards/margins": 0.8032194972038269,
+      "rewards/rejected": -5.266392230987549,
+      "sft_loss": 4.272570610046387,
+      "step": 4615
+    },
+    {
+      "epoch": 2.4726542900150528,
+      "grad_norm": 0.4668769656142938,
+      "learning_rate": 9.103531961664118e-08,
+      "logits/chosen": -0.7989420890808105,
+      "logits/rejected": -0.6214526295661926,
+      "logps/chosen": -4.455315113067627,
+      "logps/rejected": -5.171011447906494,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.455315113067627,
+      "rewards/margins": 0.7156961560249329,
+      "rewards/rejected": -5.171011447906494,
+      "sft_loss": 4.132755279541016,
+      "step": 4620
+    },
+    {
+      "epoch": 2.475330322796454,
+      "grad_norm": 0.47924486796798166,
+      "learning_rate": 9.014130214003269e-08,
+      "logits/chosen": -0.8464191555976868,
+      "logits/rejected": -0.8963597416877747,
+      "logps/chosen": -4.2732834815979,
+      "logps/rejected": -4.967657089233398,
+      "loss": 0.051,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.2732834815979,
+      "rewards/margins": 0.6943733096122742,
+      "rewards/rejected": -4.967657089233398,
+      "sft_loss": 3.9865031242370605,
+      "step": 4625
+    },
+    {
+      "epoch": 2.4780063555778558,
+      "grad_norm": 0.32552256494364223,
+      "learning_rate": 8.925126111430848e-08,
+      "logits/chosen": -0.7193098068237305,
+      "logits/rejected": -0.675899088382721,
+      "logps/chosen": -4.4091105461120605,
+      "logps/rejected": -5.017752647399902,
+      "loss": 0.051,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.4091105461120605,
+      "rewards/margins": 0.6086419224739075,
+      "rewards/rejected": -5.017752647399902,
+      "sft_loss": 4.017719268798828,
+      "step": 4630
+    },
+    {
+      "epoch": 2.4806823883592575,
+      "grad_norm": 0.5496220681748125,
+      "learning_rate": 8.83652051746504e-08,
+      "logits/chosen": -0.689524233341217,
+      "logits/rejected": -0.5122482180595398,
+      "logps/chosen": -4.472263813018799,
+      "logps/rejected": -5.335396766662598,
+      "loss": 0.051,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.472263813018799,
+      "rewards/margins": 0.8631328344345093,
+      "rewards/rejected": -5.335396766662598,
+      "sft_loss": 4.2152910232543945,
+      "step": 4635
+    },
+    {
+      "epoch": 2.483358421140659,
+      "grad_norm": 0.31765867467775627,
+      "learning_rate": 8.748314291757696e-08,
+      "logits/chosen": -0.7355210781097412,
+      "logits/rejected": -0.6390531659126282,
+      "logps/chosen": -4.5320844650268555,
+      "logps/rejected": -5.168013572692871,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.5320844650268555,
+      "rewards/margins": 0.635929524898529,
+      "rewards/rejected": -5.168013572692871,
+      "sft_loss": 4.182335376739502,
+      "step": 4640
+    },
+    {
+      "epoch": 2.4860344539220605,
+      "grad_norm": 0.5771252870993228,
+      "learning_rate": 8.660508290086032e-08,
+      "logits/chosen": -0.7999386191368103,
+      "logits/rejected": -0.6518659591674805,
+      "logps/chosen": -4.393448829650879,
+      "logps/rejected": -5.0774006843566895,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.393448829650879,
+      "rewards/margins": 0.6839522123336792,
+      "rewards/rejected": -5.0774006843566895,
+      "sft_loss": 4.168224811553955,
+      "step": 4645
+    },
+    {
+      "epoch": 2.488710486703462,
+      "grad_norm": 0.5150308708468381,
+      "learning_rate": 8.573103364344231e-08,
+      "logits/chosen": -0.8791343569755554,
+      "logits/rejected": -0.6491331458091736,
+      "logps/chosen": -4.423648834228516,
+      "logps/rejected": -5.2099127769470215,
+      "loss": 0.0502,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.423648834228516,
+      "rewards/margins": 0.7862640619277954,
+      "rewards/rejected": -5.2099127769470215,
+      "sft_loss": 4.084849834442139,
+      "step": 4650
+    },
+    {
+      "epoch": 2.4913865194848634,
+      "grad_norm": 0.3661674336980175,
+      "learning_rate": 8.486100362535292e-08,
+      "logits/chosen": -0.8177053332328796,
+      "logits/rejected": -0.6740037202835083,
+      "logps/chosen": -4.4994401931762695,
+      "logps/rejected": -5.0591583251953125,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.4994401931762695,
+      "rewards/margins": 0.5597187280654907,
+      "rewards/rejected": -5.0591583251953125,
+      "sft_loss": 4.204558372497559,
+      "step": 4655
+    },
+    {
+      "epoch": 2.494062552266265,
+      "grad_norm": 0.5140280133573151,
+      "learning_rate": 8.399500128762693e-08,
+      "logits/chosen": -0.8273393511772156,
+      "logits/rejected": -0.7141873240470886,
+      "logps/chosen": -4.562862396240234,
+      "logps/rejected": -5.185477256774902,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.562862396240234,
+      "rewards/margins": 0.6226149201393127,
+      "rewards/rejected": -5.185477256774902,
+      "sft_loss": 4.275196552276611,
+      "step": 4660
+    },
+    {
+      "epoch": 2.496738585047667,
+      "grad_norm": 0.42529640095647614,
+      "learning_rate": 8.313303503222313e-08,
+      "logits/chosen": -0.8213428258895874,
+      "logits/rejected": -0.7776002883911133,
+      "logps/chosen": -4.575163841247559,
+      "logps/rejected": -5.10188102722168,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.575163841247559,
+      "rewards/margins": 0.5267173051834106,
+      "rewards/rejected": -5.10188102722168,
+      "sft_loss": 4.26285982131958,
+      "step": 4665
+    },
+    {
+      "epoch": 2.4994146178290686,
+      "grad_norm": 0.4007308001400934,
+      "learning_rate": 8.227511322194164e-08,
+      "logits/chosen": -0.7592180371284485,
+      "logits/rejected": -0.637819766998291,
+      "logps/chosen": -4.361072063446045,
+      "logps/rejected": -4.973891258239746,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.361072063446045,
+      "rewards/margins": 0.6128195524215698,
+      "rewards/rejected": -4.973891258239746,
+      "sft_loss": 4.0938920974731445,
+      "step": 4670
+    },
+    {
+      "epoch": 2.50209065061047,
+      "grad_norm": 0.40321531518970055,
+      "learning_rate": 8.142124418034385e-08,
+      "logits/chosen": -0.7266249656677246,
+      "logits/rejected": -0.5235757827758789,
+      "logps/chosen": -4.5698018074035645,
+      "logps/rejected": -5.1615800857543945,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.5698018074035645,
+      "rewards/margins": 0.5917780995368958,
+      "rewards/rejected": -5.1615800857543945,
+      "sft_loss": 4.234898567199707,
+      "step": 4675
+    },
+    {
+      "epoch": 2.5047666833918716,
+      "grad_norm": 0.39867424326365275,
+      "learning_rate": 8.057143619167073e-08,
+      "logits/chosen": -0.7164444923400879,
+      "logits/rejected": -0.5923494696617126,
+      "logps/chosen": -4.497181415557861,
+      "logps/rejected": -5.251903533935547,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.497181415557861,
+      "rewards/margins": 0.7547226548194885,
+      "rewards/rejected": -5.251903533935547,
+      "sft_loss": 4.232115745544434,
+      "step": 4680
+    },
+    {
+      "epoch": 2.507442716173273,
+      "grad_norm": 0.5680624236093563,
+      "learning_rate": 7.97256975007633e-08,
+      "logits/chosen": -0.7831182479858398,
+      "logits/rejected": -0.565954327583313,
+      "logps/chosen": -4.234208106994629,
+      "logps/rejected": -5.022221565246582,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.234208106994629,
+      "rewards/margins": 0.7880129814147949,
+      "rewards/rejected": -5.022221565246582,
+      "sft_loss": 4.004078388214111,
+      "step": 4685
+    },
+    {
+      "epoch": 2.5101187489546746,
+      "grad_norm": 0.3881217825976462,
+      "learning_rate": 7.888403631298186e-08,
+      "logits/chosen": -0.6825178861618042,
+      "logits/rejected": -0.6623591184616089,
+      "logps/chosen": -4.475518226623535,
+      "logps/rejected": -5.0694780349731445,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.475518226623535,
+      "rewards/margins": 0.593960165977478,
+      "rewards/rejected": -5.0694780349731445,
+      "sft_loss": 4.17282247543335,
+      "step": 4690
+    },
+    {
+      "epoch": 2.5127947817360763,
+      "grad_norm": 0.4290121459253616,
+      "learning_rate": 7.804646079412719e-08,
+      "logits/chosen": -0.7428678274154663,
+      "logits/rejected": -0.5591611266136169,
+      "logps/chosen": -4.560364246368408,
+      "logps/rejected": -5.201272964477539,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.560364246368408,
+      "rewards/margins": 0.6409088969230652,
+      "rewards/rejected": -5.201272964477539,
+      "sft_loss": 4.295758247375488,
+      "step": 4695
+    },
+    {
+      "epoch": 2.515470814517478,
+      "grad_norm": 0.41012547747397127,
+      "learning_rate": 7.72129790703604e-08,
+      "logits/chosen": -0.8270618319511414,
+      "logits/rejected": -0.6811308264732361,
+      "logps/chosen": -4.522173881530762,
+      "logps/rejected": -5.09686803817749,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.522173881530762,
+      "rewards/margins": 0.5746942758560181,
+      "rewards/rejected": -5.09686803817749,
+      "sft_loss": 4.222050666809082,
+      "step": 4700
+    },
+    {
+      "epoch": 2.5181468472988793,
+      "grad_norm": 0.3557932248875748,
+      "learning_rate": 7.638359922812504e-08,
+      "logits/chosen": -0.7023937702178955,
+      "logits/rejected": -0.6598516702651978,
+      "logps/chosen": -4.509191989898682,
+      "logps/rejected": -5.237663269042969,
+      "loss": 0.0502,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.509191989898682,
+      "rewards/margins": 0.7284715175628662,
+      "rewards/rejected": -5.237663269042969,
+      "sft_loss": 4.194537162780762,
+      "step": 4705
+    },
+    {
+      "epoch": 2.520822880080281,
+      "grad_norm": 0.5817691255500771,
+      "learning_rate": 7.555832931406774e-08,
+      "logits/chosen": -0.8006316423416138,
+      "logits/rejected": -0.5774275064468384,
+      "logps/chosen": -4.520382881164551,
+      "logps/rejected": -5.25664758682251,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.520382881164551,
+      "rewards/margins": 0.7362645864486694,
+      "rewards/rejected": -5.25664758682251,
+      "sft_loss": 4.245955467224121,
+      "step": 4710
+    },
+    {
+      "epoch": 2.5234989128616827,
+      "grad_norm": 0.3995963858342414,
+      "learning_rate": 7.47371773349611e-08,
+      "logits/chosen": -0.7040305733680725,
+      "logits/rejected": -0.7341457605361938,
+      "logps/chosen": -4.4279961585998535,
+      "logps/rejected": -5.22509241104126,
+      "loss": 0.0496,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.4279961585998535,
+      "rewards/margins": 0.7970967888832092,
+      "rewards/rejected": -5.22509241104126,
+      "sft_loss": 4.074034690856934,
+      "step": 4715
+    },
+    {
+      "epoch": 2.526174945643084,
+      "grad_norm": 0.3769854449599952,
+      "learning_rate": 7.392015125762496e-08,
+      "logits/chosen": -0.8018277883529663,
+      "logits/rejected": -0.6637119054794312,
+      "logps/chosen": -4.422866344451904,
+      "logps/rejected": -5.083056449890137,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.422866344451904,
+      "rewards/margins": 0.6601905822753906,
+      "rewards/rejected": -5.083056449890137,
+      "sft_loss": 4.099396705627441,
+      "step": 4720
+    },
+    {
+      "epoch": 2.5288509784244857,
+      "grad_norm": 0.4314105030286715,
+      "learning_rate": 7.310725900885018e-08,
+      "logits/chosen": -0.8106725811958313,
+      "logits/rejected": -0.795625627040863,
+      "logps/chosen": -4.441800594329834,
+      "logps/rejected": -4.995269298553467,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.441800594329834,
+      "rewards/margins": 0.5534688234329224,
+      "rewards/rejected": -4.995269298553467,
+      "sft_loss": 4.204402923583984,
+      "step": 4725
+    },
+    {
+      "epoch": 2.5315270112058874,
+      "grad_norm": 0.5168731561480471,
+      "learning_rate": 7.229850847532076e-08,
+      "logits/chosen": -0.7228676080703735,
+      "logits/rejected": -0.5544254183769226,
+      "logps/chosen": -4.531182765960693,
+      "logps/rejected": -5.202092170715332,
+      "loss": 0.051,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.531182765960693,
+      "rewards/margins": 0.6709097623825073,
+      "rewards/rejected": -5.202092170715332,
+      "sft_loss": 4.208251953125,
+      "step": 4730
+    },
+    {
+      "epoch": 2.5342030439872887,
+      "grad_norm": 0.7606664409858177,
+      "learning_rate": 7.149390750353779e-08,
+      "logits/chosen": -0.6411561965942383,
+      "logits/rejected": -0.7863910794258118,
+      "logps/chosen": -4.404947757720947,
+      "logps/rejected": -4.9736504554748535,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.404947757720947,
+      "rewards/margins": 0.5687026381492615,
+      "rewards/rejected": -4.9736504554748535,
+      "sft_loss": 4.057023048400879,
+      "step": 4735
+    },
+    {
+      "epoch": 2.5368790767686904,
+      "grad_norm": 0.39633161239377634,
+      "learning_rate": 7.069346389974374e-08,
+      "logits/chosen": -0.8106769323348999,
+      "logits/rejected": -0.6289903521537781,
+      "logps/chosen": -4.468173027038574,
+      "logps/rejected": -5.0915846824646,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.468173027038574,
+      "rewards/margins": 0.623412013053894,
+      "rewards/rejected": -5.0915846824646,
+      "sft_loss": 4.241403102874756,
+      "step": 4740
+    },
+    {
+      "epoch": 2.539555109550092,
+      "grad_norm": 0.41054122270576154,
+      "learning_rate": 6.989718542984563e-08,
+      "logits/chosen": -0.7652484774589539,
+      "logits/rejected": -0.7496183514595032,
+      "logps/chosen": -4.5117692947387695,
+      "logps/rejected": -5.081242084503174,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.5117692947387695,
+      "rewards/margins": 0.5694732666015625,
+      "rewards/rejected": -5.081242084503174,
+      "sft_loss": 4.277536869049072,
+      "step": 4745
+    },
+    {
+      "epoch": 2.5422311423314934,
+      "grad_norm": 0.6095427658221313,
+      "learning_rate": 6.9105079819341e-08,
+      "logits/chosen": -0.708454430103302,
+      "logits/rejected": -0.4635780453681946,
+      "logps/chosen": -4.330394744873047,
+      "logps/rejected": -5.344074249267578,
+      "loss": 0.0498,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -4.330394744873047,
+      "rewards/margins": 1.0136792659759521,
+      "rewards/rejected": -5.344074249267578,
+      "sft_loss": 4.1580681800842285,
+      "step": 4750
+    },
+    {
+      "epoch": 2.544907175112895,
+      "grad_norm": 0.40158170861414555,
+      "learning_rate": 6.831715475324163e-08,
+      "logits/chosen": -0.8929897546768188,
+      "logits/rejected": -0.6932271718978882,
+      "logps/chosen": -4.430346488952637,
+      "logps/rejected": -5.198345184326172,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.430346488952637,
+      "rewards/margins": 0.7679981589317322,
+      "rewards/rejected": -5.198345184326172,
+      "sft_loss": 4.236446857452393,
+      "step": 4755
+    },
+    {
+      "epoch": 2.547583207894297,
+      "grad_norm": 0.9430362390645314,
+      "learning_rate": 6.753341787600026e-08,
+      "logits/chosen": -0.8296493291854858,
+      "logits/rejected": -0.733299970626831,
+      "logps/chosen": -4.424052715301514,
+      "logps/rejected": -5.159595489501953,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.424052715301514,
+      "rewards/margins": 0.7355419397354126,
+      "rewards/rejected": -5.159595489501953,
+      "sft_loss": 4.191681861877441,
+      "step": 4760
+    },
+    {
+      "epoch": 2.5502592406756985,
+      "grad_norm": 0.5193369946732992,
+      "learning_rate": 6.67538767914353e-08,
+      "logits/chosen": -0.7756398916244507,
+      "logits/rejected": -0.5511414408683777,
+      "logps/chosen": -4.501200199127197,
+      "logps/rejected": -5.170102596282959,
+      "loss": 0.0543,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.501200199127197,
+      "rewards/margins": 0.668903112411499,
+      "rewards/rejected": -5.170102596282959,
+      "sft_loss": 4.344412326812744,
+      "step": 4765
+    },
+    {
+      "epoch": 2.5529352734571,
+      "grad_norm": 0.625624192281797,
+      "learning_rate": 6.597853906265793e-08,
+      "logits/chosen": -0.7071702480316162,
+      "logits/rejected": -0.5840771794319153,
+      "logps/chosen": -4.405869483947754,
+      "logps/rejected": -5.307399749755859,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.405869483947754,
+      "rewards/margins": 0.901530385017395,
+      "rewards/rejected": -5.307399749755859,
+      "sft_loss": 4.106754779815674,
+      "step": 4770
+    },
+    {
+      "epoch": 2.5556113062385015,
+      "grad_norm": 0.44989926116378653,
+      "learning_rate": 6.5207412211998e-08,
+      "logits/chosen": -0.6850845217704773,
+      "logits/rejected": -0.5953121185302734,
+      "logps/chosen": -4.426508903503418,
+      "logps/rejected": -5.078802108764648,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.426508903503418,
+      "rewards/margins": 0.6522935628890991,
+      "rewards/rejected": -5.078802108764648,
+      "sft_loss": 4.111783027648926,
+      "step": 4775
+    },
+    {
+      "epoch": 2.558287339019903,
+      "grad_norm": 0.3869486131303834,
+      "learning_rate": 6.444050372093186e-08,
+      "logits/chosen": -0.7650526165962219,
+      "logits/rejected": -0.6805169582366943,
+      "logps/chosen": -4.382111549377441,
+      "logps/rejected": -5.045238971710205,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.382111549377441,
+      "rewards/margins": 0.6631268858909607,
+      "rewards/rejected": -5.045238971710205,
+      "sft_loss": 4.1749267578125,
+      "step": 4780
+    },
+    {
+      "epoch": 2.5609633718013045,
+      "grad_norm": 0.5142574862060383,
+      "learning_rate": 6.367782103000873e-08,
+      "logits/chosen": -0.7648038268089294,
+      "logits/rejected": -0.7223659753799438,
+      "logps/chosen": -4.393990993499756,
+      "logps/rejected": -4.955014228820801,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.393990993499756,
+      "rewards/margins": 0.5610231161117554,
+      "rewards/rejected": -4.955014228820801,
+      "sft_loss": 4.111966133117676,
+      "step": 4785
+    },
+    {
+      "epoch": 2.5636394045827062,
+      "grad_norm": 0.4258157669046418,
+      "learning_rate": 6.29193715387798e-08,
+      "logits/chosen": -0.8260219693183899,
+      "logits/rejected": -0.7080743312835693,
+      "logps/chosen": -4.401946544647217,
+      "logps/rejected": -5.105975151062012,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.401946544647217,
+      "rewards/margins": 0.7040285468101501,
+      "rewards/rejected": -5.105975151062012,
+      "sft_loss": 4.163133144378662,
+      "step": 4790
+    },
+    {
+      "epoch": 2.566315437364108,
+      "grad_norm": 0.6189933185149098,
+      "learning_rate": 6.216516260572502e-08,
+      "logits/chosen": -0.7578010559082031,
+      "logits/rejected": -0.657882809638977,
+      "logps/chosen": -4.521510124206543,
+      "logps/rejected": -5.196859359741211,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.521510124206543,
+      "rewards/margins": 0.6753486394882202,
+      "rewards/rejected": -5.196859359741211,
+      "sft_loss": 4.304436683654785,
+      "step": 4795
+    },
+    {
+      "epoch": 2.568991470145509,
+      "grad_norm": 0.567926027513947,
+      "learning_rate": 6.141520154818297e-08,
+      "logits/chosen": -0.7929534912109375,
+      "logits/rejected": -0.6780120134353638,
+      "logps/chosen": -4.525482654571533,
+      "logps/rejected": -5.13308048248291,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.525482654571533,
+      "rewards/margins": 0.6075973510742188,
+      "rewards/rejected": -5.13308048248291,
+      "sft_loss": 4.307214736938477,
+      "step": 4800
+    },
+    {
+      "epoch": 2.568991470145509,
+      "eval_logits/chosen": -0.2741285264492035,
+      "eval_logits/rejected": -0.18904297053813934,
+      "eval_logps/chosen": -4.448305130004883,
+      "eval_logps/rejected": -5.142607688903809,
+      "eval_loss": 0.05042795091867447,
+      "eval_rewards/accuracies": 0.6832343935966492,
+      "eval_rewards/chosen": -4.448305130004883,
+      "eval_rewards/margins": 0.6943021416664124,
+      "eval_rewards/rejected": -5.142607688903809,
+      "eval_runtime": 43.661,
+      "eval_samples_per_second": 30.805,
+      "eval_sft_loss": 4.069952011108398,
+      "eval_steps_per_second": 7.719,
+      "step": 4800
+    },
+    {
+      "epoch": 2.571667502926911,
+      "grad_norm": 0.36636097203702034,
+      "learning_rate": 6.066949564227897e-08,
+      "logits/chosen": -0.7946752905845642,
+      "logits/rejected": -0.716626763343811,
+      "logps/chosen": -4.3803510665893555,
+      "logps/rejected": -5.064523696899414,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.3803510665893555,
+      "rewards/margins": 0.6841726303100586,
+      "rewards/rejected": -5.064523696899414,
+      "sft_loss": 4.091052055358887,
+      "step": 4805
+    },
+    {
+      "epoch": 2.574343535708312,
+      "grad_norm": 0.7677640156297736,
+      "learning_rate": 5.992805212285523e-08,
+      "logits/chosen": -0.7754315137863159,
+      "logits/rejected": -0.7091799378395081,
+      "logps/chosen": -4.335581302642822,
+      "logps/rejected": -5.124228477478027,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.335581302642822,
+      "rewards/margins": 0.7886467576026917,
+      "rewards/rejected": -5.124228477478027,
+      "sft_loss": 4.099038600921631,
+      "step": 4810
+    },
+    {
+      "epoch": 2.577019568489714,
+      "grad_norm": 0.5748646729931978,
+      "learning_rate": 5.9190878183399684e-08,
+      "logits/chosen": -0.7570444345474243,
+      "logits/rejected": -0.613994300365448,
+      "logps/chosen": -4.440479755401611,
+      "logps/rejected": -5.164405822753906,
+      "loss": 0.052,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.440479755401611,
+      "rewards/margins": 0.7239259481430054,
+      "rewards/rejected": -5.164405822753906,
+      "sft_loss": 4.215497016906738,
+      "step": 4815
+    },
+    {
+      "epoch": 2.5796956012711156,
+      "grad_norm": 0.4191146750687918,
+      "learning_rate": 5.845798097597748e-08,
+      "logits/chosen": -0.7721574306488037,
+      "logits/rejected": -0.6697301268577576,
+      "logps/chosen": -4.579713821411133,
+      "logps/rejected": -5.104306221008301,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.579713821411133,
+      "rewards/margins": 0.5245919227600098,
+      "rewards/rejected": -5.104306221008301,
+      "sft_loss": 4.305607795715332,
+      "step": 4820
+    },
+    {
+      "epoch": 2.5823716340525174,
+      "grad_norm": 0.45022059402292774,
+      "learning_rate": 5.772936761116026e-08,
+      "logits/chosen": -0.7701061964035034,
+      "logits/rejected": -0.6287974715232849,
+      "logps/chosen": -4.400467395782471,
+      "logps/rejected": -5.098750114440918,
+      "loss": 0.051,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.400467395782471,
+      "rewards/margins": 0.6982828378677368,
+      "rewards/rejected": -5.098750114440918,
+      "sft_loss": 4.131747245788574,
+      "step": 4825
+    },
+    {
+      "epoch": 2.5850476668339186,
+      "grad_norm": 0.4799590961567342,
+      "learning_rate": 5.700504515795829e-08,
+      "logits/chosen": -0.8211865425109863,
+      "logits/rejected": -0.6615251302719116,
+      "logps/chosen": -4.437475681304932,
+      "logps/rejected": -5.135467529296875,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.437475681304932,
+      "rewards/margins": 0.6979917287826538,
+      "rewards/rejected": -5.135467529296875,
+      "sft_loss": 4.257898330688477,
+      "step": 4830
+    },
+    {
+      "epoch": 2.5877236996153203,
+      "grad_norm": 0.42527193100918037,
+      "learning_rate": 5.628502064375101e-08,
+      "logits/chosen": -0.8720036745071411,
+      "logits/rejected": -0.6801129579544067,
+      "logps/chosen": -4.475311279296875,
+      "logps/rejected": -5.252869129180908,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.475311279296875,
+      "rewards/margins": 0.777558445930481,
+      "rewards/rejected": -5.252869129180908,
+      "sft_loss": 4.221250057220459,
+      "step": 4835
+    },
+    {
+      "epoch": 2.5903997323967216,
+      "grad_norm": 0.5115033248703027,
+      "learning_rate": 5.55693010542197e-08,
+      "logits/chosen": -0.7869556546211243,
+      "logits/rejected": -0.5591322779655457,
+      "logps/chosen": -4.349687099456787,
+      "logps/rejected": -5.18341064453125,
+      "loss": 0.0485,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.349687099456787,
+      "rewards/margins": 0.8337236642837524,
+      "rewards/rejected": -5.18341064453125,
+      "sft_loss": 3.9441981315612793,
+      "step": 4840
+    },
+    {
+      "epoch": 2.5930757651781233,
+      "grad_norm": 0.7067733700435774,
+      "learning_rate": 5.485789333327856e-08,
+      "logits/chosen": -0.7598763108253479,
+      "logits/rejected": -0.7221094369888306,
+      "logps/chosen": -4.49146842956543,
+      "logps/rejected": -5.119671821594238,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.49146842956543,
+      "rewards/margins": 0.628203809261322,
+      "rewards/rejected": -5.119671821594238,
+      "sft_loss": 4.2429094314575195,
+      "step": 4845
+    },
+    {
+      "epoch": 2.595751797959525,
+      "grad_norm": 0.45260597716363027,
+      "learning_rate": 5.4150804383008675e-08,
+      "logits/chosen": -0.910417914390564,
+      "logits/rejected": -0.7389578819274902,
+      "logps/chosen": -4.464565277099609,
+      "logps/rejected": -5.181889533996582,
+      "loss": 0.053,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.464565277099609,
+      "rewards/margins": 0.7173237204551697,
+      "rewards/rejected": -5.181889533996582,
+      "sft_loss": 4.224325656890869,
+      "step": 4850
+    },
+    {
+      "epoch": 2.5984278307409268,
+      "grad_norm": 0.5973972322326072,
+      "learning_rate": 5.344804106359002e-08,
+      "logits/chosen": -0.6693606376647949,
+      "logits/rejected": -0.5169059634208679,
+      "logps/chosen": -4.443115711212158,
+      "logps/rejected": -5.097043037414551,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.443115711212158,
+      "rewards/margins": 0.653927206993103,
+      "rewards/rejected": -5.097043037414551,
+      "sft_loss": 4.195808410644531,
+      "step": 4855
+    },
+    {
+      "epoch": 2.601103863522328,
+      "grad_norm": 0.5755470171232112,
+      "learning_rate": 5.274961019323559e-08,
+      "logits/chosen": -0.7551354169845581,
+      "logits/rejected": -0.6369063854217529,
+      "logps/chosen": -4.399575710296631,
+      "logps/rejected": -5.089932918548584,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.399575710296631,
+      "rewards/margins": 0.6903573870658875,
+      "rewards/rejected": -5.089932918548584,
+      "sft_loss": 4.221864700317383,
+      "step": 4860
+    },
+    {
+      "epoch": 2.6037798963037297,
+      "grad_norm": 0.4840213634050733,
+      "learning_rate": 5.205551854812451e-08,
+      "logits/chosen": -0.8901827931404114,
+      "logits/rejected": -0.8028414845466614,
+      "logps/chosen": -4.3761115074157715,
+      "logps/rejected": -5.150932312011719,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.3761115074157715,
+      "rewards/margins": 0.7748211026191711,
+      "rewards/rejected": -5.150932312011719,
+      "sft_loss": 4.163114070892334,
+      "step": 4865
+    },
+    {
+      "epoch": 2.606455929085131,
+      "grad_norm": 0.8958131855189024,
+      "learning_rate": 5.1365772862337177e-08,
+      "logits/chosen": -0.6926102638244629,
+      "logits/rejected": -0.6045705080032349,
+      "logps/chosen": -4.361238956451416,
+      "logps/rejected": -5.185271263122559,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.361238956451416,
+      "rewards/margins": 0.8240326642990112,
+      "rewards/rejected": -5.185271263122559,
+      "sft_loss": 4.081574440002441,
+      "step": 4870
+    },
+    {
+      "epoch": 2.6091319618665327,
+      "grad_norm": 0.6988608914200577,
+      "learning_rate": 5.068037982778905e-08,
+      "logits/chosen": -0.6733156442642212,
+      "logits/rejected": -0.5771912336349487,
+      "logps/chosen": -4.478516101837158,
+      "logps/rejected": -5.19637393951416,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.478516101837158,
+      "rewards/margins": 0.7178576588630676,
+      "rewards/rejected": -5.19637393951416,
+      "sft_loss": 4.19520902633667,
+      "step": 4875
+    },
+    {
+      "epoch": 2.6118079946479344,
+      "grad_norm": 0.5200460676604345,
+      "learning_rate": 4.999934609416656e-08,
+      "logits/chosen": -0.7352452278137207,
+      "logits/rejected": -0.6223322153091431,
+      "logps/chosen": -4.333428859710693,
+      "logps/rejected": -5.200177192687988,
+      "loss": 0.051,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.333428859710693,
+      "rewards/margins": 0.8667477369308472,
+      "rewards/rejected": -5.200177192687988,
+      "sft_loss": 4.1180853843688965,
+      "step": 4880
+    },
+    {
+      "epoch": 2.614484027429336,
+      "grad_norm": 0.5114132721276469,
+      "learning_rate": 4.932267826886183e-08,
+      "logits/chosen": -0.6905040144920349,
+      "logits/rejected": -0.6525358557701111,
+      "logps/chosen": -4.4714531898498535,
+      "logps/rejected": -5.135130882263184,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.4714531898498535,
+      "rewards/margins": 0.6636782884597778,
+      "rewards/rejected": -5.135130882263184,
+      "sft_loss": 4.293010711669922,
+      "step": 4885
+    },
+    {
+      "epoch": 2.6171600602107374,
+      "grad_norm": 0.3541675492702616,
+      "learning_rate": 4.8650382916909206e-08,
+      "logits/chosen": -0.8800719976425171,
+      "logits/rejected": -0.6866661310195923,
+      "logps/chosen": -4.409409523010254,
+      "logps/rejected": -5.073835849761963,
+      "loss": 0.052,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.409409523010254,
+      "rewards/margins": 0.6644265055656433,
+      "rewards/rejected": -5.073835849761963,
+      "sft_loss": 4.160528182983398,
+      "step": 4890
+    },
+    {
+      "epoch": 2.619836092992139,
+      "grad_norm": 0.5684592362716069,
+      "learning_rate": 4.7982466560920976e-08,
+      "logits/chosen": -0.7659167051315308,
+      "logits/rejected": -0.7323800921440125,
+      "logps/chosen": -4.4443440437316895,
+      "logps/rejected": -5.052102088928223,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.4443440437316895,
+      "rewards/margins": 0.6077579259872437,
+      "rewards/rejected": -5.052102088928223,
+      "sft_loss": 4.219048976898193,
+      "step": 4895
+    },
+    {
+      "epoch": 2.622512125773541,
+      "grad_norm": 0.6689392592042074,
+      "learning_rate": 4.7318935681024685e-08,
+      "logits/chosen": -0.7107598185539246,
+      "logits/rejected": -0.5510147213935852,
+      "logps/chosen": -4.424412250518799,
+      "logps/rejected": -5.242027282714844,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.424412250518799,
+      "rewards/margins": 0.8176156282424927,
+      "rewards/rejected": -5.242027282714844,
+      "sft_loss": 4.165490627288818,
+      "step": 4900
+    },
+    {
+      "epoch": 2.625188158554942,
+      "grad_norm": 0.5722256802272633,
+      "learning_rate": 4.6659796714799745e-08,
+      "logits/chosen": -0.7005605697631836,
+      "logits/rejected": -0.576447606086731,
+      "logps/chosen": -4.384562969207764,
+      "logps/rejected": -5.328797817230225,
+      "loss": 0.0493,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.384562969207764,
+      "rewards/margins": 0.9442348480224609,
+      "rewards/rejected": -5.328797817230225,
+      "sft_loss": 4.149122714996338,
+      "step": 4905
+    },
+    {
+      "epoch": 2.627864191336344,
+      "grad_norm": 0.5219836508629162,
+      "learning_rate": 4.60050560572155e-08,
+      "logits/chosen": -0.7817445993423462,
+      "logits/rejected": -0.8545292615890503,
+      "logps/chosen": -4.5263991355896,
+      "logps/rejected": -5.273439884185791,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.5263991355896,
+      "rewards/margins": 0.7470411658287048,
+      "rewards/rejected": -5.273439884185791,
+      "sft_loss": 4.291499137878418,
+      "step": 4910
+    },
+    {
+      "epoch": 2.6305402241177456,
+      "grad_norm": 0.38646571365518173,
+      "learning_rate": 4.535472006056834e-08,
+      "logits/chosen": -0.7773014903068542,
+      "logits/rejected": -0.585507333278656,
+      "logps/chosen": -4.419951438903809,
+      "logps/rejected": -5.040675640106201,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.419951438903809,
+      "rewards/margins": 0.6207249164581299,
+      "rewards/rejected": -5.040675640106201,
+      "sft_loss": 4.128361225128174,
+      "step": 4915
+    },
+    {
+      "epoch": 2.6332162568991473,
+      "grad_norm": 0.4415088424078639,
+      "learning_rate": 4.470879503442132e-08,
+      "logits/chosen": -0.7938684225082397,
+      "logits/rejected": -0.6925408244132996,
+      "logps/chosen": -4.400835990905762,
+      "logps/rejected": -5.0848212242126465,
+      "loss": 0.051,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.400835990905762,
+      "rewards/margins": 0.6839861869812012,
+      "rewards/rejected": -5.0848212242126465,
+      "sft_loss": 4.152151584625244,
+      "step": 4920
+    },
+    {
+      "epoch": 2.6358922896805486,
+      "grad_norm": 0.5438520820103692,
+      "learning_rate": 4.406728724554154e-08,
+      "logits/chosen": -0.9053106307983398,
+      "logits/rejected": -0.6270970106124878,
+      "logps/chosen": -4.350544452667236,
+      "logps/rejected": -5.182711601257324,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.350544452667236,
+      "rewards/margins": 0.8321673274040222,
+      "rewards/rejected": -5.182711601257324,
+      "sft_loss": 4.118407249450684,
+      "step": 4925
+    },
+    {
+      "epoch": 2.6385683224619503,
+      "grad_norm": 0.40830510209473897,
+      "learning_rate": 4.3430202917840664e-08,
+      "logits/chosen": -0.7182999849319458,
+      "logits/rejected": -0.5021204352378845,
+      "logps/chosen": -4.545349597930908,
+      "logps/rejected": -5.3235087394714355,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.545349597930908,
+      "rewards/margins": 0.7781594395637512,
+      "rewards/rejected": -5.3235087394714355,
+      "sft_loss": 4.261865615844727,
+      "step": 4930
+    },
+    {
+      "epoch": 2.6412443552433515,
+      "grad_norm": 0.46964279049381513,
+      "learning_rate": 4.279754823231346e-08,
+      "logits/chosen": -0.776440441608429,
+      "logits/rejected": -0.5842413902282715,
+      "logps/chosen": -4.306726932525635,
+      "logps/rejected": -5.086239337921143,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.306726932525635,
+      "rewards/margins": 0.7795121073722839,
+      "rewards/rejected": -5.086239337921143,
+      "sft_loss": 3.9869303703308105,
+      "step": 4935
+    },
+    {
+      "epoch": 2.6439203880247533,
+      "grad_norm": 0.49471229779296944,
+      "learning_rate": 4.216932932697859e-08,
+      "logits/chosen": -0.832381546497345,
+      "logits/rejected": -0.7716125249862671,
+      "logps/chosen": -4.284350395202637,
+      "logps/rejected": -4.896607398986816,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.284350395202637,
+      "rewards/margins": 0.6122564077377319,
+      "rewards/rejected": -4.896607398986816,
+      "sft_loss": 4.027619361877441,
+      "step": 4940
+    },
+    {
+      "epoch": 2.646596420806155,
+      "grad_norm": 0.5063133009359136,
+      "learning_rate": 4.154555229681844e-08,
+      "logits/chosen": -0.7936744093894958,
+      "logits/rejected": -0.5885189175605774,
+      "logps/chosen": -4.427206993103027,
+      "logps/rejected": -5.143578052520752,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.427206993103027,
+      "rewards/margins": 0.7163704633712769,
+      "rewards/rejected": -5.143578052520752,
+      "sft_loss": 4.04074764251709,
+      "step": 4945
+    },
+    {
+      "epoch": 2.6492724535875567,
+      "grad_norm": 0.5395838576200975,
+      "learning_rate": 4.092622319372069e-08,
+      "logits/chosen": -0.7596436142921448,
+      "logits/rejected": -0.5754319429397583,
+      "logps/chosen": -4.363346099853516,
+      "logps/rejected": -5.0439653396606445,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.363346099853516,
+      "rewards/margins": 0.6806186437606812,
+      "rewards/rejected": -5.0439653396606445,
+      "sft_loss": 4.0527753829956055,
+      "step": 4950
+    },
+    {
+      "epoch": 2.651948486368958,
+      "grad_norm": 0.4273690281432811,
+      "learning_rate": 4.031134802641889e-08,
+      "logits/chosen": -0.754179060459137,
+      "logits/rejected": -0.7639588713645935,
+      "logps/chosen": -4.647191524505615,
+      "logps/rejected": -5.165177345275879,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.647191524505615,
+      "rewards/margins": 0.5179857015609741,
+      "rewards/rejected": -5.165177345275879,
+      "sft_loss": 4.326084136962891,
+      "step": 4955
+    },
+    {
+      "epoch": 2.6546245191503597,
+      "grad_norm": 0.4601641742802231,
+      "learning_rate": 3.970093276043468e-08,
+      "logits/chosen": -0.6982657313346863,
+      "logits/rejected": -0.6274687051773071,
+      "logps/chosen": -4.353320121765137,
+      "logps/rejected": -5.106142997741699,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.353320121765137,
+      "rewards/margins": 0.7528237700462341,
+      "rewards/rejected": -5.106142997741699,
+      "sft_loss": 4.145398139953613,
+      "step": 4960
+    },
+    {
+      "epoch": 2.657300551931761,
+      "grad_norm": 0.4035041863470644,
+      "learning_rate": 3.9094983318019584e-08,
+      "logits/chosen": -0.8743463754653931,
+      "logits/rejected": -0.7296475172042847,
+      "logps/chosen": -4.384496212005615,
+      "logps/rejected": -5.0877203941345215,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.384496212005615,
+      "rewards/margins": 0.7032240629196167,
+      "rewards/rejected": -5.0877203941345215,
+      "sft_loss": 4.198531150817871,
+      "step": 4965
+    },
+    {
+      "epoch": 2.6599765847131627,
+      "grad_norm": 0.4512012847530197,
+      "learning_rate": 3.849350557809789e-08,
+      "logits/chosen": -0.6640986204147339,
+      "logits/rejected": -0.653388500213623,
+      "logps/chosen": -4.503409385681152,
+      "logps/rejected": -5.102181434631348,
+      "loss": 0.0502,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.503409385681152,
+      "rewards/margins": 0.5987719297409058,
+      "rewards/rejected": -5.102181434631348,
+      "sft_loss": 4.060650825500488,
+      "step": 4970
+    },
+    {
+      "epoch": 2.6626526174945644,
+      "grad_norm": 0.6901863489404572,
+      "learning_rate": 3.789650537620903e-08,
+      "logits/chosen": -0.688258945941925,
+      "logits/rejected": -0.6832031011581421,
+      "logps/chosen": -4.540500164031982,
+      "logps/rejected": -5.2547407150268555,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.540500164031982,
+      "rewards/margins": 0.7142406105995178,
+      "rewards/rejected": -5.2547407150268555,
+      "sft_loss": 4.2646074295043945,
+      "step": 4975
+    },
+    {
+      "epoch": 2.665328650275966,
+      "grad_norm": 0.42588906837966245,
+      "learning_rate": 3.730398850445182e-08,
+      "logits/chosen": -0.6220318078994751,
+      "logits/rejected": -0.6261580586433411,
+      "logps/chosen": -4.496462821960449,
+      "logps/rejected": -5.189952373504639,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.496462821960449,
+      "rewards/margins": 0.6934901475906372,
+      "rewards/rejected": -5.189952373504639,
+      "sft_loss": 4.15579080581665,
+      "step": 4980
+    },
+    {
+      "epoch": 2.6680046830573674,
+      "grad_norm": 0.4940453819815409,
+      "learning_rate": 3.671596071142735e-08,
+      "logits/chosen": -0.7255562543869019,
+      "logits/rejected": -0.5626224875450134,
+      "logps/chosen": -4.4464030265808105,
+      "logps/rejected": -5.3239946365356445,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.4464030265808105,
+      "rewards/margins": 0.8775907754898071,
+      "rewards/rejected": -5.3239946365356445,
+      "sft_loss": 4.130679130554199,
+      "step": 4985
+    },
+    {
+      "epoch": 2.670680715838769,
+      "grad_norm": 0.533231803665976,
+      "learning_rate": 3.6132427702183996e-08,
+      "logits/chosen": -0.898454487323761,
+      "logits/rejected": -0.7337983846664429,
+      "logps/chosen": -4.380088806152344,
+      "logps/rejected": -5.301820278167725,
+      "loss": 0.0502,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.380088806152344,
+      "rewards/margins": 0.9217315912246704,
+      "rewards/rejected": -5.301820278167725,
+      "sft_loss": 4.153786659240723,
+      "step": 4990
+    },
+    {
+      "epoch": 2.6733567486201704,
+      "grad_norm": 0.4560585718830667,
+      "learning_rate": 3.555339513816147e-08,
+      "logits/chosen": -0.8258197903633118,
+      "logits/rejected": -0.8702704310417175,
+      "logps/chosen": -4.535338401794434,
+      "logps/rejected": -5.168300151824951,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.535338401794434,
+      "rewards/margins": 0.6329620480537415,
+      "rewards/rejected": -5.168300151824951,
+      "sft_loss": 4.283738136291504,
+      "step": 4995
+    },
+    {
+      "epoch": 2.676032781401572,
+      "grad_norm": 0.4913083081311584,
+      "learning_rate": 3.497886863713639e-08,
+      "logits/chosen": -0.7957427501678467,
+      "logits/rejected": -0.7848941087722778,
+      "logps/chosen": -4.596715927124023,
+      "logps/rejected": -5.212932586669922,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.596715927124023,
+      "rewards/margins": 0.6162165403366089,
+      "rewards/rejected": -5.212932586669922,
+      "sft_loss": 4.317381381988525,
+      "step": 5000
+    },
+    {
+      "epoch": 2.678708814182974,
+      "grad_norm": 0.5146051345946518,
+      "learning_rate": 3.440885377316721e-08,
+      "logits/chosen": -0.7044495344161987,
+      "logits/rejected": -0.6470869779586792,
+      "logps/chosen": -4.40933895111084,
+      "logps/rejected": -4.96150016784668,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.40933895111084,
+      "rewards/margins": 0.5521610975265503,
+      "rewards/rejected": -4.96150016784668,
+      "sft_loss": 4.148724555969238,
+      "step": 5005
+    },
+    {
+      "epoch": 2.6813848469643755,
+      "grad_norm": 0.5127153318494577,
+      "learning_rate": 3.384335607654082e-08,
+      "logits/chosen": -0.662913978099823,
+      "logits/rejected": -0.6405649185180664,
+      "logps/chosen": -4.492884159088135,
+      "logps/rejected": -5.25106143951416,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.492884159088135,
+      "rewards/margins": 0.7581772804260254,
+      "rewards/rejected": -5.25106143951416,
+      "sft_loss": 4.1412672996521,
+      "step": 5010
+    },
+    {
+      "epoch": 2.684060879745777,
+      "grad_norm": 0.49440876947556955,
+      "learning_rate": 3.328238103371811e-08,
+      "logits/chosen": -0.755716860294342,
+      "logits/rejected": -0.7262305021286011,
+      "logps/chosen": -4.506894111633301,
+      "logps/rejected": -5.315282344818115,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.506894111633301,
+      "rewards/margins": 0.8083890676498413,
+      "rewards/rejected": -5.315282344818115,
+      "sft_loss": 4.149899482727051,
+      "step": 5015
+    },
+    {
+      "epoch": 2.6867369125271785,
+      "grad_norm": 0.5390351293879576,
+      "learning_rate": 3.272593408728169e-08,
+      "logits/chosen": -0.8326853513717651,
+      "logits/rejected": -0.5620445013046265,
+      "logps/chosen": -4.3972320556640625,
+      "logps/rejected": -5.159958839416504,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.3972320556640625,
+      "rewards/margins": 0.7627268433570862,
+      "rewards/rejected": -5.159958839416504,
+      "sft_loss": 4.1451311111450195,
+      "step": 5020
+    },
+    {
+      "epoch": 2.6894129453085798,
+      "grad_norm": 0.4278067734510489,
+      "learning_rate": 3.217402063588204e-08,
+      "logits/chosen": -0.8494710922241211,
+      "logits/rejected": -0.6724889874458313,
+      "logps/chosen": -4.459986686706543,
+      "logps/rejected": -5.137322902679443,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.459986686706543,
+      "rewards/margins": 0.6773372888565063,
+      "rewards/rejected": -5.137322902679443,
+      "sft_loss": 4.2519850730896,
+      "step": 5025
+    },
+    {
+      "epoch": 2.6920889780899815,
+      "grad_norm": 0.5935683504662128,
+      "learning_rate": 3.162664603418608e-08,
+      "logits/chosen": -0.7728666067123413,
+      "logits/rejected": -0.7169617414474487,
+      "logps/chosen": -4.379161357879639,
+      "logps/rejected": -5.187068939208984,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.379161357879639,
+      "rewards/margins": 0.807907223701477,
+      "rewards/rejected": -5.187068939208984,
+      "sft_loss": 4.1201324462890625,
+      "step": 5030
+    },
+    {
+      "epoch": 2.694765010871383,
+      "grad_norm": 0.38628884775927563,
+      "learning_rate": 3.1083815592824416e-08,
+      "logits/chosen": -0.752131462097168,
+      "logits/rejected": -0.6836836934089661,
+      "logps/chosen": -4.516790866851807,
+      "logps/rejected": -5.249283790588379,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.516790866851807,
+      "rewards/margins": 0.7324928045272827,
+      "rewards/rejected": -5.249283790588379,
+      "sft_loss": 4.276049613952637,
+      "step": 5035
+    },
+    {
+      "epoch": 2.697441043652785,
+      "grad_norm": 0.5871576493438839,
+      "learning_rate": 3.054553457834053e-08,
+      "logits/chosen": -0.6166585087776184,
+      "logits/rejected": -0.6771060824394226,
+      "logps/chosen": -4.626035213470459,
+      "logps/rejected": -5.179619789123535,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.626035213470459,
+      "rewards/margins": 0.5535842180252075,
+      "rewards/rejected": -5.179619789123535,
+      "sft_loss": 4.302709102630615,
+      "step": 5040
+    },
+    {
+      "epoch": 2.700117076434186,
+      "grad_norm": 0.6160064556957795,
+      "learning_rate": 3.0011808213139036e-08,
+      "logits/chosen": -0.6177735328674316,
+      "logits/rejected": -0.6710564494132996,
+      "logps/chosen": -4.500868320465088,
+      "logps/rejected": -5.045173168182373,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.500868320465088,
+      "rewards/margins": 0.5443046689033508,
+      "rewards/rejected": -5.045173168182373,
+      "sft_loss": 4.213364124298096,
+      "step": 5045
+    },
+    {
+      "epoch": 2.702793109215588,
+      "grad_norm": 0.6859916514482486,
+      "learning_rate": 2.948264167543568e-08,
+      "logits/chosen": -0.7522085905075073,
+      "logits/rejected": -0.6835848093032837,
+      "logps/chosen": -4.500675678253174,
+      "logps/rejected": -5.0596089363098145,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.500675678253174,
+      "rewards/margins": 0.5589330792427063,
+      "rewards/rejected": -5.0596089363098145,
+      "sft_loss": 4.171204566955566,
+      "step": 5050
+    },
+    {
+      "epoch": 2.7054691419969896,
+      "grad_norm": 0.4799906826336578,
+      "learning_rate": 2.8958040099206216e-08,
+      "logits/chosen": -0.8550014495849609,
+      "logits/rejected": -0.7934740781784058,
+      "logps/chosen": -4.4887800216674805,
+      "logps/rejected": -5.191119194030762,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.4887800216674805,
+      "rewards/margins": 0.7023388743400574,
+      "rewards/rejected": -5.191119194030762,
+      "sft_loss": 4.201326847076416,
+      "step": 5055
+    },
+    {
+      "epoch": 2.708145174778391,
+      "grad_norm": 0.7859840677597958,
+      "learning_rate": 2.843800857413775e-08,
+      "logits/chosen": -0.6860078573226929,
+      "logits/rejected": -0.654365062713623,
+      "logps/chosen": -4.445427417755127,
+      "logps/rejected": -4.997714996337891,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.445427417755127,
+      "rewards/margins": 0.5522874593734741,
+      "rewards/rejected": -4.997714996337891,
+      "sft_loss": 4.158383846282959,
+      "step": 5060
+    },
+    {
+      "epoch": 2.7108212075597926,
+      "grad_norm": 0.4526236800027281,
+      "learning_rate": 2.7922552145578203e-08,
+      "logits/chosen": -0.7144232988357544,
+      "logits/rejected": -0.45682835578918457,
+      "logps/chosen": -4.429794788360596,
+      "logps/rejected": -5.128900051116943,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.429794788360596,
+      "rewards/margins": 0.6991047263145447,
+      "rewards/rejected": -5.128900051116943,
+      "sft_loss": 4.1685028076171875,
+      "step": 5065
+    },
+    {
+      "epoch": 2.7134972403411943,
+      "grad_norm": 0.4394735274325847,
+      "learning_rate": 2.7411675814488277e-08,
+      "logits/chosen": -0.697012722492218,
+      "logits/rejected": -0.56499183177948,
+      "logps/chosen": -4.419262886047363,
+      "logps/rejected": -5.056243896484375,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.419262886047363,
+      "rewards/margins": 0.6369813680648804,
+      "rewards/rejected": -5.056243896484375,
+      "sft_loss": 4.206353664398193,
+      "step": 5070
+    },
+    {
+      "epoch": 2.7161732731225956,
+      "grad_norm": 0.5891818486629351,
+      "learning_rate": 2.690538453739216e-08,
+      "logits/chosen": -0.7441337704658508,
+      "logits/rejected": -0.706308901309967,
+      "logps/chosen": -4.351699352264404,
+      "logps/rejected": -4.927661895751953,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.351699352264404,
+      "rewards/margins": 0.5759629011154175,
+      "rewards/rejected": -4.927661895751953,
+      "sft_loss": 4.168587684631348,
+      "step": 5075
+    },
+    {
+      "epoch": 2.7188493059039973,
+      "grad_norm": 0.43729963617249973,
+      "learning_rate": 2.6403683226330298e-08,
+      "logits/chosen": -0.8585704565048218,
+      "logits/rejected": -0.6886411309242249,
+      "logps/chosen": -4.4380598068237305,
+      "logps/rejected": -5.095976829528809,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.4380598068237305,
+      "rewards/margins": 0.6579168438911438,
+      "rewards/rejected": -5.095976829528809,
+      "sft_loss": 4.219568729400635,
+      "step": 5080
+    },
+    {
+      "epoch": 2.721525338685399,
+      "grad_norm": 0.45646069193158206,
+      "learning_rate": 2.5906576748810804e-08,
+      "logits/chosen": -0.8393117189407349,
+      "logits/rejected": -0.7262551784515381,
+      "logps/chosen": -4.536147117614746,
+      "logps/rejected": -5.36475944519043,
+      "loss": 0.051,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.536147117614746,
+      "rewards/margins": 0.8286125063896179,
+      "rewards/rejected": -5.36475944519043,
+      "sft_loss": 4.3527655601501465,
+      "step": 5085
+    },
+    {
+      "epoch": 2.7242013714668003,
+      "grad_norm": 0.40349095053106443,
+      "learning_rate": 2.5414069927763016e-08,
+      "logits/chosen": -0.9187151789665222,
+      "logits/rejected": -0.7296867370605469,
+      "logps/chosen": -4.362442970275879,
+      "logps/rejected": -5.214807033538818,
+      "loss": 0.0498,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -4.362442970275879,
+      "rewards/margins": 0.8523637056350708,
+      "rewards/rejected": -5.214807033538818,
+      "sft_loss": 4.102611064910889,
+      "step": 5090
+    },
+    {
+      "epoch": 2.726877404248202,
+      "grad_norm": 0.31381315759075395,
+      "learning_rate": 2.4926167541490185e-08,
+      "logits/chosen": -0.941425621509552,
+      "logits/rejected": -0.7067974805831909,
+      "logps/chosen": -4.382145404815674,
+      "logps/rejected": -5.232102870941162,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.382145404815674,
+      "rewards/margins": 0.849958062171936,
+      "rewards/rejected": -5.232102870941162,
+      "sft_loss": 4.168827056884766,
+      "step": 5095
+    },
+    {
+      "epoch": 2.7295534370296037,
+      "grad_norm": 0.4666743335275937,
+      "learning_rate": 2.4442874323623574e-08,
+      "logits/chosen": -0.7078942060470581,
+      "logits/rejected": -0.5640963912010193,
+      "logps/chosen": -4.456545829772949,
+      "logps/rejected": -5.237743377685547,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.456545829772949,
+      "rewards/margins": 0.7811979055404663,
+      "rewards/rejected": -5.237743377685547,
+      "sft_loss": 4.171876430511475,
+      "step": 5100
+    },
+    {
+      "epoch": 2.7322294698110055,
+      "grad_norm": 0.5009611946781228,
+      "learning_rate": 2.396419496307589e-08,
+      "logits/chosen": -0.7781413197517395,
+      "logits/rejected": -0.5991483330726624,
+      "logps/chosen": -4.574313163757324,
+      "logps/rejected": -5.172600746154785,
+      "loss": 0.051,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.574313163757324,
+      "rewards/margins": 0.5982874631881714,
+      "rewards/rejected": -5.172600746154785,
+      "sft_loss": 4.223634243011475,
+      "step": 5105
+    },
+    {
+      "epoch": 2.7349055025924067,
+      "grad_norm": 0.40995838518576144,
+      "learning_rate": 2.349013410399653e-08,
+      "logits/chosen": -0.769730269908905,
+      "logits/rejected": -0.6913172006607056,
+      "logps/chosen": -4.531946182250977,
+      "logps/rejected": -5.277296543121338,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.531946182250977,
+      "rewards/margins": 0.7453504800796509,
+      "rewards/rejected": -5.277296543121338,
+      "sft_loss": 4.089993476867676,
+      "step": 5110
+    },
+    {
+      "epoch": 2.7375815353738084,
+      "grad_norm": 0.4386421790875959,
+      "learning_rate": 2.3020696345725954e-08,
+      "logits/chosen": -0.8437735438346863,
+      "logits/rejected": -0.6403151750564575,
+      "logps/chosen": -4.284999847412109,
+      "logps/rejected": -5.3409104347229,
+      "loss": 0.0495,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.284999847412109,
+      "rewards/margins": 1.0559107065200806,
+      "rewards/rejected": -5.3409104347229,
+      "sft_loss": 4.055171966552734,
+      "step": 5115
+    },
+    {
+      "epoch": 2.7402575681552097,
+      "grad_norm": 0.6096353214247594,
+      "learning_rate": 2.2555886242751398e-08,
+      "logits/chosen": -0.7667422294616699,
+      "logits/rejected": -0.7132569551467896,
+      "logps/chosen": -4.329689025878906,
+      "logps/rejected": -5.106996059417725,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -4.329689025878906,
+      "rewards/margins": 0.7773071527481079,
+      "rewards/rejected": -5.106996059417725,
+      "sft_loss": 4.15250825881958,
+      "step": 5120
+    },
+    {
+      "epoch": 2.7429336009366114,
+      "grad_norm": 0.739154952389392,
+      "learning_rate": 2.2095708304662453e-08,
+      "logits/chosen": -0.8948475122451782,
+      "logits/rejected": -0.6626867055892944,
+      "logps/chosen": -4.5080671310424805,
+      "logps/rejected": -5.133249282836914,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.5080671310424805,
+      "rewards/margins": 0.6251822710037231,
+      "rewards/rejected": -5.133249282836914,
+      "sft_loss": 4.2104692459106445,
+      "step": 5125
+    },
+    {
+      "epoch": 2.745609633718013,
+      "grad_norm": 0.40146489525306356,
+      "learning_rate": 2.16401669961076e-08,
+      "logits/chosen": -0.8997477293014526,
+      "logits/rejected": -0.6744705438613892,
+      "logps/chosen": -4.385453701019287,
+      "logps/rejected": -5.082536220550537,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.385453701019287,
+      "rewards/margins": 0.6970824003219604,
+      "rewards/rejected": -5.082536220550537,
+      "sft_loss": 4.138113975524902,
+      "step": 5130
+    },
+    {
+      "epoch": 2.748285666499415,
+      "grad_norm": 0.5550146219227535,
+      "learning_rate": 2.1189266736750532e-08,
+      "logits/chosen": -0.6366299390792847,
+      "logits/rejected": -0.6059743165969849,
+      "logps/chosen": -4.567104816436768,
+      "logps/rejected": -5.097182273864746,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.567104816436768,
+      "rewards/margins": 0.5300775766372681,
+      "rewards/rejected": -5.097182273864746,
+      "sft_loss": 4.227593898773193,
+      "step": 5135
+    },
+    {
+      "epoch": 2.750961699280816,
+      "grad_norm": 0.7238548964288543,
+      "learning_rate": 2.0743011901227623e-08,
+      "logits/chosen": -0.7072452902793884,
+      "logits/rejected": -0.5823957920074463,
+      "logps/chosen": -4.160729885101318,
+      "logps/rejected": -4.983979225158691,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.160729885101318,
+      "rewards/margins": 0.8232491612434387,
+      "rewards/rejected": -4.983979225158691,
+      "sft_loss": 3.919672727584839,
+      "step": 5140
+    },
+    {
+      "epoch": 2.753637732062218,
+      "grad_norm": 0.6313164538400142,
+      "learning_rate": 2.030140681910508e-08,
+      "logits/chosen": -0.7149503827095032,
+      "logits/rejected": -0.5912491083145142,
+      "logps/chosen": -4.391037464141846,
+      "logps/rejected": -5.0824480056762695,
+      "loss": 0.051,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.391037464141846,
+      "rewards/margins": 0.6914108991622925,
+      "rewards/rejected": -5.0824480056762695,
+      "sft_loss": 4.110098838806152,
+      "step": 5145
+    },
+    {
+      "epoch": 2.756313764843619,
+      "grad_norm": 0.36572611483915707,
+      "learning_rate": 1.986445577483753e-08,
+      "logits/chosen": -0.7998876571655273,
+      "logits/rejected": -0.6524479985237122,
+      "logps/chosen": -4.311178207397461,
+      "logps/rejected": -5.040547847747803,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.311178207397461,
+      "rewards/margins": 0.7293696999549866,
+      "rewards/rejected": -5.040547847747803,
+      "sft_loss": 4.074983596801758,
+      "step": 5150
+    },
+    {
+      "epoch": 2.758989797625021,
+      "grad_norm": 0.40350920608337093,
+      "learning_rate": 1.9432163007725765e-08,
+      "logits/chosen": -0.8454571962356567,
+      "logits/rejected": -0.7627595663070679,
+      "logps/chosen": -4.490768909454346,
+      "logps/rejected": -5.0736775398254395,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.490768909454346,
+      "rewards/margins": 0.5829084515571594,
+      "rewards/rejected": -5.0736775398254395,
+      "sft_loss": 4.289803504943848,
+      "step": 5155
+    },
+    {
+      "epoch": 2.7616658304064226,
+      "grad_norm": 0.35743425580419486,
+      "learning_rate": 1.9004532711876297e-08,
+      "logits/chosen": -0.7591687440872192,
+      "logits/rejected": -0.7784875631332397,
+      "logps/chosen": -4.46417760848999,
+      "logps/rejected": -5.038629531860352,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.46417760848999,
+      "rewards/margins": 0.5744518637657166,
+      "rewards/rejected": -5.038629531860352,
+      "sft_loss": 4.209836483001709,
+      "step": 5160
+    },
+    {
+      "epoch": 2.7643418631878243,
+      "grad_norm": 0.40974843728329524,
+      "learning_rate": 1.8581569036159928e-08,
+      "logits/chosen": -0.7757205367088318,
+      "logits/rejected": -0.5620787739753723,
+      "logps/chosen": -4.352284908294678,
+      "logps/rejected": -5.135004997253418,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.352284908294678,
+      "rewards/margins": 0.7827197909355164,
+      "rewards/rejected": -5.135004997253418,
+      "sft_loss": 4.145803928375244,
+      "step": 5165
+    },
+    {
+      "epoch": 2.7670178959692255,
+      "grad_norm": 0.38651979743253223,
+      "learning_rate": 1.8163276084172285e-08,
+      "logits/chosen": -0.7541450262069702,
+      "logits/rejected": -0.6550502181053162,
+      "logps/chosen": -4.484699249267578,
+      "logps/rejected": -5.219845771789551,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.484699249267578,
+      "rewards/margins": 0.7351458072662354,
+      "rewards/rejected": -5.219845771789551,
+      "sft_loss": 4.190016746520996,
+      "step": 5170
+    },
+    {
+      "epoch": 2.7696939287506273,
+      "grad_norm": 0.4746599739406412,
+      "learning_rate": 1.7749657914193194e-08,
+      "logits/chosen": -0.7308141589164734,
+      "logits/rejected": -0.6888738870620728,
+      "logps/chosen": -4.515814304351807,
+      "logps/rejected": -5.2973856925964355,
+      "loss": 0.0499,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.515814304351807,
+      "rewards/margins": 0.7815715074539185,
+      "rewards/rejected": -5.2973856925964355,
+      "sft_loss": 4.202316761016846,
+      "step": 5175
+    },
+    {
+      "epoch": 2.7723699615320285,
+      "grad_norm": 0.4964614722849225,
+      "learning_rate": 1.7340718539148203e-08,
+      "logits/chosen": -0.6771430969238281,
+      "logits/rejected": -0.6231086850166321,
+      "logps/chosen": -4.451423168182373,
+      "logps/rejected": -5.105251789093018,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.451423168182373,
+      "rewards/margins": 0.653827965259552,
+      "rewards/rejected": -5.105251789093018,
+      "sft_loss": 4.232422351837158,
+      "step": 5180
+    },
+    {
+      "epoch": 2.7750459943134302,
+      "grad_norm": 0.38468316076352804,
+      "learning_rate": 1.6936461926568724e-08,
+      "logits/chosen": -0.7071312665939331,
+      "logits/rejected": -0.5591712594032288,
+      "logps/chosen": -4.469587802886963,
+      "logps/rejected": -5.192925453186035,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.469587802886963,
+      "rewards/margins": 0.7233376502990723,
+      "rewards/rejected": -5.192925453186035,
+      "sft_loss": 4.177231788635254,
+      "step": 5185
+    },
+    {
+      "epoch": 2.777722027094832,
+      "grad_norm": 0.44091666285478454,
+      "learning_rate": 1.6536891998554346e-08,
+      "logits/chosen": -0.8431358337402344,
+      "logits/rejected": -0.6786155700683594,
+      "logps/chosen": -4.418313980102539,
+      "logps/rejected": -5.12216854095459,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.418313980102539,
+      "rewards/margins": 0.7038545608520508,
+      "rewards/rejected": -5.12216854095459,
+      "sft_loss": 4.129395961761475,
+      "step": 5190
+    },
+    {
+      "epoch": 2.7803980598762337,
+      "grad_norm": 0.4200872795921329,
+      "learning_rate": 1.6142012631734093e-08,
+      "logits/chosen": -0.6793020963668823,
+      "logits/rejected": -0.5708988904953003,
+      "logps/chosen": -4.42996883392334,
+      "logps/rejected": -5.1513671875,
+      "loss": 0.0499,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.42996883392334,
+      "rewards/margins": 0.7213989496231079,
+      "rewards/rejected": -5.1513671875,
+      "sft_loss": 4.167457580566406,
+      "step": 5195
+    },
+    {
+      "epoch": 2.783074092657635,
+      "grad_norm": 0.3583087592611993,
+      "learning_rate": 1.575182765722949e-08,
+      "logits/chosen": -0.8591778874397278,
+      "logits/rejected": -0.6734490990638733,
+      "logps/chosen": -4.4382429122924805,
+      "logps/rejected": -5.177542209625244,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.4382429122924805,
+      "rewards/margins": 0.7392994165420532,
+      "rewards/rejected": -5.177542209625244,
+      "sft_loss": 4.069396018981934,
+      "step": 5200
+    },
+    {
+      "epoch": 2.783074092657635,
+      "eval_logits/chosen": -0.2376498281955719,
+      "eval_logits/rejected": -0.15155045688152313,
+      "eval_logps/chosen": -4.39319372177124,
+      "eval_logps/rejected": -5.099259376525879,
+      "eval_loss": 0.05041056498885155,
+      "eval_rewards/accuracies": 0.6854599118232727,
+      "eval_rewards/chosen": -4.39319372177124,
+      "eval_rewards/margins": 0.70606529712677,
+      "eval_rewards/rejected": -5.099259376525879,
+      "eval_runtime": 43.6092,
+      "eval_samples_per_second": 30.842,
+      "eval_sft_loss": 4.013496398925781,
+      "eval_steps_per_second": 7.728,
+      "step": 5200
+    },
+    {
+      "epoch": 2.7857501254390367,
+      "grad_norm": 0.6691376618275345,
+      "learning_rate": 1.536634086061672e-08,
+      "logits/chosen": -0.7162854075431824,
+      "logits/rejected": -0.7120442986488342,
+      "logps/chosen": -4.272818088531494,
+      "logps/rejected": -5.1587724685668945,
+      "loss": 0.0498,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.272818088531494,
+      "rewards/margins": 0.8859542012214661,
+      "rewards/rejected": -5.1587724685668945,
+      "sft_loss": 4.018543243408203,
+      "step": 5205
+    },
+    {
+      "epoch": 2.788426158220438,
+      "grad_norm": 0.6331576336219715,
+      "learning_rate": 1.4985555981890495e-08,
+      "logits/chosen": -0.770778477191925,
+      "logits/rejected": -0.6682685017585754,
+      "logps/chosen": -4.637447357177734,
+      "logps/rejected": -5.280395030975342,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.637447357177734,
+      "rewards/margins": 0.6429480314254761,
+      "rewards/rejected": -5.280395030975342,
+      "sft_loss": 4.285575866699219,
+      "step": 5210
+    },
+    {
+      "epoch": 2.7911021910018396,
+      "grad_norm": 0.45844722316500985,
+      "learning_rate": 1.4609476715427226e-08,
+      "logits/chosen": -0.7665554285049438,
+      "logits/rejected": -0.7006998658180237,
+      "logps/chosen": -4.294949531555176,
+      "logps/rejected": -5.111362457275391,
+      "loss": 0.05,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.294949531555176,
+      "rewards/margins": 0.8164127469062805,
+      "rewards/rejected": -5.111362457275391,
+      "sft_loss": 4.0890374183654785,
+      "step": 5215
+    },
+    {
+      "epoch": 2.7937782237832414,
+      "grad_norm": 0.46167473933351355,
+      "learning_rate": 1.4238106709949792e-08,
+      "logits/chosen": -0.7603719234466553,
+      "logits/rejected": -0.704023003578186,
+      "logps/chosen": -4.407052993774414,
+      "logps/rejected": -5.269985198974609,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.407052993774414,
+      "rewards/margins": 0.8629329800605774,
+      "rewards/rejected": -5.269985198974609,
+      "sft_loss": 4.130857467651367,
+      "step": 5220
+    },
+    {
+      "epoch": 2.796454256564643,
+      "grad_norm": 0.6465361159968553,
+      "learning_rate": 1.3871449568491511e-08,
+      "logits/chosen": -0.6923054456710815,
+      "logits/rejected": -0.5152607560157776,
+      "logps/chosen": -4.521544456481934,
+      "logps/rejected": -5.11342716217041,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.521544456481934,
+      "rewards/margins": 0.5918827056884766,
+      "rewards/rejected": -5.11342716217041,
+      "sft_loss": 4.219715118408203,
+      "step": 5225
+    },
+    {
+      "epoch": 2.7991302893460444,
+      "grad_norm": 0.5335497897965382,
+      "learning_rate": 1.3509508848361606e-08,
+      "logits/chosen": -0.8408746719360352,
+      "logits/rejected": -0.7014719843864441,
+      "logps/chosen": -4.4506120681762695,
+      "logps/rejected": -5.232138633728027,
+      "loss": 0.0495,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.4506120681762695,
+      "rewards/margins": 0.7815271615982056,
+      "rewards/rejected": -5.232138633728027,
+      "sft_loss": 4.041218280792236,
+      "step": 5230
+    },
+    {
+      "epoch": 2.801806322127446,
+      "grad_norm": 0.4340640550491573,
+      "learning_rate": 1.3152288061110517e-08,
+      "logits/chosen": -0.8067764043807983,
+      "logits/rejected": -0.6543148159980774,
+      "logps/chosen": -4.338916778564453,
+      "logps/rejected": -5.174741744995117,
+      "loss": 0.0499,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -4.338916778564453,
+      "rewards/margins": 0.8358249664306641,
+      "rewards/rejected": -5.174741744995117,
+      "sft_loss": 4.055007457733154,
+      "step": 5235
+    },
+    {
+      "epoch": 2.804482354908848,
+      "grad_norm": 0.4702838319568216,
+      "learning_rate": 1.2799790672495814e-08,
+      "logits/chosen": -0.8347498178482056,
+      "logits/rejected": -0.6073392629623413,
+      "logps/chosen": -4.421563148498535,
+      "logps/rejected": -5.192359447479248,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.421563148498535,
+      "rewards/margins": 0.7707957029342651,
+      "rewards/rejected": -5.192359447479248,
+      "sft_loss": 4.199678897857666,
+      "step": 5240
+    },
+    {
+      "epoch": 2.807158387690249,
+      "grad_norm": 0.40117546884872646,
+      "learning_rate": 1.2452020102448835e-08,
+      "logits/chosen": -0.7439224123954773,
+      "logits/rejected": -0.6960216760635376,
+      "logps/chosen": -4.4497785568237305,
+      "logps/rejected": -5.058270454406738,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.4497785568237305,
+      "rewards/margins": 0.6084924340248108,
+      "rewards/rejected": -5.058270454406738,
+      "sft_loss": 4.284776210784912,
+      "step": 5245
+    },
+    {
+      "epoch": 2.8098344204716508,
+      "grad_norm": 0.4585374000452793,
+      "learning_rate": 1.2108979725041103e-08,
+      "logits/chosen": -0.8621365427970886,
+      "logits/rejected": -0.7040331363677979,
+      "logps/chosen": -4.3691277503967285,
+      "logps/rejected": -5.159308910369873,
+      "loss": 0.051,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.3691277503967285,
+      "rewards/margins": 0.7901814579963684,
+      "rewards/rejected": -5.159308910369873,
+      "sft_loss": 4.125119686126709,
+      "step": 5250
+    },
+    {
+      "epoch": 2.8125104532530525,
+      "grad_norm": 0.5149332418443064,
+      "learning_rate": 1.1770672868451958e-08,
+      "logits/chosen": -0.786795973777771,
+      "logits/rejected": -0.5567874908447266,
+      "logps/chosen": -4.3395161628723145,
+      "logps/rejected": -5.155367374420166,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.3395161628723145,
+      "rewards/margins": 0.8158513307571411,
+      "rewards/rejected": -5.155367374420166,
+      "sft_loss": 4.167263984680176,
+      "step": 5255
+    },
+    {
+      "epoch": 2.8151864860344538,
+      "grad_norm": 0.4589040615922559,
+      "learning_rate": 1.1437102814935872e-08,
+      "logits/chosen": -0.7088602781295776,
+      "logits/rejected": -0.6603358387947083,
+      "logps/chosen": -4.456506252288818,
+      "logps/rejected": -5.139347076416016,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.456506252288818,
+      "rewards/margins": 0.6828408241271973,
+      "rewards/rejected": -5.139347076416016,
+      "sft_loss": 4.205148220062256,
+      "step": 5260
+    },
+    {
+      "epoch": 2.8178625188158555,
+      "grad_norm": 0.41054547983463774,
+      "learning_rate": 1.1108272800791018e-08,
+      "logits/chosen": -0.8841502070426941,
+      "logits/rejected": -0.6486460566520691,
+      "logps/chosen": -4.255162239074707,
+      "logps/rejected": -5.279638290405273,
+      "loss": 0.0492,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -4.255162239074707,
+      "rewards/margins": 1.024476408958435,
+      "rewards/rejected": -5.279638290405273,
+      "sft_loss": 4.01407527923584,
+      "step": 5265
+    },
+    {
+      "epoch": 2.820538551597257,
+      "grad_norm": 0.3870327970812567,
+      "learning_rate": 1.078418601632769e-08,
+      "logits/chosen": -0.72391676902771,
+      "logits/rejected": -0.5991867780685425,
+      "logps/chosen": -4.367644309997559,
+      "logps/rejected": -5.057244777679443,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.367644309997559,
+      "rewards/margins": 0.689600944519043,
+      "rewards/rejected": -5.057244777679443,
+      "sft_loss": 4.147953033447266,
+      "step": 5270
+    },
+    {
+      "epoch": 2.8232145843786585,
+      "grad_norm": 0.49824908401362694,
+      "learning_rate": 1.0464845605837159e-08,
+      "logits/chosen": -0.729150116443634,
+      "logits/rejected": -0.6012585163116455,
+      "logps/chosen": -4.400598049163818,
+      "logps/rejected": -5.120515823364258,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.400598049163818,
+      "rewards/margins": 0.7199177742004395,
+      "rewards/rejected": -5.120515823364258,
+      "sft_loss": 4.194445610046387,
+      "step": 5275
+    },
+    {
+      "epoch": 2.82589061716006,
+      "grad_norm": 0.37218461921657003,
+      "learning_rate": 1.0150254667561642e-08,
+      "logits/chosen": -0.7861107587814331,
+      "logits/rejected": -0.5771303176879883,
+      "logps/chosen": -4.436859130859375,
+      "logps/rejected": -5.288681507110596,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.436859130859375,
+      "rewards/margins": 0.851822018623352,
+      "rewards/rejected": -5.288681507110596,
+      "sft_loss": 4.093539237976074,
+      "step": 5280
+    },
+    {
+      "epoch": 2.828566649941462,
+      "grad_norm": 0.37692200660419406,
+      "learning_rate": 9.840416253663719e-09,
+      "logits/chosen": -0.8008987307548523,
+      "logits/rejected": -0.6927303075790405,
+      "logps/chosen": -4.387301445007324,
+      "logps/rejected": -5.200824737548828,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.387301445007324,
+      "rewards/margins": 0.8135232925415039,
+      "rewards/rejected": -5.200824737548828,
+      "sft_loss": 4.164353847503662,
+      "step": 5285
+    },
+    {
+      "epoch": 2.8312426827228636,
+      "grad_norm": 0.43110480795076206,
+      "learning_rate": 9.535333370197074e-09,
+      "logits/chosen": -0.7683074474334717,
+      "logits/rejected": -0.6206791996955872,
+      "logps/chosen": -4.306056976318359,
+      "logps/rejected": -5.059770107269287,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.306056976318359,
+      "rewards/margins": 0.7537132501602173,
+      "rewards/rejected": -5.059770107269287,
+      "sft_loss": 4.051373481750488,
+      "step": 5290
+    },
+    {
+      "epoch": 2.833918715504265,
+      "grad_norm": 0.39348380949619655,
+      "learning_rate": 9.23500897707713e-09,
+      "logits/chosen": -0.8012930750846863,
+      "logits/rejected": -0.6044802069664001,
+      "logps/chosen": -4.543567180633545,
+      "logps/rejected": -5.1686201095581055,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.543567180633545,
+      "rewards/margins": 0.6250527501106262,
+      "rewards/rejected": -5.1686201095581055,
+      "sft_loss": 4.221306800842285,
+      "step": 5295
+    },
+    {
+      "epoch": 2.8365947482856666,
+      "grad_norm": 0.6265290893699348,
+      "learning_rate": 8.939445988052574e-09,
+      "logits/chosen": -0.7233133912086487,
+      "logits/rejected": -0.6620453596115112,
+      "logps/chosen": -4.482752799987793,
+      "logps/rejected": -5.183930397033691,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.482752799987793,
+      "rewards/margins": 0.7011777758598328,
+      "rewards/rejected": -5.183930397033691,
+      "sft_loss": 4.141602516174316,
+      "step": 5300
+    },
+    {
+      "epoch": 2.839270781067068,
+      "grad_norm": 0.5143935043837382,
+      "learning_rate": 8.648647270676656e-09,
+      "logits/chosen": -0.6966943740844727,
+      "logits/rejected": -0.601551353931427,
+      "logps/chosen": -4.2645697593688965,
+      "logps/rejected": -5.0711989402771,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.2645697593688965,
+      "rewards/margins": 0.8066291809082031,
+      "rewards/rejected": -5.0711989402771,
+      "sft_loss": 3.9466934204101562,
+      "step": 5305
+    },
+    {
+      "epoch": 2.8419468138484696,
+      "grad_norm": 0.5979309433728134,
+      "learning_rate": 8.362615646279991e-09,
+      "logits/chosen": -0.918570876121521,
+      "logits/rejected": -0.6350168585777283,
+      "logps/chosen": -4.436649322509766,
+      "logps/rejected": -5.3039045333862305,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.436649322509766,
+      "rewards/margins": 0.8672553300857544,
+      "rewards/rejected": -5.3039045333862305,
+      "sft_loss": 4.19779109954834,
+      "step": 5310
+    },
+    {
+      "epoch": 2.8446228466298713,
+      "grad_norm": 0.5780580944954342,
+      "learning_rate": 8.081353889942466e-09,
+      "logits/chosen": -0.6806483268737793,
+      "logits/rejected": -0.500455915927887,
+      "logps/chosen": -4.434041500091553,
+      "logps/rejected": -5.082981586456299,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.434041500091553,
+      "rewards/margins": 0.6489400267601013,
+      "rewards/rejected": -5.082981586456299,
+      "sft_loss": 4.191941261291504,
+      "step": 5315
+    },
+    {
+      "epoch": 2.847298879411273,
+      "grad_norm": 0.3981472388972543,
+      "learning_rate": 7.804864730467042e-09,
+      "logits/chosen": -0.6645959615707397,
+      "logits/rejected": -0.6632333397865295,
+      "logps/chosen": -4.3376359939575195,
+      "logps/rejected": -5.054964542388916,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.3376359939575195,
+      "rewards/margins": 0.7173280715942383,
+      "rewards/rejected": -5.054964542388916,
+      "sft_loss": 4.088263988494873,
+      "step": 5320
+    },
+    {
+      "epoch": 2.8499749121926743,
+      "grad_norm": 0.5496295929814721,
+      "learning_rate": 7.533150850352665e-09,
+      "logits/chosen": -0.6918113231658936,
+      "logits/rejected": -0.5065929889678955,
+      "logps/chosen": -4.538696765899658,
+      "logps/rejected": -5.3134541511535645,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.538696765899658,
+      "rewards/margins": 0.7747570872306824,
+      "rewards/rejected": -5.3134541511535645,
+      "sft_loss": 4.243317604064941,
+      "step": 5325
+    },
+    {
+      "epoch": 2.852650944974076,
+      "grad_norm": 0.5330143026502275,
+      "learning_rate": 7.2662148857686175e-09,
+      "logits/chosen": -0.6605895757675171,
+      "logits/rejected": -0.5792001485824585,
+      "logps/chosen": -4.514766693115234,
+      "logps/rejected": -5.122227668762207,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.514766693115234,
+      "rewards/margins": 0.6074615120887756,
+      "rewards/rejected": -5.122227668762207,
+      "sft_loss": 4.139904022216797,
+      "step": 5330
+    },
+    {
+      "epoch": 2.8553269777554773,
+      "grad_norm": 0.4151414572497506,
+      "learning_rate": 7.0040594265287635e-09,
+      "logits/chosen": -0.5867056846618652,
+      "logits/rejected": -0.68376225233078,
+      "logps/chosen": -4.496993064880371,
+      "logps/rejected": -4.990506172180176,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.496993064880371,
+      "rewards/margins": 0.49351271986961365,
+      "rewards/rejected": -4.990506172180176,
+      "sft_loss": 4.215310096740723,
+      "step": 5335
+    },
+    {
+      "epoch": 2.858003010536879,
+      "grad_norm": 0.42968389745954066,
+      "learning_rate": 6.746687016066566e-09,
+      "logits/chosen": -0.6900774240493774,
+      "logits/rejected": -0.6579724550247192,
+      "logps/chosen": -4.339878559112549,
+      "logps/rejected": -4.979432106018066,
+      "loss": 0.052,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.339878559112549,
+      "rewards/margins": 0.6395532488822937,
+      "rewards/rejected": -4.979432106018066,
+      "sft_loss": 4.117095470428467,
+      "step": 5340
+    },
+    {
+      "epoch": 2.8606790433182807,
+      "grad_norm": 0.5137218352574986,
+      "learning_rate": 6.494100151410276e-09,
+      "logits/chosen": -0.897404670715332,
+      "logits/rejected": -0.6970912218093872,
+      "logps/chosen": -4.338336944580078,
+      "logps/rejected": -5.060499668121338,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.338336944580078,
+      "rewards/margins": 0.7221625447273254,
+      "rewards/rejected": -5.060499668121338,
+      "sft_loss": 4.119289875030518,
+      "step": 5345
+    },
+    {
+      "epoch": 2.8633550760996824,
+      "grad_norm": 0.9439486911375388,
+      "learning_rate": 6.246301283158728e-09,
+      "logits/chosen": -0.6498326063156128,
+      "logits/rejected": -0.6877025365829468,
+      "logps/chosen": -4.418894290924072,
+      "logps/rejected": -5.056708812713623,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.418894290924072,
+      "rewards/margins": 0.6378144025802612,
+      "rewards/rejected": -5.056708812713623,
+      "sft_loss": 4.190070152282715,
+      "step": 5350
+    },
+    {
+      "epoch": 2.8660311088810837,
+      "grad_norm": 0.5415123457240395,
+      "learning_rate": 6.0032928154576944e-09,
+      "logits/chosen": -0.7772369980812073,
+      "logits/rejected": -0.7227431535720825,
+      "logps/chosen": -4.4435014724731445,
+      "logps/rejected": -5.072482585906982,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.4435014724731445,
+      "rewards/margins": 0.6289812326431274,
+      "rewards/rejected": -5.072482585906982,
+      "sft_loss": 4.237886905670166,
+      "step": 5355
+    },
+    {
+      "epoch": 2.8687071416624854,
+      "grad_norm": 0.6985459324311037,
+      "learning_rate": 5.76507710597629e-09,
+      "logits/chosen": -0.8166434168815613,
+      "logits/rejected": -0.6002271175384521,
+      "logps/chosen": -4.332741737365723,
+      "logps/rejected": -5.127420425415039,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.332741737365723,
+      "rewards/margins": 0.7946786284446716,
+      "rewards/rejected": -5.127420425415039,
+      "sft_loss": 4.032086372375488,
+      "step": 5360
+    },
+    {
+      "epoch": 2.8713831744438867,
+      "grad_norm": 0.3729066256025123,
+      "learning_rate": 5.531656465884438e-09,
+      "logits/chosen": -0.8140283823013306,
+      "logits/rejected": -0.6739929914474487,
+      "logps/chosen": -4.36183500289917,
+      "logps/rejected": -5.198390960693359,
+      "loss": 0.0496,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.36183500289917,
+      "rewards/margins": 0.8365559577941895,
+      "rewards/rejected": -5.198390960693359,
+      "sft_loss": 4.06744384765625,
+      "step": 5365
+    },
+    {
+      "epoch": 2.8740592072252884,
+      "grad_norm": 0.7101122740083048,
+      "learning_rate": 5.303033159830217e-09,
+      "logits/chosen": -0.6431114673614502,
+      "logits/rejected": -0.6394237279891968,
+      "logps/chosen": -4.446755409240723,
+      "logps/rejected": -4.919232368469238,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.446755409240723,
+      "rewards/margins": 0.4724767804145813,
+      "rewards/rejected": -4.919232368469238,
+      "sft_loss": 4.0875115394592285,
+      "step": 5370
+    },
+    {
+      "epoch": 2.87673524000669,
+      "grad_norm": 0.4761441037980727,
+      "learning_rate": 5.079209405917939e-09,
+      "logits/chosen": -0.7186871767044067,
+      "logits/rejected": -0.6644418835639954,
+      "logps/chosen": -4.400277614593506,
+      "logps/rejected": -5.278224945068359,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.400277614593506,
+      "rewards/margins": 0.8779473304748535,
+      "rewards/rejected": -5.278224945068359,
+      "sft_loss": 4.172907829284668,
+      "step": 5375
+    },
+    {
+      "epoch": 2.879411272788092,
+      "grad_norm": 0.44965344183186384,
+      "learning_rate": 4.860187375686664e-09,
+      "logits/chosen": -0.840808093547821,
+      "logits/rejected": -0.5630152821540833,
+      "logps/chosen": -4.355297088623047,
+      "logps/rejected": -5.174862384796143,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.355297088623047,
+      "rewards/margins": 0.8195658922195435,
+      "rewards/rejected": -5.174862384796143,
+      "sft_loss": 4.117367744445801,
+      "step": 5380
+    },
+    {
+      "epoch": 2.882087305569493,
+      "grad_norm": 0.48662423395902865,
+      "learning_rate": 4.64596919408905e-09,
+      "logits/chosen": -0.6709896922111511,
+      "logits/rejected": -0.6175445914268494,
+      "logps/chosen": -4.350368976593018,
+      "logps/rejected": -4.9599738121032715,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.350368976593018,
+      "rewards/margins": 0.6096046566963196,
+      "rewards/rejected": -4.9599738121032715,
+      "sft_loss": 4.05443811416626,
+      "step": 5385
+    },
+    {
+      "epoch": 2.884763338350895,
+      "grad_norm": 0.5097574203945324,
+      "learning_rate": 4.436556939470814e-09,
+      "logits/chosen": -0.8031808733940125,
+      "logits/rejected": -0.6307480335235596,
+      "logps/chosen": -4.699088096618652,
+      "logps/rejected": -5.155211448669434,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.699088096618652,
+      "rewards/margins": 0.456122487783432,
+      "rewards/rejected": -5.155211448669434,
+      "sft_loss": 4.452386856079102,
+      "step": 5390
+    },
+    {
+      "epoch": 2.887439371132296,
+      "grad_norm": 0.4438377766535619,
+      "learning_rate": 4.23195264355064e-09,
+      "logits/chosen": -0.935401439666748,
+      "logits/rejected": -0.6709798574447632,
+      "logps/chosen": -4.372194766998291,
+      "logps/rejected": -5.0931525230407715,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.372194766998291,
+      "rewards/margins": 0.7209590077400208,
+      "rewards/rejected": -5.0931525230407715,
+      "sft_loss": 4.168568134307861,
+      "step": 5395
+    },
+    {
+      "epoch": 2.890115403913698,
+      "grad_norm": 0.3898996829524812,
+      "learning_rate": 4.032158291400245e-09,
+      "logits/chosen": -0.7936094403266907,
+      "logits/rejected": -0.5639082193374634,
+      "logps/chosen": -4.286824703216553,
+      "logps/rejected": -5.411582946777344,
+      "loss": 0.0481,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.286824703216553,
+      "rewards/margins": 1.1247583627700806,
+      "rewards/rejected": -5.411582946777344,
+      "sft_loss": 3.914653778076172,
+      "step": 5400
+    },
+    {
+      "epoch": 2.8927914366950995,
+      "grad_norm": 0.37144975755492143,
+      "learning_rate": 3.837175821425398e-09,
+      "logits/chosen": -0.6788768172264099,
+      "logits/rejected": -0.6351056098937988,
+      "logps/chosen": -4.575131416320801,
+      "logps/rejected": -5.201429843902588,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.575131416320801,
+      "rewards/margins": 0.6262980103492737,
+      "rewards/rejected": -5.201429843902588,
+      "sft_loss": 4.266793251037598,
+      "step": 5405
+    },
+    {
+      "epoch": 2.8954674694765012,
+      "grad_norm": 0.4819502320454787,
+      "learning_rate": 3.6470071253467683e-09,
+      "logits/chosen": -0.7794539332389832,
+      "logits/rejected": -0.6866748332977295,
+      "logps/chosen": -4.382658958435059,
+      "logps/rejected": -5.218075752258301,
+      "loss": 0.0499,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.382658958435059,
+      "rewards/margins": 0.8354169130325317,
+      "rewards/rejected": -5.218075752258301,
+      "sft_loss": 4.06662654876709,
+      "step": 5410
+    },
+    {
+      "epoch": 2.8981435022579025,
+      "grad_norm": 0.4675477289758297,
+      "learning_rate": 3.461654048181939e-09,
+      "logits/chosen": -0.8172122836112976,
+      "logits/rejected": -0.5970250964164734,
+      "logps/chosen": -4.448482513427734,
+      "logps/rejected": -5.097105979919434,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.448482513427734,
+      "rewards/margins": 0.6486231088638306,
+      "rewards/rejected": -5.097105979919434,
+      "sft_loss": 4.25862979888916,
+      "step": 5415
+    },
+    {
+      "epoch": 2.9008195350393042,
+      "grad_norm": 0.481561756284469,
+      "learning_rate": 3.281118388227255e-09,
+      "logits/chosen": -0.7290286421775818,
+      "logits/rejected": -0.6467230319976807,
+      "logps/chosen": -4.468303680419922,
+      "logps/rejected": -5.0093994140625,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.468303680419922,
+      "rewards/margins": 0.5410959124565125,
+      "rewards/rejected": -5.0093994140625,
+      "sft_loss": 4.1863837242126465,
+      "step": 5420
+    },
+    {
+      "epoch": 2.903495567820706,
+      "grad_norm": 0.5204386750591277,
+      "learning_rate": 3.1054018970405048e-09,
+      "logits/chosen": -0.7600063681602478,
+      "logits/rejected": -0.6038556694984436,
+      "logps/chosen": -4.390046119689941,
+      "logps/rejected": -5.161233901977539,
+      "loss": 0.05,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.390046119689941,
+      "rewards/margins": 0.7711877822875977,
+      "rewards/rejected": -5.161233901977539,
+      "sft_loss": 4.058727741241455,
+      "step": 5425
+    },
+    {
+      "epoch": 2.906171600602107,
+      "grad_norm": 0.3750239879457203,
+      "learning_rate": 2.9345062794238207e-09,
+      "logits/chosen": -0.7968858480453491,
+      "logits/rejected": -0.5854495763778687,
+      "logps/chosen": -4.418655872344971,
+      "logps/rejected": -5.176025390625,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.418655872344971,
+      "rewards/margins": 0.757369875907898,
+      "rewards/rejected": -5.176025390625,
+      "sft_loss": 4.210319519042969,
+      "step": 5430
+    },
+    {
+      "epoch": 2.908847633383509,
+      "grad_norm": 0.42673124272737256,
+      "learning_rate": 2.7684331934072492e-09,
+      "logits/chosen": -0.8364018201828003,
+      "logits/rejected": -0.774222731590271,
+      "logps/chosen": -4.359699726104736,
+      "logps/rejected": -5.043485164642334,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.359699726104736,
+      "rewards/margins": 0.6837862133979797,
+      "rewards/rejected": -5.043485164642334,
+      "sft_loss": 4.126735210418701,
+      "step": 5435
+    },
+    {
+      "epoch": 2.9115236661649107,
+      "grad_norm": 0.52408152068924,
+      "learning_rate": 2.6071842502326526e-09,
+      "logits/chosen": -0.8465268015861511,
+      "logits/rejected": -0.6868315935134888,
+      "logps/chosen": -4.515316963195801,
+      "logps/rejected": -5.097006797790527,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.515316963195801,
+      "rewards/margins": 0.5816894173622131,
+      "rewards/rejected": -5.097006797790527,
+      "sft_loss": 4.289347171783447,
+      "step": 5440
+    },
+    {
+      "epoch": 2.9141996989463124,
+      "grad_norm": 0.5496966834501419,
+      "learning_rate": 2.450761014337888e-09,
+      "logits/chosen": -0.5394278168678284,
+      "logits/rejected": -0.48762258887290955,
+      "logps/chosen": -4.465736389160156,
+      "logps/rejected": -5.261734962463379,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.465736389160156,
+      "rewards/margins": 0.7959989905357361,
+      "rewards/rejected": -5.261734962463379,
+      "sft_loss": 4.236559867858887,
+      "step": 5445
+    },
+    {
+      "epoch": 2.9168757317277136,
+      "grad_norm": 0.7378399710882302,
+      "learning_rate": 2.299165003341985e-09,
+      "logits/chosen": -0.5353703498840332,
+      "logits/rejected": -0.4766581058502197,
+      "logps/chosen": -4.363390922546387,
+      "logps/rejected": -5.103070259094238,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.363390922546387,
+      "rewards/margins": 0.739679217338562,
+      "rewards/rejected": -5.103070259094238,
+      "sft_loss": 4.137202262878418,
+      "step": 5450
+    },
+    {
+      "epoch": 2.9195517645091154,
+      "grad_norm": 0.5686744201567238,
+      "learning_rate": 2.1523976880299945e-09,
+      "logits/chosen": -0.8194225430488586,
+      "logits/rejected": -0.6335167288780212,
+      "logps/chosen": -4.4221296310424805,
+      "logps/rejected": -4.990933895111084,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.4221296310424805,
+      "rewards/margins": 0.5688046813011169,
+      "rewards/rejected": -4.990933895111084,
+      "sft_loss": 4.245485782623291,
+      "step": 5455
+    },
+    {
+      "epoch": 2.9222277972905166,
+      "grad_norm": 0.5578072935373403,
+      "learning_rate": 2.010460492339161e-09,
+      "logits/chosen": -0.7293864488601685,
+      "logits/rejected": -0.5793944597244263,
+      "logps/chosen": -4.39241886138916,
+      "logps/rejected": -5.086266994476318,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.39241886138916,
+      "rewards/margins": 0.6938482522964478,
+      "rewards/rejected": -5.086266994476318,
+      "sft_loss": 4.0339789390563965,
+      "step": 5460
+    },
+    {
+      "epoch": 2.9249038300719183,
+      "grad_norm": 0.5421431657374898,
+      "learning_rate": 1.8733547933446614e-09,
+      "logits/chosen": -0.823691189289093,
+      "logits/rejected": -0.5740182995796204,
+      "logps/chosen": -4.491018295288086,
+      "logps/rejected": -5.095160007476807,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.491018295288086,
+      "rewards/margins": 0.6041414737701416,
+      "rewards/rejected": -5.095160007476807,
+      "sft_loss": 4.210727691650391,
+      "step": 5465
+    },
+    {
+      "epoch": 2.92757986285332,
+      "grad_norm": 0.5262232764119357,
+      "learning_rate": 1.7410819212467231e-09,
+      "logits/chosen": -0.7801578640937805,
+      "logits/rejected": -0.7355565428733826,
+      "logps/chosen": -4.562183856964111,
+      "logps/rejected": -5.0614728927612305,
+      "loss": 0.0543,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.562183856964111,
+      "rewards/margins": 0.49928945302963257,
+      "rewards/rejected": -5.0614728927612305,
+      "sft_loss": 4.305178642272949,
+      "step": 5470
+    },
+    {
+      "epoch": 2.9302558956347218,
+      "grad_norm": 0.33633377390369845,
+      "learning_rate": 1.613643159357192e-09,
+      "logits/chosen": -0.6578032374382019,
+      "logits/rejected": -0.7792307734489441,
+      "logps/chosen": -4.518008232116699,
+      "logps/rejected": -5.044106483459473,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.518008232116699,
+      "rewards/margins": 0.5260982513427734,
+      "rewards/rejected": -5.044106483459473,
+      "sft_loss": 4.252413749694824,
+      "step": 5475
+    },
+    {
+      "epoch": 2.932931928416123,
+      "grad_norm": 0.5280603635941091,
+      "learning_rate": 1.4910397440875967e-09,
+      "logits/chosen": -0.7603357434272766,
+      "logits/rejected": -0.6517842411994934,
+      "logps/chosen": -4.375611305236816,
+      "logps/rejected": -5.041906833648682,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.375611305236816,
+      "rewards/margins": 0.6662946939468384,
+      "rewards/rejected": -5.041906833648682,
+      "sft_loss": 4.093026161193848,
+      "step": 5480
+    },
+    {
+      "epoch": 2.9356079611975248,
+      "grad_norm": 0.43302048339241966,
+      "learning_rate": 1.3732728649368253e-09,
+      "logits/chosen": -0.6951078176498413,
+      "logits/rejected": -0.5088014602661133,
+      "logps/chosen": -4.342538356781006,
+      "logps/rejected": -5.075597763061523,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.342538356781006,
+      "rewards/margins": 0.7330597639083862,
+      "rewards/rejected": -5.075597763061523,
+      "sft_loss": 4.080550193786621,
+      "step": 5485
+    },
+    {
+      "epoch": 2.938283993978926,
+      "grad_norm": 0.4333874844955521,
+      "learning_rate": 1.260343664479524e-09,
+      "logits/chosen": -0.7748027443885803,
+      "logits/rejected": -0.7607916593551636,
+      "logps/chosen": -4.558234214782715,
+      "logps/rejected": -5.103475570678711,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.558234214782715,
+      "rewards/margins": 0.5452412962913513,
+      "rewards/rejected": -5.103475570678711,
+      "sft_loss": 4.141097068786621,
+      "step": 5490
+    },
+    {
+      "epoch": 2.9409600267603278,
+      "grad_norm": 0.5357566836246764,
+      "learning_rate": 1.1522532383554384e-09,
+      "logits/chosen": -0.8556427955627441,
+      "logits/rejected": -0.5980334281921387,
+      "logps/chosen": -4.472742557525635,
+      "logps/rejected": -5.171594142913818,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.472742557525635,
+      "rewards/margins": 0.6988515257835388,
+      "rewards/rejected": -5.171594142913818,
+      "sft_loss": 4.282252311706543,
+      "step": 5495
+    },
+    {
+      "epoch": 2.9436360595417295,
+      "grad_norm": 0.298521633055877,
+      "learning_rate": 1.049002635258256e-09,
+      "logits/chosen": -0.6451120972633362,
+      "logits/rejected": -0.5836285352706909,
+      "logps/chosen": -4.546640872955322,
+      "logps/rejected": -5.189908027648926,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.546640872955322,
+      "rewards/margins": 0.6432673335075378,
+      "rewards/rejected": -5.189908027648926,
+      "sft_loss": 4.219212532043457,
+      "step": 5500
+    },
+    {
+      "epoch": 2.946312092323131,
+      "grad_norm": 0.47740541750846544,
+      "learning_rate": 9.505928569258358e-10,
+      "logits/chosen": -0.6692383289337158,
+      "logits/rejected": -0.6824567317962646,
+      "logps/chosen": -4.47029972076416,
+      "logps/rejected": -5.136569499969482,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.47029972076416,
+      "rewards/margins": 0.6662701368331909,
+      "rewards/rejected": -5.136569499969482,
+      "sft_loss": 4.262817859649658,
+      "step": 5505
+    },
+    {
+      "epoch": 2.9489881251045325,
+      "grad_norm": 0.49326359074807213,
+      "learning_rate": 8.57024858130273e-10,
+      "logits/chosen": -0.7504460215568542,
+      "logits/rejected": -0.5905870199203491,
+      "logps/chosen": -4.473368167877197,
+      "logps/rejected": -5.392681121826172,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.473368167877197,
+      "rewards/margins": 0.9193128347396851,
+      "rewards/rejected": -5.392681121826172,
+      "sft_loss": 4.297789096832275,
+      "step": 5510
+    },
+    {
+      "epoch": 2.951664157885934,
+      "grad_norm": 0.39847456506363377,
+      "learning_rate": 7.682995466686826e-10,
+      "logits/chosen": -0.8444706201553345,
+      "logits/rejected": -0.6898127794265747,
+      "logps/chosen": -4.56125545501709,
+      "logps/rejected": -5.338624000549316,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.56125545501709,
+      "rewards/margins": 0.7773683071136475,
+      "rewards/rejected": -5.338624000549316,
+      "sft_loss": 4.119109153747559,
+      "step": 5515
+    },
+    {
+      "epoch": 2.9543401906673354,
+      "grad_norm": 0.4569463801314364,
+      "learning_rate": 6.844177833543741e-10,
+      "logits/chosen": -0.752629816532135,
+      "logits/rejected": -0.7100390791893005,
+      "logps/chosen": -4.399282932281494,
+      "logps/rejected": -5.064263343811035,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.399282932281494,
+      "rewards/margins": 0.6649808287620544,
+      "rewards/rejected": -5.064263343811035,
+      "sft_loss": 4.0922088623046875,
+      "step": 5520
+    },
+    {
+      "epoch": 2.957016223448737,
+      "grad_norm": 0.5095330277701998,
+      "learning_rate": 6.053803820087467e-10,
+      "logits/chosen": -0.7440576553344727,
+      "logits/rejected": -0.5561671257019043,
+      "logps/chosen": -4.5473103523254395,
+      "logps/rejected": -5.268017768859863,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.5473103523254395,
+      "rewards/margins": 0.7207077145576477,
+      "rewards/rejected": -5.268017768859863,
+      "sft_loss": 4.374567985534668,
+      "step": 5525
+    },
+    {
+      "epoch": 2.959692256230139,
+      "grad_norm": 0.5418342484558206,
+      "learning_rate": 5.311881094528514e-10,
+      "logits/chosen": -0.894533634185791,
+      "logits/rejected": -0.6330394744873047,
+      "logps/chosen": -4.468113422393799,
+      "logps/rejected": -5.109396934509277,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.468113422393799,
+      "rewards/margins": 0.6412833333015442,
+      "rewards/rejected": -5.109396934509277,
+      "sft_loss": 4.15500545501709,
+      "step": 5530
+    },
+    {
+      "epoch": 2.9623682890115406,
+      "grad_norm": 0.7560684828912863,
+      "learning_rate": 4.6184168550050806e-10,
+      "logits/chosen": -0.7792457342147827,
+      "logits/rejected": -0.7371629476547241,
+      "logps/chosen": -4.565438270568848,
+      "logps/rejected": -5.105252742767334,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.565438270568848,
+      "rewards/margins": 0.539814829826355,
+      "rewards/rejected": -5.105252742767334,
+      "sft_loss": 4.340537071228027,
+      "step": 5535
+    },
+    {
+      "epoch": 2.965044321792942,
+      "grad_norm": 0.4911869174076437,
+      "learning_rate": 3.973417829510328e-10,
+      "logits/chosen": -0.8998439908027649,
+      "logits/rejected": -0.7843471765518188,
+      "logps/chosen": -4.363260746002197,
+      "logps/rejected": -5.053696632385254,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.363260746002197,
+      "rewards/margins": 0.6904358267784119,
+      "rewards/rejected": -5.053696632385254,
+      "sft_loss": 4.130491256713867,
+      "step": 5540
+    },
+    {
+      "epoch": 2.9677203545743436,
+      "grad_norm": 0.625561234732518,
+      "learning_rate": 3.3768902758274377e-10,
+      "logits/chosen": -0.7709188461303711,
+      "logits/rejected": -0.6807830929756165,
+      "logps/chosen": -4.335400104522705,
+      "logps/rejected": -5.092108726501465,
+      "loss": 0.051,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.335400104522705,
+      "rewards/margins": 0.7567082643508911,
+      "rewards/rejected": -5.092108726501465,
+      "sft_loss": 4.07431173324585,
+      "step": 5545
+    },
+    {
+      "epoch": 2.970396387355745,
+      "grad_norm": 0.40367028372201813,
+      "learning_rate": 2.8288399814691e-10,
+      "logits/chosen": -0.6472212672233582,
+      "logits/rejected": -0.6156342625617981,
+      "logps/chosen": -4.514806270599365,
+      "logps/rejected": -5.145084381103516,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.514806270599365,
+      "rewards/margins": 0.6302779912948608,
+      "rewards/rejected": -5.145084381103516,
+      "sft_loss": 4.210458278656006,
+      "step": 5550
+    },
+    {
+      "epoch": 2.9730724201371466,
+      "grad_norm": 0.38644244219702006,
+      "learning_rate": 2.3292722636220066e-10,
+      "logits/chosen": -0.7775002717971802,
+      "logits/rejected": -0.5841083526611328,
+      "logps/chosen": -4.504853248596191,
+      "logps/rejected": -5.351606845855713,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.504853248596191,
+      "rewards/margins": 0.8467535972595215,
+      "rewards/rejected": -5.351606845855713,
+      "sft_loss": 4.162545680999756,
+      "step": 5555
+    },
+    {
+      "epoch": 2.9757484529185483,
+      "grad_norm": 0.525091240664735,
+      "learning_rate": 1.8781919690946668e-10,
+      "logits/chosen": -0.7250782251358032,
+      "logits/rejected": -0.7407873868942261,
+      "logps/chosen": -4.452993392944336,
+      "logps/rejected": -5.02133846282959,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.452993392944336,
+      "rewards/margins": 0.5683449506759644,
+      "rewards/rejected": -5.02133846282959,
+      "sft_loss": 4.157462120056152,
+      "step": 5560
+    },
+    {
+      "epoch": 2.97842448569995,
+      "grad_norm": 0.46121490745190485,
+      "learning_rate": 1.4756034742696711e-10,
+      "logits/chosen": -0.8873132467269897,
+      "logits/rejected": -0.7985233068466187,
+      "logps/chosen": -4.419460773468018,
+      "logps/rejected": -5.141184329986572,
+      "loss": 0.052,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.419460773468018,
+      "rewards/margins": 0.7217229604721069,
+      "rewards/rejected": -5.141184329986572,
+      "sft_loss": 4.184345245361328,
+      "step": 5565
+    },
+    {
+      "epoch": 2.9811005184813513,
+      "grad_norm": 0.5079173509922432,
+      "learning_rate": 1.12151068506261e-10,
+      "logits/chosen": -0.7679609060287476,
+      "logits/rejected": -0.6635790467262268,
+      "logps/chosen": -4.502461910247803,
+      "logps/rejected": -5.3459858894348145,
+      "loss": 0.0493,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.502461910247803,
+      "rewards/margins": 0.8435236215591431,
+      "rewards/rejected": -5.3459858894348145,
+      "sft_loss": 4.08714485168457,
+      "step": 5570
+    },
+    {
+      "epoch": 2.983776551262753,
+      "grad_norm": 0.6251494874129625,
+      "learning_rate": 8.159170368826629e-11,
+      "logits/chosen": -0.778547465801239,
+      "logits/rejected": -0.6166914701461792,
+      "logps/chosen": -4.4004130363464355,
+      "logps/rejected": -5.13525390625,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.4004130363464355,
+      "rewards/margins": 0.734841525554657,
+      "rewards/rejected": -5.13525390625,
+      "sft_loss": 4.074005603790283,
+      "step": 5575
+    },
+    {
+      "epoch": 2.9864525840441547,
+      "grad_norm": 0.4323529956911697,
+      "learning_rate": 5.588254946015114e-11,
+      "logits/chosen": -0.937720775604248,
+      "logits/rejected": -0.6476465463638306,
+      "logps/chosen": -4.396031856536865,
+      "logps/rejected": -5.199591636657715,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.396031856536865,
+      "rewards/margins": 0.8035598993301392,
+      "rewards/rejected": -5.199591636657715,
+      "sft_loss": 4.089269161224365,
+      "step": 5580
+    },
+    {
+      "epoch": 2.989128616825556,
+      "grad_norm": 0.4034195581734263,
+      "learning_rate": 3.502385525216978e-11,
+      "logits/chosen": -0.8808103799819946,
+      "logits/rejected": -0.7039750814437866,
+      "logps/chosen": -4.423071384429932,
+      "logps/rejected": -5.164847373962402,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.423071384429932,
+      "rewards/margins": 0.7417756915092468,
+      "rewards/rejected": -5.164847373962402,
+      "sft_loss": 4.177689552307129,
+      "step": 5585
+    },
+    {
+      "epoch": 2.9918046496069577,
+      "grad_norm": 0.3570840544619443,
+      "learning_rate": 1.901582343555308e-11,
+      "logits/chosen": -0.7723742127418518,
+      "logits/rejected": -0.709536075592041,
+      "logps/chosen": -4.62722635269165,
+      "logps/rejected": -5.315423488616943,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.62722635269165,
+      "rewards/margins": 0.688197135925293,
+      "rewards/rejected": -5.315423488616943,
+      "sft_loss": 4.275882720947266,
+      "step": 5590
+    },
+    {
+      "epoch": 2.9944806823883594,
+      "grad_norm": 0.4923520610414012,
+      "learning_rate": 7.858609320232634e-12,
+      "logits/chosen": -0.8053900599479675,
+      "logits/rejected": -0.6184626221656799,
+      "logps/chosen": -4.50331974029541,
+      "logps/rejected": -5.1998724937438965,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.50331974029541,
+      "rewards/margins": 0.696553111076355,
+      "rewards/rejected": -5.1998724937438965,
+      "sft_loss": 4.270722389221191,
+      "step": 5595
+    },
+    {
+      "epoch": 2.9971567151697607,
+      "grad_norm": 0.5386939604422419,
+      "learning_rate": 1.5523211535639624e-12,
+      "logits/chosen": -0.8037877082824707,
+      "logits/rejected": -0.6862096190452576,
+      "logps/chosen": -4.483442306518555,
+      "logps/rejected": -5.466944694519043,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.483442306518555,
+      "rewards/margins": 0.983502209186554,
+      "rewards/rejected": -5.466944694519043,
+      "sft_loss": 4.2200493812561035,
+      "step": 5600
+    },
+    {
+      "epoch": 2.9971567151697607,
+      "eval_logits/chosen": -0.3687087595462799,
+      "eval_logits/rejected": -0.2954810559749603,
+      "eval_logps/chosen": -4.423083305358887,
+      "eval_logps/rejected": -5.141829967498779,
+      "eval_loss": 0.05035361275076866,
+      "eval_rewards/accuracies": 0.6862017512321472,
+      "eval_rewards/chosen": -4.423083305358887,
+      "eval_rewards/margins": 0.7187467217445374,
+      "eval_rewards/rejected": -5.141829967498779,
+      "eval_runtime": 44.0385,
+      "eval_samples_per_second": 30.541,
+      "eval_sft_loss": 4.028059959411621,
+      "eval_steps_per_second": 7.652,
+      "step": 5600
+    },
+    {
+      "epoch": 2.999297541394882,
+      "step": 5604,
+      "total_flos": 0.0,
+      "train_loss": 0.0619825580417259,
+      "train_runtime": 31817.9407,
+      "train_samples_per_second": 5.637,
+      "train_steps_per_second": 0.176
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 5604,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 1000000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}